From 7095786f8ea6cc78ee4ccd209b9bbdd6710fae99 Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Sun, 1 Nov 2026 22:05:25 -0500 Subject: [PATCH] refactor(analysis): give IP and TCP reassembly their own foundation.analysis modules (#2581) - analysis/ip.py: ip_datagram parses an IPv4/IPv6 datagram's payload through IPv4.analyze or IPv6.analyze, by its addresses. - analysis/tcp.py: tcp_datagram parses a TCP datagram's payload through TCP.analyze by its ports. Both return None for strict=True's received runs. - IP.submit, TCP.submit: the Deferred packet runs each module's parse, bound to the engine's own __protocol_type__. Exported as dtls_handshake is. - docs: reassembly-integration point 3 holds for every engine; API pages; packet docs name the functions, with the __protocol_type__ exception. - tests: functions called directly; Extractor over PCAP and PCAP-NG, default and custom __protocol_type__; pickle round trips of an unread packet. Behaviour unchanged: a sample-capture differential matches main. --- .../conventions/reassembly-integration.rst | 32 +- .../pcapkit/foundation/analysis/index.rst | 2 + .../source/pcapkit/foundation/analysis/ip.rst | 13 + .../pcapkit/foundation/analysis/tcp.rst | 12 + .../pcapkit/foundation/reassembly/ip/ipv4.rst | 8 +- .../pcapkit/foundation/reassembly/ip/ipv6.rst | 10 +- .../pcapkit/foundation/reassembly/tcp.rst | 4 + pcapkit/all.py | 2 + pcapkit/foundation/__init__.py | 2 +- pcapkit/foundation/analysis/__init__.py | 6 + pcapkit/foundation/analysis/ip.py | 88 +++++ pcapkit/foundation/analysis/tcp.py | 87 +++++ pcapkit/foundation/reassembly/data/data.py | 20 +- pcapkit/foundation/reassembly/data/ip.py | 9 +- pcapkit/foundation/reassembly/data/tcp.py | 9 +- pcapkit/foundation/reassembly/ip.py | 11 +- pcapkit/foundation/reassembly/tcp.py | 11 +- .../test_ip_tcp_analysis_2581_unit.py | 333 ++++++++++++++++++ 18 files changed, 624 insertions(+), 35 deletions(-) create mode 100644 docs/source/pcapkit/foundation/analysis/ip.rst create mode 100644 docs/source/pcapkit/foundation/analysis/tcp.rst create mode 100644 pcapkit/foundation/analysis/ip.py create mode 100644 pcapkit/foundation/analysis/tcp.py create mode 100644 tests/foundation/reassembly/test_ip_tcp_analysis_2581_unit.py diff --git a/docs/source/contributing/conventions/reassembly-integration.rst b/docs/source/contributing/conventions/reassembly-integration.rst index 1d3f9e2a43..cb70b2d513 100644 --- a/docs/source/contributing/conventions/reassembly-integration.rst +++ b/docs/source/contributing/conventions/reassembly-integration.rst @@ -61,17 +61,29 @@ engine backend, by defining the extraction functions, by defining them raising engine level, as :mod:`pcapkit.foundation.engines.pyshark` does. DTLS defines them for the default engine and is disabled at the engine level on every other. -**Point 3 holds for DTLS alone.** -:func:`~pcapkit.foundation.analysis.dtls.dtls_handshake` parses a reassembled DTLS -handshake message (:issue:`2496`). IP and TCP have no such method: what parses their -outcome is the general-purpose -:meth:`Protocol.analyze ` dispatcher, -not a method written for reassembly specifically. +**Point 3 holds for every engine.** +:func:`~pcapkit.foundation.analysis.ip.ip_datagram` parses a reassembled IPv4 or IPv6 +datagram's payload by its payload protocol type, through +:meth:`Protocol.analyze `; +:func:`~pcapkit.foundation.analysis.tcp.tcp_datagram` parses a reassembled TCP +datagram's by its ports, through +:meth:`Transport.analyze `; +and :func:`~pcapkit.foundation.analysis.dtls.dtls_handshake` decodes a reassembled +DTLS handshake message (:issue:`2496`). The owner chose on :issue:`2581` to give IP +and TCP modules of their own, each function named for the datagram its engine +reassembles, rather than leave their outcomes to those dispatchers with no analysis +method, which kept the rule true for DTLS alone. **Where the rule is silent, DTLS takes the IP/TCP shape** (:issue:`2531`). IPv4, IPv6, TCP and DTLS reassembly each set ``__protocol_type__``, and each reports its parsed outcome as a datagram's ``packet``, deferred to first read by -:class:`~pcapkit.foundation.reassembly.data.data.Deferred`. IP and TCP run -``__protocol_type__``'s ``analyze``. DTLS runs the decoder behind ``dtls_handshake`` -instead, where point 3 places it, so its ``packet`` is what ``dtls_handshake`` returns: -:data:`None` for an incomplete message. +:class:`~pcapkit.foundation.reassembly.data.data.Deferred`. Each ``Deferred`` runs the +parse behind its engine's analysis function, so a ``packet`` is what that function +returns. IP and TCP bind that parse to ``__protocol_type__`` and call its ``analyze``, +so a subclass naming another protocol has its ``packet`` parsed by that one +(:issue:`2581`); DTLS decodes with +:meth:`DTLS.read_handshake ` +whatever ``__protocol_type__`` names. The engines differ in what an incomplete outcome +holds. IP and TCP parse a contiguous payload whether or not it is complete, as +``strict=False`` reports one, and hold :data:`None` only for the received runs +``strict=True`` reports; DTLS holds :data:`None` for any incomplete message. diff --git a/docs/source/pcapkit/foundation/analysis/index.rst b/docs/source/pcapkit/foundation/analysis/index.rst index ca5b2d3123..e14cd4d60e 100644 --- a/docs/source/pcapkit/foundation/analysis/index.rst +++ b/docs/source/pcapkit/foundation/analysis/index.rst @@ -14,4 +14,6 @@ so that a caller does not interpret the raw reassembled octets itself .. toctree:: :maxdepth: 2 + ip + tcp dtls diff --git a/docs/source/pcapkit/foundation/analysis/ip.rst b/docs/source/pcapkit/foundation/analysis/ip.rst new file mode 100644 index 0000000000..997c2e62a1 --- /dev/null +++ b/docs/source/pcapkit/foundation/analysis/ip.rst @@ -0,0 +1,13 @@ +==================== +IP Datagram Analysis +==================== + +.. module:: pcapkit.foundation.analysis.ip + +:mod:`pcapkit.foundation.analysis.ip` contains +:func:`~pcapkit.foundation.analysis.ip.ip_datagram` only, which parses the +payload of an IPv4 or IPv6 datagram that +:class:`~pcapkit.foundation.reassembly.ipv4.IPv4` or +:class:`~pcapkit.foundation.reassembly.ipv6.IPv6` reassembled. + +.. autofunction:: pcapkit.foundation.analysis.ip.ip_datagram diff --git a/docs/source/pcapkit/foundation/analysis/tcp.rst b/docs/source/pcapkit/foundation/analysis/tcp.rst new file mode 100644 index 0000000000..e838b8c8b3 --- /dev/null +++ b/docs/source/pcapkit/foundation/analysis/tcp.rst @@ -0,0 +1,12 @@ +===================== +TCP Datagram Analysis +===================== + +.. module:: pcapkit.foundation.analysis.tcp + +:mod:`pcapkit.foundation.analysis.tcp` contains +:func:`~pcapkit.foundation.analysis.tcp.tcp_datagram` only, which parses the +application-layer payload of a TCP datagram that +:class:`~pcapkit.foundation.reassembly.tcp.TCP` reassembled. + +.. autofunction:: pcapkit.foundation.analysis.tcp.tcp_datagram diff --git a/docs/source/pcapkit/foundation/reassembly/ip/ipv4.rst b/docs/source/pcapkit/foundation/reassembly/ip/ipv4.rst index e7e1f4a119..6a21060fae 100644 --- a/docs/source/pcapkit/foundation/reassembly/ip/ipv4.rst +++ b/docs/source/pcapkit/foundation/reassembly/ip/ipv4.rst @@ -95,9 +95,11 @@ Terminology .. note:: - ``packet`` is analysed on the first read, not when the datagram is - submitted. A datagram is submitted for *every* frame whose **DF** flag - is clear -- an unfragmented one included -- and the analysis + ``packet`` is what :func:`~pcapkit.foundation.analysis.ip.ip_datagram` + parses ``payload`` to, unless ``__protocol_type__`` names another + protocol, and it is analysed on the first read, not when the + datagram is submitted. A datagram is submitted for *every* frame + whose **DF** flag is clear -- an unfragmented one included -- and the analysis is a second full parse of the payload, so running it eagerly would charge every caller for a result most never read. Reading the attribute, or any mapping view of it (``datagram['packet']``, ``to_dict()``, ``items()``, diff --git a/docs/source/pcapkit/foundation/reassembly/ip/ipv6.rst b/docs/source/pcapkit/foundation/reassembly/ip/ipv6.rst index 21f586c21a..94ba306009 100644 --- a/docs/source/pcapkit/foundation/reassembly/ip/ipv6.rst +++ b/docs/source/pcapkit/foundation/reassembly/ip/ipv6.rst @@ -140,10 +140,12 @@ Terminology .. note:: - ``packet`` is analysed on the first read, not when the datagram is - submitted. The analysis is a second full parse of the payload, so - running it eagerly would charge every caller for a result most never - read. Reading the attribute, or any mapping view of it (``datagram['packet']``, ``to_dict()``, ``items()``, + ``packet`` is what :func:`~pcapkit.foundation.analysis.ip.ip_datagram` + parses ``payload`` to, unless ``__protocol_type__`` names another + protocol, and it is analysed on the first read, not when the + datagram is submitted. The analysis is a second full parse of the + payload, so running it eagerly would charge every caller for a result + most never read. Reading the attribute, or any mapping view of it (``datagram['packet']``, ``to_dict()``, ``items()``, ``repr()``), runs it and keeps the result; see :class:`~pcapkit.foundation.reassembly.data.data.Deferred`. diff --git a/docs/source/pcapkit/foundation/reassembly/tcp.rst b/docs/source/pcapkit/foundation/reassembly/tcp.rst index 363b720775..b5cf3f2c17 100644 --- a/docs/source/pcapkit/foundation/reassembly/tcp.rst +++ b/docs/source/pcapkit/foundation/reassembly/tcp.rst @@ -259,6 +259,10 @@ Terminology | | |--> ... |--> (Info) data ... + ``packet`` is what :func:`~pcapkit.foundation.analysis.tcp.tcp_datagram` + parses ``payload`` to, on its first read, unless ``__protocol_type__`` + names another protocol. + ``completed`` and ``conflict`` are independent signals: a datagram can be :attr:`~pcapkit.foundation.reassembly.data.data.Completion.COMPLETE` and still carry a non-empty ``conflict`` -- the resolution of a diff --git a/pcapkit/all.py b/pcapkit/all.py index 3b39c0a982..2a42c401ea 100644 --- a/pcapkit/all.py +++ b/pcapkit/all.py @@ -85,6 +85,8 @@ 'DTLS_Reassembly', # DTLS Reassembly # pcapkit.foundation.analysis + 'ip_datagram', # IP Analysis + 'tcp_datagram', # TCP Analysis 'dtls_handshake', # DTLS Analysis # pcapkit.foundation.registry diff --git a/pcapkit/foundation/__init__.py b/pcapkit/foundation/__init__.py index 1ba63a7ccf..541dcecbdf 100644 --- a/pcapkit/foundation/__init__.py +++ b/pcapkit/foundation/__init__.py @@ -25,7 +25,7 @@ 'IPv4_Reassembly', 'IPv6_Reassembly', 'TCP_Reassembly', 'DTLS_Reassembly', 'ReassemblyManager', - 'dtls_handshake', + 'ip_datagram', 'tcp_datagram', 'dtls_handshake', 'TCP_TraceFlow', 'TraceFlowManager', diff --git a/pcapkit/foundation/analysis/__init__.py b/pcapkit/foundation/analysis/__init__.py index 62fde606f1..4a2e2712d3 100644 --- a/pcapkit/foundation/analysis/__init__.py +++ b/pcapkit/foundation/analysis/__init__.py @@ -15,7 +15,13 @@ """ # Analysis of DTLS handshake messages from pcapkit.foundation.analysis.dtls import dtls_handshake +# Analysis of IP datagrams +from pcapkit.foundation.analysis.ip import ip_datagram +# Analysis of TCP datagrams +from pcapkit.foundation.analysis.tcp import tcp_datagram __all__ = [ + 'ip_datagram', # IP Analysis + 'tcp_datagram', # TCP Analysis 'dtls_handshake', # DTLS Analysis ] diff --git a/pcapkit/foundation/analysis/ip.py b/pcapkit/foundation/analysis/ip.py new file mode 100644 index 0000000000..a6b7d9b19e --- /dev/null +++ b/pcapkit/foundation/analysis/ip.py @@ -0,0 +1,88 @@ +# -*- coding: utf-8 -*- +"""IP Datagram Analysis +======================== + +.. module:: pcapkit.foundation.analysis.ip + +:mod:`pcapkit.foundation.analysis.ip` contains +:func:`~pcapkit.foundation.analysis.ip.ip_datagram` only, which parses the +payload of an IPv4 or IPv6 datagram that +:class:`~pcapkit.foundation.reassembly.ipv4.IPv4` or +:class:`~pcapkit.foundation.reassembly.ipv6.IPv6` reassembled. + +.. versionadded:: 1.6.0 + +""" +from ipaddress import IPv6Address +from typing import TYPE_CHECKING + +from pcapkit.protocols.internet.ipv4 import IPv4 +from pcapkit.protocols.internet.ipv6 import IPv6 + +if TYPE_CHECKING: + from typing import Optional, Type + + from pcapkit.const.reg.transtype import TransType + from pcapkit.foundation.reassembly.data.ip import Datagram as IP_Datagram + from pcapkit.protocols.internet.ip import IP as IP_Protocol + from pcapkit.protocols.protocol import ProtocolBase + +__all__ = ['ip_datagram'] + + +def ip_datagram(datagram: 'IP_Datagram') -> 'Optional[ProtocolBase]': + """Parse the payload of an IP datagram reassembled by + :class:`~pcapkit.foundation.reassembly.ipv4.IPv4` or + :class:`~pcapkit.foundation.reassembly.ipv6.IPv6`. + + The payload is dispatched on the datagram's payload protocol type -- the + **Protocol** field for IPv4, the Fragment header's **Next Header** for + IPv6 -- by :meth:`Protocol.analyze + ` of + :class:`~pcapkit.protocols.internet.ipv4.IPv4` or + :class:`~pcapkit.protocols.internet.ipv6.IPv6`, whichever the datagram's + addresses belong to. + + Args: + datagram: Reassembled datagram (:term:`reasm.ipv4.datagram` or + :term:`reasm.ipv6.datagram`). + + Returns: + The parsed payload. A payload reported as one contiguous run is parsed + whether or not ``datagram`` is complete, as ``strict=False`` reports an + incomplete one. :data:`None` if the payload is the tuple of runs + received, as ``strict=True`` reports an incomplete one. + + Note: + :attr:`Datagram.packet ` + holds the same result, parsed on its first read, unless the + reassembly's ``__protocol_type__`` names another protocol, which then + parses it instead. + + """ + if isinstance(datagram.payload, tuple): + return None + protocol = IPv6 if isinstance(datagram.id.src, IPv6Address) else IPv4 + return _analyze_payload(protocol, datagram.id.proto, datagram.payload) + + +def _analyze_payload(protocol: 'Type[IP_Protocol]', proto: 'TransType', + payload: 'bytes') -> 'ProtocolBase': + """Parse a reassembled payload by its payload protocol type. + + :func:`ip_datagram` runs this, and so does the + :class:`~pcapkit.foundation.reassembly.data.data.Deferred` that + :meth:`IP.submit ` stores as a + datagram's ``packet``, with the reassembly's own protocol bound as + ``protocol``. + + Args: + protocol: The IP protocol whose ``analyze`` parses ``payload``. + proto: The payload protocol type. + payload: The reassembled payload. + + Returns: + The parsed payload. + + """ + return protocol.analyze(proto, payload) diff --git a/pcapkit/foundation/analysis/tcp.py b/pcapkit/foundation/analysis/tcp.py new file mode 100644 index 0000000000..3191fc2fbb --- /dev/null +++ b/pcapkit/foundation/analysis/tcp.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- +"""TCP Datagram Analysis +========================= + +.. module:: pcapkit.foundation.analysis.tcp + +:mod:`pcapkit.foundation.analysis.tcp` contains +:func:`~pcapkit.foundation.analysis.tcp.tcp_datagram` only, which parses the +application-layer payload of a TCP datagram that +:class:`~pcapkit.foundation.reassembly.tcp.TCP` reassembled. + +.. versionadded:: 1.6.0 + +""" +from typing import TYPE_CHECKING + +from pcapkit.const.reg.apptype import TransportProtocol as Enum_TransportProtocol +from pcapkit.protocols.transport.tcp import TCP + +if TYPE_CHECKING: + from typing import Optional, Type + + from pcapkit.const.reg.apptype import TCP as AppType_TCP + from pcapkit.foundation.reassembly.data.tcp import Datagram as TCP_Datagram + from pcapkit.protocols.protocol import ProtocolBase + +__all__ = ['tcp_datagram'] + + +def tcp_datagram(datagram: 'TCP_Datagram') -> 'Optional[ProtocolBase]': + """Parse the payload of a TCP datagram reassembled by + :class:`~pcapkit.foundation.reassembly.tcp.TCP`. + + A reassembled datagram carries no protocol number, so its payload is + dispatched on its ``(source, destination)`` ports by + :meth:`Transport.analyze ` + of :class:`~pcapkit.protocols.transport.tcp.TCP`. Each port is resolved to + its :class:`~pcapkit.const.reg.apptype.tcp.TCP` member by + :meth:`Transport._make_port + `, as + :meth:`TCP._make_ports ` + resolves them for the reassembly. + + Args: + datagram: Reassembled datagram (:term:`reasm.tcp.datagram`). + + Returns: + The parsed payload. A payload reported as one contiguous run is parsed + whether or not ``datagram`` is complete, as ``strict=False`` reports an + incomplete one. :data:`None` if the payload is the tuple of runs + received, as ``strict=True`` reports an incomplete one. + + Note: + :attr:`Datagram.packet ` + holds the same result, parsed on its first read, unless the + reassembly's ``__protocol_type__`` names another protocol, which then + parses it instead. + + """ + if isinstance(datagram.payload, tuple): + return None + transport = Enum_TransportProtocol.tcp + ports = (TCP._make_port(datagram.id.src[1], transport), # pylint: disable=protected-access + TCP._make_port(datagram.id.dst[1], transport)) # pylint: disable=protected-access + return _analyze_payload(TCP, ports, datagram.payload) + + +def _analyze_payload(protocol: 'Type[TCP]', ports: 'tuple[AppType_TCP, AppType_TCP]', + payload: 'bytes') -> 'ProtocolBase': + """Parse a reassembled payload by its datagram's ports. + + :func:`tcp_datagram` runs this, and so does the + :class:`~pcapkit.foundation.reassembly.data.data.Deferred` that + :meth:`TCP.submit ` stores as + a datagram's ``packet``, with the reassembly's own protocol bound as + ``protocol``. + + Args: + protocol: The TCP protocol whose ``analyze`` parses ``payload``. + ports: The datagram's ``(source, destination)`` ports. + payload: The reassembled payload. + + Returns: + The parsed payload. + + """ + return protocol.analyze(ports, payload) diff --git a/pcapkit/foundation/reassembly/data/data.py b/pcapkit/foundation/reassembly/data/data.py index 828258ec12..45b8fd8d25 100644 --- a/pcapkit/foundation/reassembly/data/data.py +++ b/pcapkit/foundation/reassembly/data/data.py @@ -160,12 +160,15 @@ class Deferred(Generic[_KT]): * :meth:`IP.submit ` builds a ``Deferred[TransType]``. It passes the payload protocol type, a :class:`~pcapkit.const.reg.transtype.TransType` read from the reassembled - datagram's **Protocol** / **Next Header** field, to + datagram's **Protocol** / **Next Header** field, to the parse + :func:`~pcapkit.foundation.analysis.ip.ip_datagram` runs: :meth:`Protocol.analyze `, which looks the next layer up by protocol registry number. * :meth:`TCP.submit ` builds a ``Deferred[tuple[AppType_TCP, AppType_TCP]]``. It passes a - ``(source, destination)`` port pair, because :meth:`Transport.analyze + ``(source, destination)`` port pair to the parse + :func:`~pcapkit.foundation.analysis.tcp.tcp_datagram` runs, because + :meth:`Transport.analyze ` overrides that parameter to one: a reassembled transport-layer payload carries no protocol number, so its application protocol is inferred from the ports. @@ -209,10 +212,10 @@ class Deferred(Generic[_KT]): either way. Args: - analyze: The analyser to call: - :meth:`Protocol.analyze ` - bound to the reassembly object's protocol for IP and TCP, the - handshake decoder for DTLS. + analyze: The analyser to call: the parse + :mod:`pcapkit.foundation.analysis` runs for the engine, bound to + the reassembly object's protocol for IP and TCP, whose ``analyze`` + it calls, and the handshake decoder for DTLS. proto: What the analyser dispatches the payload on -- the payload protocol type for IP, the port pair for TCP, the handshake header for DTLS, as above. @@ -224,7 +227,10 @@ class Deferred(Generic[_KT]): Also constrained to the DTLS handshake header, and the analyser may return a :class:`~pcapkit.protocols.data.application.dtls.DTLSHandshake` - (:issue:`2531`). + (:issue:`2531`). IP's and TCP's analyser is the parse their + :mod:`pcapkit.foundation.analysis` module runs, bound to the + reassembly's protocol, rather than that protocol's bound ``analyze`` + (:issue:`2581`). """ diff --git a/pcapkit/foundation/reassembly/data/ip.py b/pcapkit/foundation/reassembly/data/ip.py index 79675a92e0..187200b111 100644 --- a/pcapkit/foundation/reassembly/data/ip.py +++ b/pcapkit/foundation/reassembly/data/ip.py @@ -113,9 +113,12 @@ class Datagram(DeferredPacket, Info, Generic[_AT]): header: 'bytes' #: Reassembled IP payload. payload: 'bytes | tuple[bytes, ...]' - #: Parsed IP payload. Analysed on first read, not at construction time; a - #: :class:`~pcapkit.foundation.reassembly.data.data.Deferred` may be passed in its - #: place, and reading this attribute then runs it and keeps the result. + #: Parsed IP payload, as :func:`~pcapkit.foundation.analysis.ip.ip_datagram` + #: returns it unless the reassembly's ``__protocol_type__`` names another + #: protocol, which then parses it. Analysed on first read, not at + #: construction time; a :class:`~pcapkit.foundation.reassembly.data.data.Deferred` + #: may be passed in its place, and reading this attribute then runs it and + #: keeps the result. packet: 'Optional[ProtocolBase]' #: Octet ranges, absolute into the reassembled payload and both #: **inclusive** -- the same convention as :attr:`Packet.fo` combined with diff --git a/pcapkit/foundation/reassembly/data/tcp.py b/pcapkit/foundation/reassembly/data/tcp.py index 31e43f806a..cca5297604 100644 --- a/pcapkit/foundation/reassembly/data/tcp.py +++ b/pcapkit/foundation/reassembly/data/tcp.py @@ -145,9 +145,12 @@ class Datagram(DeferredPacket, Info, Generic[_AT]): header: 'bytes' #: Reassembled payload (application layer data). payload: 'bytes | tuple[bytes, ...]' - #: Parsed TCP payload. Analysed on first read rather than at construction; - #: a :class:`~pcapkit.foundation.reassembly.data.data.Deferred` may be passed in - #: its place, and reading this attribute then runs it and keeps the result. + #: Parsed TCP payload, as :func:`~pcapkit.foundation.analysis.tcp.tcp_datagram` + #: returns it unless the reassembly's ``__protocol_type__`` names another + #: protocol, which then parses it. Analysed on first read rather than at + #: construction; a :class:`~pcapkit.foundation.reassembly.data.data.Deferred` + #: may be passed in its place, and reading this attribute then runs it and + #: keeps the result. packet: 'Optional[ProtocolBase]' #: Sequence ranges on which two segments disagreed, i.e. where an arriving #: segment overlapped bytes already buffered but did not repeat them. diff --git a/pcapkit/foundation/reassembly/ip.py b/pcapkit/foundation/reassembly/ip.py index 14f5f2b239..e89d992df5 100644 --- a/pcapkit/foundation/reassembly/ip.py +++ b/pcapkit/foundation/reassembly/ip.py @@ -16,8 +16,10 @@ """ import bisect import sys +from functools import partial from typing import TYPE_CHECKING, Generic +from pcapkit.foundation.analysis.ip import _analyze_payload from pcapkit.foundation.reassembly.data.data import Completion from pcapkit.foundation.reassembly.data.ip import (_AT, Buffer, BufferID, Datagram, DatagramID, Deferred, Packet) @@ -510,7 +512,12 @@ def submit(self, buf: 'Buffer[_AT]', *, bufid: 'tuple[_AT, _AT, int, TransType]' and never as complete Returns: - Reassembled packets. + Reassembled packets. Where the payload is one contiguous run, its + ``packet`` is the parse + :func:`~pcapkit.foundation.analysis.ip.ip_datagram` runs, by this + reassembly's own + :attr:`protocol `, + deferred to its first read. """ TDL = buf.TDL @@ -626,7 +633,7 @@ def submit(self, buf: 'Buffer[_AT]', *, bufid: 'tuple[_AT, _AT, int, TransType]' # charges every caller for a result most of them never read. # ``Deferred`` postpones it to the first read of # ``Datagram.packet``. - packet=Deferred(self.protocol.analyze, bufid[3], payload), + packet=Deferred(partial(_analyze_payload, self.protocol), bufid[3], payload), conflict=conflict, ) ret.append(packet) diff --git a/pcapkit/foundation/reassembly/tcp.py b/pcapkit/foundation/reassembly/tcp.py index e2268ddf94..d4096b4bbf 100644 --- a/pcapkit/foundation/reassembly/tcp.py +++ b/pcapkit/foundation/reassembly/tcp.py @@ -10,9 +10,11 @@ """ import math +from functools import partial from typing import TYPE_CHECKING from pcapkit.const.reg.apptype import TransportProtocol as Enum_TransportProtocol +from pcapkit.foundation.analysis.tcp import _analyze_payload from pcapkit.foundation.reassembly.data.data import Completion, Deferred from pcapkit.foundation.reassembly.data.tcp import (HOLE_OPEN_BOUND, Buffer, BufferID, Datagram, DatagramID, Fragment, HoleDescriptor, Packet) @@ -746,7 +748,12 @@ def submit(self, buf: 'Buffer', *, bufid: 'BufferID', # type: ignore[override] abandoned it under the reassembly timeout Returns: - Reassembled :term:`packets `. + Reassembled :term:`packets `. Where the payload + is one contiguous run, its ``packet`` is the parse + :func:`~pcapkit.foundation.analysis.tcp.tcp_datagram` runs, by this + reassembly's own + :attr:`protocol `, + deferred to its first read. Important: The ``(source, destination)`` pair handed to @@ -860,7 +867,7 @@ def submit(self, buf: 'Buffer', *, bufid: 'BufferID', # type: ignore[override] index=tuple(buffer.ind), header=buf.hdr, payload=bytes(payload), - packet=Deferred(self.protocol.analyze, + packet=Deferred(partial(_analyze_payload, self.protocol), self._make_ports(bufid), bytes(payload)), conflict=conflict, ) diff --git a/tests/foundation/reassembly/test_ip_tcp_analysis_2581_unit.py b/tests/foundation/reassembly/test_ip_tcp_analysis_2581_unit.py new file mode 100644 index 0000000000..edfcbfc1f1 --- /dev/null +++ b/tests/foundation/reassembly/test_ip_tcp_analysis_2581_unit.py @@ -0,0 +1,333 @@ +# -*- coding: utf-8 -*- +"""IP and TCP reassembly parse through their own analysis modules (#2581). + +:func:`pcapkit.foundation.analysis.ip.ip_datagram` parses a reassembled IPv4 or +IPv6 datagram's payload, and :func:`pcapkit.foundation.analysis.tcp.tcp_datagram` +a reassembled TCP datagram's, as +:func:`pcapkit.foundation.analysis.dtls.dtls_handshake` decodes a DTLS message. +A reassembled datagram's deferred ``packet`` runs the same parse, bound to the +reassembly's own protocol: it holds what the analysis function returns for the +default protocol, and what a custom ``__protocol_type__`` returns otherwise. + +Every capture is built in memory, as PCAP and as PCAP-NG; no sample capture is +read. + +""" + +from __future__ import annotations + +import ipaddress +import pickle +import struct +import unittest +from typing import TYPE_CHECKING +from unittest import mock + +from tests._support import reimport_once_per_class +from tests.foundation.reassembly.test_dtls_handshake_reassembly_2440_unit import ( + _extract, _pcap, _pcapng) + +if TYPE_CHECKING: + from typing import Any, Callable + +#: Test tier (#1958): pickling, whose protocols differ by interpreter. +__tier__ = 'ALL' + +#: The containers each case runs over. +CONTAINERS = (('pcap', _pcap), ('pcapng', _pcapng)) + +V4_SRC, V4_DST = ipaddress.ip_address('192.0.2.1'), ipaddress.ip_address('198.51.100.2') +V6_SRC, V6_DST = ipaddress.ip_address('2001:db8::1'), ipaddress.ip_address('2001:db8::2') + +#: A UDP datagram, the payload both IP captures fragment in two. +UDP = struct.pack('!HHHH', 1234, 9, 32, 0) + bytes(range(24)) +#: An HTTP request, which the TCP captures carry in two segments. +REQUEST = (b'GET / HTTP/1.1\r\n', b'Host: example\r\n\r\n') + + +def _ether(packet: bytes, ethertype: int) -> bytes: + return b'\xbb' * 6 + b'\xaa' * 6 + ethertype.to_bytes(2, 'big') + packet + + +def _ipv4(payload: bytes, *, offset: int, mf: bool, protocol: int = 17, df: bool = False) -> bytes: + flags = (0x4000 if df else 0) | (0x2000 if mf else 0) | offset // 8 + return _ether(struct.pack('!BBHHHBBH4s4s', 0x45, 0, 20 + len(payload), 0x1234, flags, 64, + protocol, 0, V4_SRC.packed, V4_DST.packed) + payload, 0x0800) + + +def _ipv6(payload: bytes, *, offset: int, mf: bool) -> bytes: + fragment = struct.pack('!BBHI', 17, 0, offset | int(mf), 0x5678) + payload + return _ether(struct.pack('!IHBB16s16s', 6 << 28, len(fragment), 44, 64, + V6_SRC.packed, V6_DST.packed) + fragment, 0x86DD) + + +def _tcp(payload: bytes, *, seq: int, fin: bool = False) -> bytes: + flags = 0x18 | (0x01 if fin else 0) # PSH, ACK, and FIN if asked + segment = struct.pack('!HHIIBBHHH', 1234, 80, seq, 1, 5 << 4, flags, 0xFFFF, 0, 0) + payload + return _ipv4(segment, offset=0, mf=False, protocol=6, df=True) + + +#: Each engine's frames: whole, and with a fragment or segment lost. +FRAMES = { + 'ipv4': ([_ipv4(UDP[:16], offset=0, mf=True), _ipv4(UDP[16:], offset=16, mf=False)], + [_ipv4(UDP[:16], offset=0, mf=True)]), + 'ipv6': ([_ipv6(UDP[:16], offset=0, mf=True), _ipv6(UDP[16:], offset=16, mf=False)], + [_ipv6(UDP[:16], offset=0, mf=True)]), + 'tcp': ([_tcp(REQUEST[0], seq=1000), _tcp(REQUEST[1], seq=1000 + len(REQUEST[0]), fin=True)], + [_tcp(REQUEST[0], seq=1000), _tcp(REQUEST[1], seq=1008 + len(REQUEST[0]), fin=True)]), +} + + +def _held(datagram: 'Any') -> 'Any': + """``datagram``'s ``packet`` as stored, without resolving it.""" + return datagram.__dict__[datagram.__map__.get('packet', 'packet')] + + +def _parsed(packet: 'Any') -> 'Any': + """What a parse produced, comparable by value.""" + return None if packet is None else (type(packet), packet.info) + + +class CustomProtocol: + """A custom engine's ``__protocol_type__``, whose parse is tagged as its own. + + Module-level, so a datagram whose ``packet`` it is bound to still pickles. + + """ + + @classmethod + def analyze(cls, proto: 'Any', payload: bytes) -> 'tuple[str, Any, bytes]': + return ('custom', proto, payload) + + +def _analysis(kind: str) -> 'tuple[Callable[[Any], Any], Any, Any]': + """The public function, the default protocol and the engine for ``kind``.""" + from pcapkit.foundation.analysis import ip_datagram, tcp_datagram + from pcapkit.foundation.reassembly import ipv4, ipv6, tcp + from pcapkit.protocols.internet.ipv4 import IPv4 + from pcapkit.protocols.internet.ipv6 import IPv6 + from pcapkit.protocols.transport.tcp import TCP + + return { + 'ipv4': (ip_datagram, IPv4, ipv4.IPv4), + 'ipv6': (ip_datagram, IPv6, ipv6.IPv6), + 'tcp': (tcp_datagram, TCP, tcp.TCP), + }[kind] + + +def _expected(kind: str) -> 'Any': + """The class the default protocol parses ``kind``'s whole payload to.""" + from pcapkit.protocols.application.httpv1 import HTTPv1 + from pcapkit.protocols.transport.udp import UDP as UDP_Protocol + + return HTTPv1 if kind == 'tcp' else UDP_Protocol + + +def _key(kind: str, proto: 'Any') -> 'Any': + """What the parse of ``kind``'s datagram dispatches on, comparable by value.""" + return [port.port for port in proto] if kind == 'tcp' else proto + + +class AnalysisFunctionTests(unittest.TestCase): + """``ip_datagram`` and ``tcp_datagram``, called on a datagram built directly.""" + + def setUp(self) -> None: + reimport_once_per_class(self) + + @staticmethod + def _ip(payload: 'bytes | tuple[bytes, ...]', *, v6: bool = False, + completed: str = 'COMPLETE') -> 'Any': + from pcapkit.const.reg.transtype import TransType + from pcapkit.foundation.reassembly.data.data import Completion + from pcapkit.foundation.reassembly.data.ip import Datagram, DatagramID + + src, dst = (V6_SRC, V6_DST) if v6 else (V4_SRC, V4_DST) + return Datagram(completed=Completion[completed], + id=DatagramID(src=src, dst=dst, id=1, proto=TransType.UDP), + index=(1, 2), header=b'', payload=payload, packet=None, conflict=()) + + @staticmethod + def _tcp(payload: 'bytes | tuple[bytes, ...]', *, dport: int = 80, + completed: str = 'COMPLETE') -> 'Any': + from pcapkit.foundation.reassembly.data.data import Completion + from pcapkit.foundation.reassembly.data.tcp import Datagram, DatagramID + + return Datagram(completed=Completion[completed], + id=DatagramID(src=(V4_SRC, 1234), dst=(V4_DST, dport), ack=1), + index=(1, 2), header=b'', payload=payload, packet=None, conflict=()) + + def test_ip_datagram_parses_by_the_payload_protocol_type(self) -> None: + from pcapkit.const.reg.transtype import TransType + from pcapkit.foundation.analysis import ip_datagram + from pcapkit.protocols.internet.ipv4 import IPv4 + from pcapkit.protocols.transport.udp import UDP as UDP_Protocol + + packet = ip_datagram(self._ip(UDP)) + self.assertIsInstance(packet, UDP_Protocol) + self.assertEqual(_parsed(packet), _parsed(IPv4.analyze(TransType.UDP, UDP))) + self.assertEqual((packet.info.srcport, packet.info.dstport), (1234, 9)) + + def test_ip_datagram_takes_the_protocol_its_addresses_belong_to(self) -> None: + from pcapkit.foundation.analysis import ip_datagram + from pcapkit.protocols.internet.ipv4 import IPv4 + from pcapkit.protocols.internet.ipv6 import IPv6 + + for v6, called, idle in ((False, IPv4, IPv6), (True, IPv6, IPv4)): + with self.subTest(v6=v6): + with mock.patch.object(called, 'analyze', wraps=called.analyze) as used, \ + mock.patch.object(idle, 'analyze', wraps=idle.analyze) as unused: + ip_datagram(self._ip(UDP, v6=v6)) + used.assert_called_once() + self.assertEqual(used.call_args.args[1], UDP) + unused.assert_not_called() + + def test_tcp_datagram_parses_by_its_ports(self) -> None: + from pcapkit.const.reg.apptype import TCP as AppType_TCP + from pcapkit.foundation.analysis import tcp_datagram + from pcapkit.protocols.application.httpv1 import HTTPv1 as HTTP + from pcapkit.protocols.misc.raw import Raw + from pcapkit.protocols.transport.tcp import TCP + + payload = b''.join(REQUEST) + with mock.patch.object(TCP, 'analyze', wraps=TCP.analyze) as analyze: + self.assertIsInstance(tcp_datagram(self._tcp(payload)), HTTP) + ports, = {call.args[0] for call in analyze.call_args_list} + self.assertEqual([type(port) for port in ports], [AppType_TCP, AppType_TCP]) + self.assertEqual([port.port for port in ports], [1234, 80]) + self.assertIsInstance(tcp_datagram(self._tcp(b'not a protocol', dport=50000)), Raw) + + def test_a_contiguous_incomplete_payload_is_parsed(self) -> None: + from pcapkit.foundation.analysis import ip_datagram, tcp_datagram + from pcapkit.protocols.application.httpv1 import HTTPv1 as HTTP + from pcapkit.protocols.transport.udp import UDP as UDP_Protocol + + request = b''.join(REQUEST) + for completed in ('PARTIAL', 'TIMEOUT'): + with self.subTest(completed=completed): + self.assertIsInstance(ip_datagram(self._ip(UDP, completed=completed)), UDP_Protocol) + self.assertIsInstance(tcp_datagram(self._tcp(request, completed=completed)), HTTP) + + def test_received_runs_are_not_parsed(self) -> None: + from pcapkit.foundation.analysis import ip_datagram, tcp_datagram + + runs = (UDP[:8], UDP[16:]) + self.assertIsNone(ip_datagram(self._ip(runs, completed='PARTIAL'))) + self.assertIsNone(ip_datagram(self._ip(runs, v6=True, completed='TIMEOUT'))) + self.assertIsNone(tcp_datagram(self._tcp(REQUEST, completed='PARTIAL'))) + + def test_the_functions_are_exported_as_reassembly_names_are(self) -> None: + import pcapkit.all + import pcapkit.foundation + import pcapkit.foundation.analysis + from pcapkit.foundation.analysis.ip import ip_datagram + from pcapkit.foundation.analysis.tcp import tcp_datagram + + for module in (pcapkit.foundation.analysis, pcapkit.foundation, pcapkit.all): + for name, function in (('ip_datagram', ip_datagram), ('tcp_datagram', tcp_datagram)): + with self.subTest(module=module.__name__, name=name): + self.assertIn(name, module.__all__) + self.assertIs(getattr(module, name), function) + + +class ExtractorReassemblyTests(unittest.TestCase): + """A reassembled datagram's ``packet`` is the analysis function's result, deferred.""" + + def setUp(self) -> None: + reimport_once_per_class(self) + + def _datagram(self, kind: str, write: 'Any', *, whole: bool = True, **kwargs: 'Any') -> 'Any': + frames = FRAMES[kind][0 if whole else 1] + extractor = _extract(self, write(frames), reassembly=True, ip=True, tcp=True, **kwargs) + datagram, = getattr(extractor.reassembly, kind) + return datagram + + def test_reading_packet_parses_as_the_analysis_function_does(self) -> None: + for kind in FRAMES: + function, _, _ = _analysis(kind) + for container, write in CONTAINERS: + with self.subTest(kind=kind, container=container): + datagram = self._datagram(kind, write) + self.assertTrue(datagram.completed) + packet = datagram.packet + self.assertIsInstance(packet, _expected(kind)) + self.assertEqual(_parsed(packet), _parsed(function(datagram))) + self.assertIs(datagram.packet, packet) + + def test_a_custom_protocol_type_parses_the_packet(self) -> None: + from pcapkit.const.reg.transtype import TransType + from pcapkit.foundation.reassembly.data.data import Deferred + + for kind in FRAMES: + function, _, engine = _analysis(kind) + key = [1234, 80] if kind == 'tcp' else TransType.UDP + for container, write in CONTAINERS: + with self.subTest(kind=kind, container=container): + with mock.patch.object(engine, '__protocol_type__', CustomProtocol): + datagram = self._datagram(kind, write) + self.assertIsInstance(_held(datagram), Deferred) + tag, proto, payload = datagram.packet + self.assertEqual((tag, _key(kind, proto), payload), + ('custom', key, datagram.payload)) + # the analysis function parses with the default protocol + self.assertIsInstance(function(datagram), _expected(kind)) + + def test_the_parse_runs_on_first_read_and_only_once(self) -> None: + for kind in FRAMES: + function, protocol, _ = _analysis(kind) + for container, write in CONTAINERS: + with self.subTest(kind=kind, container=container): + with mock.patch.object(protocol, 'analyze', wraps=protocol.analyze) as analyze: + datagram = self._datagram(kind, write) + self.assertEqual(analyze.call_count, 0) + packet = datagram.packet + datagram['packet'] # pylint: disable=pointless-statement + repr(datagram) + self.assertEqual(analyze.call_count, 1) + self.assertEqual(_parsed(packet), _parsed(function(datagram))) + + def test_an_incomplete_datagram_is_parsed_unless_strict(self) -> None: + from pcapkit.foundation.reassembly.data.data import Deferred + + for kind in FRAMES: + function, _, _ = _analysis(kind) + for container, write in CONTAINERS: + for strict in (True, False): + with self.subTest(kind=kind, container=container, strict=strict): + datagram = self._datagram(kind, write, whole=False, reasm_strict=strict) + self.assertEqual(datagram.completed.name, 'PARTIAL') + self.assertIsInstance(datagram.payload, tuple if strict else bytes) + self.assertIsInstance(_held(datagram), type(None) if strict else Deferred) + self.assertEqual(_parsed(datagram.packet), _parsed(function(datagram))) + self.assertEqual(datagram.packet is None, strict) + + def test_a_deferred_datagram_pickles_and_still_parses(self) -> None: + from pcapkit.foundation.reassembly.data.data import Deferred + + for kind in FRAMES: + function, _, engine = _analysis(kind) + for container, write in CONTAINERS: + default = self._datagram(kind, write) + with mock.patch.object(engine, '__protocol_type__', CustomProtocol): + custom = self._datagram(kind, write) + for name, datagram in (('default', default), ('custom', custom)): + restored = [] + for number in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(kind=kind, container=container, engine=name, + protocol=number): + copy = pickle.loads(pickle.dumps(datagram, protocol=number)) + self.assertIsInstance(_held(datagram), Deferred) + self.assertIsInstance(_held(copy), Deferred) + restored.append(copy.packet) + with self.subTest(kind=kind, container=container, engine=name): + if name == 'default': + expected = _parsed(function(datagram)) + self.assertEqual([_parsed(packet) for packet in restored], + [expected] * len(restored)) + self.assertEqual(_parsed(datagram.packet), expected) + else: + self.assertEqual(datagram.packet[0], 'custom') + self.assertEqual(restored, [datagram.packet] * len(restored)) + + +if __name__ == '__main__': + unittest.main()