fix(toolkit): pyshark skips partial TCP headers and traces first fragments (#1611, #1612)

docs/source/pcapkit/foundation/engines/index.rst

@@ -76,7 +76,10 @@ offers, and the ones that cannot say so rather than quietly doing less:
 | Engine                                                          | Gap, and how it is surfaced                                   |
 +=================================================================+===============================================================+
 | :class:`~pcapkit.foundation.engines.pyshark.PyShark`            | no reassembly -- disabled with an                             |
-|                                                                 | :class:`~pcapkit.utilities.warnings.AttributeWarning`         |
+|                                                                 | :class:`~pcapkit.utilities.warnings.AttributeWarning`, and    |
+|                                                                 | :program:`tshark`'s own IP reassembly is turned off, so a     |
+|                                                                 | fragmented datagram is read frame by frame, as the built-in   |
+|                                                                 | engine reads it                                               |
 +-----------------------------------------------------------------+---------------------------------------------------------------+
 | :class:`~pcapkit.foundation.engines.pypcap.PyPCAP`              | no protocol dissection at all, hence no reassembly and no     |
 |                                                                 | flow tracing -- both disabled with an                         |

pcapkit/foundation/engines/pyshark.py

@@ -106,6 +106,29 @@ class PyShark(EngineBase['PySharkPacket']):
     #: event loop exists instead of quietly creating one.
     PYTHON_CEILING = (3, 14)
 
+    #: :program:`tshark` preferences :meth:`run` overrides, through
+    #: ``pyshark``'s ``override_prefs``: IPv4 and IPv6 reassembly off. tshark
+    #: reassembles fragments by default and dissects what the datagram carries
+    #: at the frame that completes it, so TCP was traced at the last fragment.
+    #: The default engine reads each frame as captured, TCP from a first
+    #: fragment (:issue:`1612`), and pcapkit's own reassembly is disabled on
+    #: this engine.
+    #:
+    #: Turning tshark's off also changes its TCP analysis, which is not
+    #: confined to fragments. Measured with tshark 4.6.9: tshark sizes a
+    #: fragmented segment from its first fragment alone, so its sequence
+    #: analysis sees a gap after it. Later frames of that stream, fragments or
+    #: not, then gain ``tcp.analysis.lost_segment``, and their other
+    #: ``tcp.analysis`` fields, ``tcp.completeness``, ``tcp.time_delta`` and
+    #: ``tcp.stream.*.contiguity_count`` change. Those fields reach the frame
+    #: records :func:`~pcapkit.toolkit.pyshark.packet2dict` builds. But
+    #: :func:`~pcapkit.toolkit.pyshark.tcp_traceflow` reads none of them, so
+    #: flow tracing is unaffected. The 23 captures under
+    #: :file:`examples/captures` carry no fragmented TCP. On them, every field
+    #: of every frame is the same either way, except the first and last
+    #: fragments of the one fragmented UDP datagram in :file:`ipv6.pcap`.
+    OVERRIDE_PREFS = {'ip.defragment': 'FALSE', 'ipv6.defragment': 'FALSE'}  # type: dict[str, str]
+
     ##########################################################################
     # Class methods.
     ##########################################################################
@@ -214,7 +237,8 @@ def run(self) -> 'None':
 
         This method assigns :attr:`self._expkg <PyShark._expkg>`
         as :mod:`pyshark` and :attr:`self._extmp <PyShark._extmp>`
-        as an iterator from :class:`pyshark.FileCapture`.
+        as an iterator from :class:`pyshark.FileCapture`, with
+        :program:`tshark`'s IP reassembly off (:attr:`OVERRIDE_PREFS`).
 
         The global header is parsed and dumped first, by
         :meth:`self.extractor.record_header <pcapkit.foundation.extraction.Extractor.record_header>`.
@@ -267,7 +291,8 @@ def run(self) -> 'None':
 
         # extract & analyse file
         logger.debug('pyshark: opening %s', ext._ifnm)
-        self._extmp = self._expkg.FileCapture(ext._ifnm, keep_packets=False)
+        self._extmp = self._expkg.FileCapture(ext._ifnm, keep_packets=False,
+                                              override_prefs=dict(self.OVERRIDE_PREFS))
 
     def read_frame(self) -> 'PySharkPacket':
         """Read frames with PyShark engine.

pcapkit/toolkit/pyshark.py

@@ -483,6 +483,36 @@ def _carrier(packet: 'Packet') -> 'tuple[Any, Any]':
     return ip, None
 
 
+def _has_fixed_header(tcp: 'Any') -> 'bool':
+    """Whether tshark dissected the whole fixed TCP header of a PyShark TCP layer.
+
+    Args:
+        tcp: PyShark TCP layer.
+
+    Returns:
+        Whether the layer carries ``tcp.urgent_pointer``.
+
+    Note:
+        The default engine traces a segment only if its 20-octet fixed header
+        (:rfc:`9293#section-3.1`) was captured and its Data Offset is at least
+        5 (:func:`pcapkit.toolkit.pcap.tcp_segment`). tshark omits fields in
+        both other cases, measured with tshark 4.6.9 (:issue:`1611`):
+
+        * a Data Offset below 5 stops the dissection at ``tcp.hdr_len``, so
+          ``tcp.ack`` and the ``tcp.flags`` fields are absent;
+        * a header cut at octet *k* keeps the fields that fit: no TCP layer
+          below 4, the ports only below 14, and no ``tcp.urgent_pointer``
+          below 20.
+
+        ``tcp.urgent_pointer`` covers octets 18 and 19, the last of the fixed
+        header. ``dissect_tcp`` in :file:`epan/dissectors/packet-tcp.c` adds
+        it unconditionally once the Data Offset check passes, in 2.6.0, 3.0.0
+        and 4.6.0, after every field :func:`tcp_traceflow` reads.
+
+    """
+    return 'urgent_pointer' in tcp.field_names
+
+
 def tcp_traceflow(packet: 'Packet', *, count: 'int' = -1) -> 'TF_TCP_Packet | None':
     """Trace packet flow for TCP.
 
@@ -496,7 +526,8 @@ def tcp_traceflow(packet: 'Packet', *, count: 'int' = -1) -> 'TF_TCP_Packet | No
         Data for TCP flow tracing.
 
         * If the ``packet`` can be used for TCP flow tracing. A packet can be reassembled
-          if it contains TCP layer, keyed by the IP layer that carries it -- in an
+          if it contains TCP layer with a whole fixed header (see
+          :func:`_has_fixed_header`), keyed by the IP layer that carries it -- in an
           IP-in-IP tunnel, the innermost (see :func:`_carrier`).
         * If the ``packet`` can be reassembled, then the :obj:`dict` mapping of data for TCP
           flow tracing (:term:`trace.tcp.packet`) will be returned; otherwise, returns :data:`None`.
@@ -512,7 +543,7 @@ def tcp_traceflow(packet: 'Packet', *, count: 'int' = -1) -> 'TF_TCP_Packet | No
 
     """
     ip, tcp = _carrier(packet)
-    if ip is not None and tcp is not None:
+    if ip is not None and tcp is not None and _has_fixed_header(tcp):
 
         # NOTE: the link type is keyed on ``frame.encap_type`` -- Wireshark's
         # internal ``WTAP_ENCAP_*`` number -- through

tests/foundation/engines/test_runtime_engines.py

@@ -448,7 +448,8 @@ def close(self) -> None:
         engine = PyShark(extractor)
         with mock.patch.object(engine._expkg, 'FileCapture', return_value=capture) as file_capture:
             engine.run()
-        file_capture.assert_called_once_with('capture.pcap', keep_packets=False)
+        file_capture.assert_called_once_with('capture.pcap', keep_packets=False,
+                                             override_prefs=PyShark.OVERRIDE_PREFS)
         extractor.record_header.assert_called_once_with()
         self.assertFalse(extractor._flag_r)
         self.assertIsNone(extractor._reasm.ipv4)