feat(application): dissect every whole DTLS record in a datagram

docs/source/contributing/conventions/protocol-family-grouping.rst

@@ -230,12 +230,13 @@ are all ``False``; only ``frame.protochain`` matches them.
 Chained Records
 ~~~~~~~~~~~~~~~
 
-**TLS and SSLv2 records sharing a TCP segment are chained layers, not a list on one
-layer** (:issue:`2517`, :issue:`2567`): each record dissected after the first is the
-previous record's payload, so a TLS record's dynamic alias names its own version -- a
-``ServerHello`` then a ``ChangeCipherSpec`` read ``TCP:TLS 1.3:TLS 1.2`` -- and an SSLv2
-``CLIENT-HELLO`` then a ``CLIENT-MASTER-KEY`` read ``TCP:SSLv2:SSLv2``. DTLS does not
-chain: two DTLS records in one datagram read ``DTLS 1.2:Raw`` (:issue:`2535`).
+**TLS and SSLv2 records sharing a TCP segment, and DTLS records sharing a UDP datagram,
+are chained layers, not a list on one layer** (:issue:`2517`, :issue:`2567`,
+:issue:`2535`): each record dissected after the first is the previous record's payload,
+so a TLS or DTLS record's dynamic alias names its own version -- a ``ServerHello`` then a
+``ChangeCipherSpec`` read ``TCP:TLS 1.3:TLS 1.2``, and a DTLS 1.2 record then a DTLS 1.3
+unified-header record read ``UDP:DTLS 1.2:DTLS 1.3`` -- and an SSLv2 ``CLIENT-HELLO``
+then a ``CLIENT-MASTER-KEY`` read ``TCP:SSLv2:SSLv2``.
 
 **HTTP/2 frames chain the same way** (:issue:`2536`): a ``SETTINGS`` frame then a
 ``WINDOW_UPDATE`` read ``TCP:HTTP/2:HTTP/2``. A later whole frame that fails to decode

docs/source/pcapkit/protocols/application/dtls.rst

@@ -46,6 +46,28 @@ A unified-header record has no legacy version to read. Its first octet's top
 three bits are ``001``, which no legacy content type (20 through 27) sets, so
 the two formats are told apart by that octet alone.
 
+Records in One Datagram
+-----------------------
+
+A whole record after the first in a UDP datagram is dissected as a further
+:class:`~pcapkit.protocols.application.dtls.DTLS` layer, the previous record's
+payload (:rfc:`6347#section-4.1.1`, :rfc:`9147#section-4.3`, :issue:`2535`): a
+``ChangeCipherSpec`` followed by an ``Alert`` reads ``UDP:DTLS 1.2:DTLS 1.2``.
+A unified-header record is delimited by its length; one without a length runs
+to the end of the datagram, so it is the last (:rfc:`9147#section-4`). The
+exceptions:
+
+* A whole record that fails to decode becomes
+  :class:`~pcapkit.protocols.misc.raw.Raw` together with everything after it.
+* Octets that open no whole record stay raw, with no error: an unsupported
+  content type, a legacy version outside ``0xFExx``, a header cut short, a
+  length past the end of the datagram, or a unified header with a connection
+  ID, which leaves the record's end unknown.
+* A parse told to stop at DTLS (``protocol='DTLS'`` or ``layer='Application'``)
+  keeps the rest of the datagram raw.
+* Each record is a layer, so :data:`~pcapkit.utilities.config.FRAME_LAYER_LIMIT`
+  bounds how many are dissected.
+
 Known Limitations
 -----------------
 

pcapkit/protocols/application/application.py

@@ -45,8 +45,9 @@ class Application(ProtocolBase[_PT, _ST], Generic[_PT, _ST]):  # pylint: disable
     for the rest of the packet undissected and resolves to
     :class:`~pcapkit.protocols.misc.raw.Raw` -- trailing bytes are not a further
     protocol layer, unless the subclass dissects them, as
-    :class:`~pcapkit.protocols.application.tls.TLS` and
-    :class:`~pcapkit.protocols.application.sslv2.SSLv2` do for a further record and
+    :class:`~pcapkit.protocols.application.tls.TLS`,
+    :class:`~pcapkit.protocols.application.sslv2.SSLv2` and
+    :class:`~pcapkit.protocols.application.dtls.DTLS` do for a further record and
     :class:`~pcapkit.protocols.application.httpv2.HTTPv2` for a further frame --
     or to :class:`~pcapkit.protocols.misc.null.NoPayload` when nothing remains.
 

pcapkit/protocols/application/dtls.py

@@ -13,6 +13,27 @@
 :meth:`~DTLS.read` sets :attr:`~DTLS.alias` from the record's legacy version,
 or to ``DTLS 1.3`` for a unified-header record.
 
+Records in One Datagram
+-----------------------
+
+A whole record after the first in a UDP datagram is dissected as a further
+:class:`DTLS` layer, the previous record's payload (:rfc:`6347#section-4.1.1`,
+:rfc:`9147#section-4.3`, :issue:`2535`): a ``ChangeCipherSpec`` followed by an
+``Alert`` reads ``UDP:DTLS 1.2:DTLS 1.2``. A unified-header record is delimited
+by its length; one without a length runs to the end of the datagram, so it is
+the last (:rfc:`9147#section-4`). The exceptions:
+
+* A whole record that fails to decode becomes
+  :class:`~pcapkit.protocols.misc.raw.Raw` together with everything after it.
+* Octets that open no whole record stay raw, with no error: an unsupported
+  content type, a legacy version outside ``0xFExx``, a header cut short, a
+  length past the end of the datagram, or a unified header with a connection
+  ID, which leaves the record's end unknown.
+* A parse told to stop at DTLS (``protocol='DTLS'`` or ``layer='Application'``)
+  keeps the rest of the datagram raw.
+* Each record is a layer, so :data:`~pcapkit.utilities.config.FRAME_LAYER_LIMIT`
+  bounds how many are dissected.
+
 Known Limitations
 -----------------
 
@@ -41,7 +62,9 @@
 from typing import TYPE_CHECKING, cast
 
 from pcapkit.const.tls.content_type import ContentType as Enum_ContentType
-from pcapkit.protocols.application.ssl import DTLS13_VERSION, SSL
+from pcapkit.corekit.decorators import beholder
+from pcapkit.protocols.application.ssl import (_DTLS_VERSION_MAJOR, _WIRE_CONTENT_TYPES,
+                                               DTLS13_VERSION, SSL)
 from pcapkit.protocols.data.application.dtls import DTLS as Data_DTLS
 from pcapkit.protocols.data.application.dtls import DTLSCiphertext as Data_DTLSCiphertext
 from pcapkit.protocols.schema.application.dtls import DTLS as Schema_DTLS
@@ -61,6 +84,10 @@
 
 __all__ = ['DTLS']
 
+#: Length of a legacy record header: content type, legacy version, epoch,
+#: sequence number and length (:rfc:`9147#section-4`).
+_LEGACY_HEADER_LENGTH = 13
+
 #: Legacy record version DTLS 1.0 carries (:rfc:`4347#section-4.1`).
 _DTLSV1_0_VERSION = 0xFEFF
 #: Legacy record version DTLS 1.2 carries, and the value DTLS 1.3 freezes its
@@ -105,6 +132,44 @@ def _ciphertext_header_length(flags: 'dict[str, int]') -> 'int':
     return 1 + (2 if flags['sequence_long'] else 1) + (2 if flags['length_present'] else 0)
 
 
+def _opens_whole_record(data: 'bytes') -> 'bool':
+    """Whether ``data`` opens with a whole DTLS record.
+
+    Only the record header is read. A legacy header gets the checks of
+    :meth:`SSL._validate_tls_record
+    <pcapkit.protocols.application.ssl.SSL._validate_tls_record>`: a content
+    type a genuine record carries, a ``0xFExx`` legacy version, and a fragment
+    that ends inside ``data``. A unified header must carry no connection ID,
+    whose negotiated length leaves the record's end unknown, and a fragment
+    that ends inside ``data``; without a length it runs to the end of ``data``,
+    as the last record of a datagram does (:rfc:`9147#section-4`).
+
+    Args:
+        data: Octets after a record.
+
+    Returns:
+        Whether a further record opens ``data`` and ends inside it.
+
+    """
+    if not data:
+        return False
+    if _is_unified_header(data[0]):
+        # NOTE: the ``C``, ``S`` and ``L`` bits of ``001CSLEE``, as the header schema reads them.
+        flags = {'connection_id': data[0] >> 4 & 1, 'sequence_long': data[0] >> 3 & 1,
+                 'length_present': data[0] >> 2 & 1}
+        header = _ciphertext_header_length(flags)
+        if flags['connection_id'] or len(data) < header:
+            return False
+        if not flags['length_present']:
+            return True
+    else:
+        header = _LEGACY_HEADER_LENGTH
+        if len(data) < header or data[0] not in _WIRE_CONTENT_TYPES or data[1] != _DTLS_VERSION_MAJOR:
+            return False
+    # Either header ends with its 16-bit length.
+    return header + int.from_bytes(data[header - 2:header], 'big') <= len(data)
+
+
 class DTLS(SSL[Data_DTLS, Schema_DTLS], schema=Schema_DTLS, data=Data_DTLS):
     """This class implements all protocols in the DTLS record-layer family.
 
@@ -148,7 +213,7 @@ def length(self) -> 'int':
         schema = getattr(self, '__header__', None)
         if isinstance(schema, Schema_DTLSCiphertext):
             return _ciphertext_header_length(schema.flags)
-        return 13
+        return _LEGACY_HEADER_LENGTH
 
     ##########################################################################
     # Methods.
@@ -323,7 +388,7 @@ def read_handshake(cls, message: 'bytes') -> 'Data_DTLSHandshake':
 
     def __length_hint__(self) -> 'Literal[13]':
         """Return an estimated length for the object."""
-        return 13
+        return _LEGACY_HEADER_LENGTH  # type: ignore[return-value]
 
     ##########################################################################
     # Utilities.
@@ -472,3 +537,53 @@ def _read_dtls_ciphertext(self, length: 'Optional[int]' = None, **kwargs: 'Any')
         )
         remaining = available - (self.length + len(schema.fragment))
         return self._decode_next_layer(dtls, -1, remaining)
+
+    def _import_next_layer(self, proto: 'int', length: 'Optional[int]' = None, *,  # type: ignore[override]
+                           packet: 'Optional[dict[str, Any]]' = None) -> 'ProtocolBase':
+        """Import next layer extractor: a further whole record, or the rest undissected.
+
+        A UDP datagram may carry several records (:rfc:`9147#section-4.3`,
+        :issue:`2535`). So the ``-1`` remainder is dissected as this class when
+        :func:`_opens_whole_record` finds a whole record opening it, and is
+        otherwise kept as :class:`~pcapkit.protocols.misc.raw.Raw`, as
+        :meth:`Application._import_next_layer
+        <pcapkit.protocols.application.application.Application._import_next_layer>`
+        keeps it.
+
+        Arguments:
+            proto: next layer protocol index; only the ``-1`` sentinel
+                (the rest of the datagram) is accepted
+            length: valid (*non-padding*) length
+            packet: packet info (passed from :meth:`self.unpack <pcapkit.protocols.protocol.Protocol.unpack>`)
+
+        Returns:
+            Instance of this class for a further record, or as
+            :meth:`Application._import_next_layer
+            <pcapkit.protocols.application.application.Application._import_next_layer>`
+            returns otherwise.
+
+        """
+        if proto == -1 and not self._sigterm and _opens_whole_record(self._get_payload()[:length]):
+            return self._import_next_record(proto, length, packet=packet)  # type: ignore[misc,call-arg]
+        return super()._import_next_layer(proto, length, packet=packet)
+
+    @beholder  # type: ignore[arg-type]
+    def _import_next_record(self, proto: 'int', length: 'Optional[int]' = None, *,  # type: ignore[misc]
+                            packet: 'Optional[dict[str, Any]]' = None) -> 'ProtocolBase':
+        """Import the further record :meth:`_import_next_layer` found.
+
+        Arguments:
+            proto: next layer protocol index, the ``-1`` sentinel
+            length: valid (*non-padding*) length
+            packet: packet info (passed from :meth:`self.unpack <pcapkit.protocols.protocol.Protocol.unpack>`)
+
+        Returns:
+            Instance of this class, or of :class:`~pcapkit.protocols.misc.raw.Raw`
+            when :func:`~pcapkit.corekit.decorators.beholder` replaces a record
+            that fails to parse.
+
+        """
+        file_ = self._get_payload()
+        return self._parse_next_layer(type(self), file_, len(file_) if length is None else length,
+                                      alias=proto, packet=packet, layer=self._exlayer,
+                                      protocol=self._exproto, __context__=self._exctx)

tests/protocols/application/test_dtls13_unified_header_unit.py

@@ -18,7 +18,7 @@
 _L_SHORT_SEQ_EPOCH1 = 0x25  # S=0, L=1, EE=01
 _L_LONG_SEQ_EPOCH2 = 0x2E   # S=1, L=1, EE=10
 _NO_LENGTH_EPOCH3 = 0x23    # S=0, L=0, EE=11
-_CID_PRESENT = 0x38         # C=1, S=0, L=1, EE=00
+_CID_PRESENT = 0x34         # C=1, S=0, L=1, EE=00
 
 #: A 16-octet stand-in for an AEAD ciphertext, the shortest the record-number
 #: sample allows (:rfc:`9147#section-4.2.3`).
@@ -187,6 +187,58 @@ def test_a_datagram_the_record_format_forbids_degrades_to_raw_on_dispatch(self)
         self.assertEqual(str(proto.protochain), 'UDP:Raw')
 
 
+class UnifiedHeaderChainTests(unittest.TestCase):
+    """How a unified-header record chains with the records sharing its datagram
+    (RFC 9147 section 4.3, #2535)."""
+
+    #: A legacy-format ``change_cipher_spec`` record, DTLS 1.2.
+    CCS = b'\x14\xfe\xfd\x00\x00' + (1).to_bytes(6, 'big') + b'\x00\x01\x01'
+
+    def setUp(self) -> None:
+        reimport_once_per_class(self)
+
+    def test_a_record_after_one_with_a_length_is_a_further_layer(self) -> None:
+        from pcapkit.protocols.application.dtls import DTLS
+
+        first = _unified(_L_SHORT_SEQ_EPOCH1, b'\x07', len(_FRAGMENT).to_bytes(2, 'big'), _FRAGMENT)
+        second = _unified(_L_LONG_SEQ_EPOCH2, b'\x12\x34', (20).to_bytes(2, 'big'), bytes(20))
+        for name, wire, chain in (('a unified header', first + second, 'DTLS 1.3:DTLS 1.3'),
+                                  ('a legacy header', first + self.CCS, 'DTLS 1.3:DTLS 1.2'),
+                                  ('after a legacy header', self.CCS + first, 'DTLS 1.2:DTLS 1.3')):
+            with self.subTest(next=name):
+                parsed = DTLS(wire, len(wire))
+                self.assertEqual(str(parsed.protochain), chain)
+                self.assertEqual(bytes(DTLS.from_data(parsed.info)), wire)
+        parsed = DTLS(first + second, len(first + second)).payload
+        self.assertEqual((parsed.info.epoch, parsed.info.sequence_number, parsed.info.length), (2, 0x1234, 20))
+
+    def test_a_record_without_a_length_is_the_last(self) -> None:
+        """It runs to the end of the datagram, so octets that would open a record are its ciphertext."""
+        from pcapkit.protocols.application.dtls import DTLS
+        from pcapkit.protocols.misc.null import NoPayload
+
+        last = _unified(_NO_LENGTH_EPOCH3, b'\x07', None, _FRAGMENT + self.CCS)
+        wire = self.CCS + last
+        parsed = DTLS(wire, len(wire))
+        self.assertEqual(str(parsed.protochain), 'DTLS 1.2:DTLS 1.3')
+        self.assertEqual(parsed.payload.info.content, _FRAGMENT + self.CCS)
+        self.assertIsInstance(parsed.payload.payload, NoPayload)
+        self.assertEqual(bytes(DTLS.from_data(parsed.info)), wire)
+
+    def test_a_connection_id_after_a_record_stays_raw_without_an_error(self) -> None:
+        """Its negotiated length leaves the record's end unknown, so no whole record opens there."""
+        from pcapkit.protocols.application.dtls import DTLS
+        from pcapkit.protocols.misc.raw import Raw
+
+        refused = _unified(_CID_PRESENT, b'\x07', len(_FRAGMENT).to_bytes(2, 'big'), _FRAGMENT)
+        wire = self.CCS * 2 + refused
+        parsed = DTLS(wire, len(wire))
+        self.assertEqual(str(parsed.protochain), 'DTLS 1.2:DTLS 1.2:Raw')
+        self.assertIs(type(parsed.payload.payload), Raw)
+        self.assertEqual(bytes(parsed.payload.payload), refused)
+        self.assertIsNone(parsed.payload.payload.info.error)
+
+
 class UnifiedHeaderMakeTests(unittest.TestCase):
     def setUp(self) -> None:
         reimport_once_per_class(self)