feat(application): dissect every HTTP/2 frame in a segment

docs/source/contributing/conventions/protocol-family-grouping.rst

@@ -237,6 +237,13 @@ previous record's payload, so a TLS record's dynamic alias names its own version
 ``CLIENT-HELLO`` then a ``CLIENT-MASTER-KEY`` read ``TCP:SSLv2:SSLv2``. DTLS does not
 chain: two DTLS records in one datagram read ``DTLS 1.2:Raw`` (:issue:`2535`).
 
+**HTTP/2 frames chain the same way** (:issue:`2536`): a ``SETTINGS`` frame then a
+``WINDOW_UPDATE`` read ``TCP:HTTP/2:HTTP/2``. A later whole frame that fails to decode
+is :class:`~pcapkit.protocols.misc.raw.Raw` with the rest of the segment and the
+parser's ``error``, a frame the segment ends inside stays
+:class:`~pcapkit.protocols.misc.raw.Raw` with none, and ``protocol=``, ``layer=`` and
+:data:`~pcapkit.utilities.config.FRAME_LAYER_LIMIT` stop the chain, as for TLS.
+
 .. _protocol-family-canonical-names:
 
 Canonical Names

docs/source/pcapkit/protocols/application/httpv2.rst

@@ -20,6 +20,26 @@ Octets      Bits        Name                    Description
   9          72   ``http.payload``            Frame Payload
 ======= ========= ===================== ==========================
 
+Frames in One Segment
+---------------------
+
+A whole frame after the first in a TCP segment is dissected as a further
+:class:`~pcapkit.protocols.application.httpv2.HTTPv2` layer, the previous
+frame's payload (:issue:`2536`), as
+:class:`~pcapkit.protocols.application.tls.TLS` chains records: a ``SETTINGS``
+frame followed by a ``WINDOW_UPDATE`` reads ``TCP:HTTP/2:HTTP/2``. The
+exceptions:
+
+* A whole frame that fails to decode becomes
+  :class:`~pcapkit.protocols.misc.raw.Raw` together with everything after it,
+  with the parser's ``error``.
+* A frame the segment ends inside, its header or its payload cut short, stays
+  raw, with no error.
+* A parse told to stop at HTTP/2 (``protocol='HTTP/2'`` or
+  ``layer='Application'``) keeps the rest of the segment raw.
+* Each frame is a layer, so :data:`~pcapkit.utilities.config.FRAME_LAYER_LIMIT`
+  bounds how many are dissected.
+
 .. versionchanged:: 1.5.0
 
    A frame's Length counts its payload alone, not the 9-octet frame header,

pcapkit/protocols/application/application.py

@@ -46,7 +46,8 @@ class Application(ProtocolBase[_PT, _ST], Generic[_PT, _ST]):  # pylint: disable
     :class:`~pcapkit.protocols.misc.raw.Raw` -- trailing bytes are not a further
     protocol layer, unless the subclass dissects them, as
     :class:`~pcapkit.protocols.application.tls.TLS` and
-    :class:`~pcapkit.protocols.application.sslv2.SSLv2` do for a further record --
+    :class:`~pcapkit.protocols.application.sslv2.SSLv2` do for a further record and
+    :class:`~pcapkit.protocols.application.httpv2.HTTPv2` for a further frame --
     or to :class:`~pcapkit.protocols.misc.null.NoPayload` when nothing remains.
 
     """

pcapkit/protocols/application/httpv2.py

@@ -23,6 +23,25 @@
 
 .. [*] https://en.wikipedia.org/wiki/HTTP/2
 
+Frames in One Segment
+---------------------
+
+A whole frame after the first in a TCP segment is dissected as a further
+:class:`HTTPv2` layer, the previous frame's payload (:issue:`2536`), as
+:class:`~pcapkit.protocols.application.tls.TLS` chains records: a ``SETTINGS``
+frame followed by a ``WINDOW_UPDATE`` reads ``TCP:HTTP/2:HTTP/2``. The
+exceptions:
+
+* A whole frame that fails to decode becomes
+  :class:`~pcapkit.protocols.misc.raw.Raw` together with everything after it,
+  with the parser's ``error``.
+* A frame the segment ends inside, its header or its payload cut short, stays
+  raw, with no error.
+* A parse told to stop at HTTP/2 (``protocol='HTTP/2'`` or
+  ``layer='Application'``) keeps the rest of the segment raw.
+* Each frame is a layer, so :data:`~pcapkit.utilities.config.FRAME_LAYER_LIMIT`
+  bounds how many are dissected.
+
 .. versionchanged:: 1.5.0
 
    A frame's Length counts its payload alone, not the 9-octet frame header,
@@ -47,6 +66,7 @@ class is renamed :class:`~pcapkit.protocols.application.httpv2.HTTPv2`,
 from pcapkit.const.http.error_code import ErrorCode as Enum_ErrorCode
 from pcapkit.const.http.frame import Frame as Enum_Frame
 from pcapkit.const.http.setting import Setting as Enum_Setting
+from pcapkit.corekit.decorators import beholder
 from pcapkit.corekit.multidict import MultiDict, OrderedMultiDict
 from pcapkit.protocols.application.http import _HTTP2_PREFACE
 from pcapkit.protocols.application.http import HTTP as HTTPBase
@@ -115,6 +135,24 @@ class is renamed :class:`~pcapkit.protocols.application.httpv2.HTTPv2`,
 __all__ = ['HTTPv2']
 
 
+def _opens_whole_frame(data: 'bytes') -> 'bool':
+    """Whether ``data`` opens with a whole HTTP/2 frame.
+
+    Only the 9-octet frame header is read: the frame is whole when its Length,
+    which counts the frame payload alone (:rfc:`9113#section-4.1`), ends inside
+    ``data``. The type, flags and stream are left to :meth:`HTTPv2.read`, so a
+    whole frame it refuses is kept with its error.
+
+    Args:
+        data: Octets after a frame.
+
+    Returns:
+        Whether a further frame opens ``data`` and ends inside it.
+
+    """
+    return len(data) >= 9 and 9 + int.from_bytes(data[:3], 'big') <= len(data)
+
+
 class HTTPv2(HTTPBase[Data_HTTP, Schema_HTTP],
            schema=Schema_HTTP, data=Data_HTTP):
     """This class implements Hypertext Transfer Protocol (HTTP/2).
@@ -162,10 +200,14 @@ class HTTPv2(HTTPBase[Data_HTTP, Schema_HTTP],
 
     .. versionchanged:: 1.6.0
 
-       A payload opening with the connection preface parses, with the preface
-       kept as ``preface`` and counted in :attr:`length`, and :meth:`make` takes
-       ``preface``, since a dispatched HTTP/2 payload reaches this class rather
-       than :class:`~pcapkit.protocols.application.http.HTTP` (:issue:`2524`).
+       * A payload opening with the connection preface parses, with the preface
+         kept as ``preface`` and counted in :attr:`length`, and :meth:`make`
+         takes ``preface``, since a dispatched HTTP/2 payload reaches this class
+         rather than :class:`~pcapkit.protocols.application.http.HTTP`
+         (:issue:`2524`).
+       * A whole frame after the first in a segment is a further layer of this
+         class, where it was kept as :class:`~pcapkit.protocols.misc.raw.Raw`
+         (:issue:`2536`).
 
     """
 
@@ -361,8 +403,8 @@ def read(self, length: 'Optional[int]' = None, **kwargs: 'Any') -> 'Data_HTTP':
                 'preface': _HTTP2_PREFACE,
             })
 
-        # The frame ends at its declared Length; whatever follows it in the
-        # buffer (e.g. the next frame) is the undissected remainder.
+        # The frame ends at its declared Length; what follows it is a further
+        # whole frame, dissected as one, or else the undissected remainder (#2536).
         return self._decode_next_layer(http, -1, length - 9 - schema.length)
 
     def make(self,  # type: ignore[override]
@@ -1622,3 +1664,55 @@ def _make_http_continuation(self, frame: 'Optional[Data_ContinuationFrame]' = No
         return Schema_ContinuationFrame(
             fragment=fragment,
         ), flags
+
+    def _import_next_layer(self, proto: 'int', length: 'Optional[int]' = None, *,  # type: ignore[override]
+                           packet: 'Optional[dict[str, Any]]' = None) -> 'ProtocolBase':
+        """Import next layer extractor: a further whole frame, or the rest undissected.
+
+        A TCP segment may carry several frames (:issue:`2536`). So the ``-1``
+        remainder is dissected as this class when :func:`_opens_whole_frame`
+        finds a whole frame opening it, and is otherwise kept as
+        :class:`~pcapkit.protocols.misc.raw.Raw`, as
+        :meth:`Application._import_next_layer
+        <pcapkit.protocols.application.application.Application._import_next_layer>`
+        keeps it. A frame the segment ends inside is the reassembly limit, and
+        stays raw.
+
+        Arguments:
+            proto: next layer protocol index; only the ``-1`` sentinel
+                (the rest of the segment) is accepted
+            length: valid (*non-padding*) length
+            packet: packet info (passed from :meth:`self.unpack <pcapkit.protocols.protocol.Protocol.unpack>`)
+
+        Returns:
+            Instance of this class for a further frame, or as
+            :meth:`Application._import_next_layer
+            <pcapkit.protocols.application.application.Application._import_next_layer>`
+            returns otherwise.
+
+        """
+        if proto == -1 and _opens_whole_frame(self._get_payload()[:length]) and not self._sigterm:
+            return self._import_next_frame(proto, length, packet=packet)  # type: ignore[misc,call-arg]
+        return super()._import_next_layer(proto, length, packet=packet)
+
+    @beholder  # type: ignore[arg-type]
+    def _import_next_frame(self, proto: 'int', length: 'Optional[int]' = None, *,  # type: ignore[misc]
+                           packet: 'Optional[dict[str, Any]]' = None) -> 'ProtocolBase':
+        """Import the further frame :meth:`_import_next_layer` found.
+
+        Arguments:
+            proto: next layer protocol index, the ``-1`` sentinel
+            length: valid (*non-padding*) length
+            packet: packet info (passed from :meth:`self.unpack <pcapkit.protocols.protocol.Protocol.unpack>`)
+
+        Returns:
+            Instance of this class, or of :class:`~pcapkit.protocols.misc.raw.Raw`
+            carrying the parser's ``error`` when
+            :func:`~pcapkit.corekit.decorators.beholder` replaces a frame that
+            fails to parse.
+
+        """
+        file_ = self._get_payload()
+        return self._parse_next_layer(type(self), file_, len(file_) if length is None else length,
+                                      alias=proto, packet=packet, layer=self._exlayer,
+                                      protocol=self._exproto, __context__=self._exctx)

tests/protocols/application/test_http2_roundtrip_unit.py

@@ -15,6 +15,9 @@
 * #1251 (HTTP/2 half) -- ``PING`` opaque data of the wrong width was padded or
   cut rather than refused.
 
+Segments of several frames rebuild both ways too, each frame a layer of its own
+(#2536).
+
 Every case builds its own octets in memory and reads no capture. Classes are
 imported inside each test, after :func:`~tests._support.reimport_once_per_class`.
 
@@ -50,6 +53,44 @@
     '1247 SETTINGS A, B, A': bytes.fromhex('000012040000000000' '000100000001' '000200000000' '000100000003'),
 }
 
+SETTINGS = bytes.fromhex('000006040000000000' '000300000064')
+WINDOW_UPDATE = bytes.fromhex('000004080000000000' '0000ffff')
+
+#: Segments of several frames, every one dissected as an HTTP/2 layer (#2536).
+SEGMENTS = {
+    'SETTINGS + WINDOW_UPDATE': SETTINGS + WINDOW_UPDATE,
+    'SETTINGS + SETTINGS ACK + WINDOW_UPDATE': SETTINGS + SETTINGS_ACK + WINDOW_UPDATE,
+    'preface + SETTINGS + SETTINGS ACK + WINDOW_UPDATE': PREFACE + SETTINGS + SETTINGS_ACK + WINDOW_UPDATE,
+    'HEADERS + DATA PADDED + DATA END_STREAM': (bytes.fromhex('000002010400000001' '8286')
+                                                + FRAMES['1244 DATA PADDED, Pad Length 0']
+                                                + bytes.fromhex('000003000100000001616263')),
+}
+
+#: A frame the segment ends inside: 7 octets of a 13-octet ``WINDOW_UPDATE``.
+PARTIAL_FRAME = WINDOW_UPDATE[:7]
+
+
+def _split_frames(wire: bytes) -> 'list[bytes]':
+    """``wire`` past any preface, cut at each frame's Length (RFC 9113, section 4.1)."""
+    wire = wire[len(PREFACE):] if wire.startswith(PREFACE) else wire
+    frames = []
+    while wire:
+        end = 9 + int.from_bytes(wire[:3], 'big')
+        frames.append(wire[:end])
+        wire = wire[end:]
+    return frames
+
+
+def _layers(proto: 'object') -> 'tuple[list, object]':
+    """Each HTTP/2 layer of ``proto``'s chain, in order, and the layer that ends it."""
+    from pcapkit.protocols.application.httpv2 import HTTPv2
+
+    layers = []
+    while isinstance(proto, HTTPv2):
+        layers.append(proto)
+        proto = proto.payload
+    return layers, proto
+
 
 class TestHTTP2RoundTrip(unittest.TestCase):
     """Pin the rebuild of parsed HTTP/2 frames, and ``make``'s input checks."""
@@ -180,5 +221,84 @@ def test_make_ping_rejects_wrong_width_opaque_data(self) -> None:
                          b'01234567')
 
 
+class TestHTTP2SegmentRoundTrip(unittest.TestCase):
+    """Pin every frame of a segment as its own layer, rebuilt both directions (#2536)."""
+
+    def setUp(self) -> None:
+        reimport_once_per_class(self)
+
+    def test_every_frame_is_an_http2_layer(self) -> None:
+        """Each layer holds the frame header the specification frames (RFC 9113, section 4.1)."""
+        from pcapkit.protocols.application.httpv2 import HTTPv2
+        from pcapkit.protocols.misc.null import NoPayload
+
+        for name, wire in SEGMENTS.items():
+            with self.subTest(segment=name):
+                frames = _split_frames(wire)
+                layers, rest = _layers(HTTPv2(wire, len(wire)))
+                self.assertEqual(len(layers), len(frames))
+                self.assertIsInstance(rest, NoPayload)
+                for layer, frame in zip(layers, frames):
+                    self.assertEqual(layer.info.length, len(frame) - 9)
+                    self.assertEqual(int(layer.info.type), frame[3])
+                    self.assertEqual(int(layer.info.flags.__value__), frame[4])
+                    self.assertEqual(layer.info.sid, int.from_bytes(frame[5:9], 'big') & 0x7FFFFFFF)
+
+    def test_parse_then_rebuild_is_byte_exact(self) -> None:
+        from pcapkit.protocols.application.http import HTTP
+        from pcapkit.protocols.application.httpv2 import HTTPv2
+        from pcapkit.protocols.transport.tcp import TCP
+
+        tcp_to_80 = bytes.fromhex('c350005000000001000000015010ffff00000000')
+        for name, wire in SEGMENTS.items():
+            with self.subTest(segment=name):
+                parsed = HTTP(wire, len(wire))
+                for rebuilt in (HTTP.from_data(parsed.info), HTTPv2.from_data(parsed.info),
+                                HTTP.from_data(parsed.info.to_dict())):
+                    self.assertEqual(bytes(rebuilt).hex(), wire.hex())
+                    self.assertEqual(len(_layers(rebuilt)[0]), len(_split_frames(wire)))
+
+                segment = tcp_to_80 + wire
+                tcp = TCP(segment, len(segment))
+                self.assertEqual(str(tcp.protochain), f'TCP:{parsed.protochain}')
+                self.assertEqual(bytes(TCP.from_data(tcp.info)).hex(), segment.hex())
+
+    def test_make_then_parse_is_byte_exact(self) -> None:
+        """Each frame made from its parsed fields, carrying the next as ``payload``."""
+        from pcapkit.protocols.application.httpv2 import HTTPv2
+
+        for name, wire in SEGMENTS.items():
+            with self.subTest(segment=name):
+                parsed = HTTPv2(wire, len(wire))
+                layers, _ = _layers(parsed)
+                maker = object.__new__(HTTPv2)
+                schema = b''  # type: object
+                for layer in reversed(layers[1:]):
+                    schema = maker.make(type=layer.info.type, flags=layer.info.flags.__value__,
+                                        reserved=layer.info.reserved, sid=layer.info.sid,
+                                        frame=layer.info, payload=schema)
+                first = layers[0].info
+                made = HTTPv2(type=first.type, flags=first.flags.__value__, reserved=first.reserved,
+                              sid=first.sid, frame=first, payload=schema, preface='preface' in first)
+                self.assertEqual(bytes(made).hex(), wire.hex())
+                self.assertEqual(str(made.protochain), str(parsed.protochain))
+                self.assertEqual(str(HTTPv2(bytes(made), len(wire)).protochain), str(parsed.protochain))
+
+    def test_trailing_partial_frame_stays_raw(self) -> None:
+        """The frame a segment ends inside is the reassembly limit, kept as its own octets."""
+        from pcapkit.protocols.application.http import HTTP
+        from pcapkit.protocols.misc.raw import Raw
+
+        for name, segment in SEGMENTS.items():
+            with self.subTest(segment=name):
+                wire = segment + PARTIAL_FRAME
+                parsed = HTTP(wire, len(wire))
+                layers, rest = _layers(parsed)
+                self.assertEqual(len(layers), len(_split_frames(segment)))
+                self.assertIs(type(rest), Raw)
+                self.assertEqual(bytes(rest), PARTIAL_FRAME)
+                self.assertEqual(bytes(HTTP.from_data(parsed.info)).hex(), wire.hex())
+
+
 if __name__ == '__main__':
     unittest.main()