diff --git a/docs/source/contributing/conventions/reassembly-integration.rst b/docs/source/contributing/conventions/reassembly-integration.rst index 93ccf40328..58edecadc8 100644 --- a/docs/source/contributing/conventions/reassembly-integration.rst +++ b/docs/source/contributing/conventions/reassembly-integration.rst @@ -22,6 +22,12 @@ The Rule message, accept overlaps -- the engine does the same and adds no timeout, cap or ordering rule the RFC does not call for. +**TCP's gap cap is a safety limit, not a reassembly rule.** No RFC calls for +:data:`~pcapkit.utilities.config.TCP_GAP_LIMIT`, the widest gap TCP reassembly +zero-fills to merge a segment. It bounds the memory a capture can have zero-filled, as +:data:`~pcapkit.utilities.config.MAX_ZERO_PAD_LENGTH` does for a field, and ``-1`` +lifts it to give the uncapped reassembly point 4 asks for. Ruled on :issue:`2546`. + Compliance, Measured ~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/source/pcapkit/foundation/reassembly/tcp.rst b/docs/source/pcapkit/foundation/reassembly/tcp.rst index c29db428d2..363b720775 100644 --- a/docs/source/pcapkit/foundation/reassembly/tcp.rst +++ b/docs/source/pcapkit/foundation/reassembly/tcp.rst @@ -21,7 +21,6 @@ which reconstructs fragmented TCP packets back to origin. .. autoattribute:: __callback_fn__ :no-value: .. autoattribute:: __timeout__ - .. autoattribute:: __window__ Algorithm ========= diff --git a/docs/source/pcapkit/utilities/config.rst b/docs/source/pcapkit/utilities/config.rst index 7bdbd780d8..0c24658fed 100644 --- a/docs/source/pcapkit/utilities/config.rst +++ b/docs/source/pcapkit/utilities/config.rst @@ -61,6 +61,11 @@ Limits - ``32`` - ``>= 0``, or ``-1`` - each capture remembered pins one packet in memory + * - :envvar:`PCAPKIT_TCP_GAP_LIMIT` + - ``16777216`` + - ``>= 0``, or ``-1`` + - a TCP gap that wide is zero-filled in memory, up to 2 GiB at ``-1`` + (:issue:`2546`) * - :envvar:`PCAPKIT_FALLBACK_LIMIT` - ``10000`` - ``>= 0``, or ``-1`` @@ -99,6 +104,10 @@ Limits Sets :data:`~pcapkit.utilities.config.IPV6_TCP_WARNED_LIMIT`. +.. envvar:: PCAPKIT_TCP_GAP_LIMIT + + Sets :data:`~pcapkit.utilities.config.TCP_GAP_LIMIT`. + .. envvar:: PCAPKIT_FALLBACK_LIMIT Sets :data:`~pcapkit.utilities.config.FALLBACK_LIMIT`. @@ -127,6 +136,7 @@ Limits .. autodata:: pcapkit.utilities.config.FRAME_LAYER_LIMIT .. autodata:: pcapkit.utilities.config.EXTENSION_HEADER_LIMIT .. autodata:: pcapkit.utilities.config.IPV6_TCP_WARNED_LIMIT +.. autodata:: pcapkit.utilities.config.TCP_GAP_LIMIT .. autodata:: pcapkit.utilities.config.FALLBACK_LIMIT .. autodata:: pcapkit.utilities.config.DETECT_CACHE_SIZE .. autodata:: pcapkit.utilities.config.READ_CHUNK diff --git a/examples/generators/legacy.py b/examples/generators/legacy.py index 2cea99493c..569ccd67f2 100644 --- a/examples/generators/legacy.py +++ b/examples/generators/legacy.py @@ -47,8 +47,9 @@ each direction is a buffer of its own. :meth:`~pcapkit.foundation.reassembly.tcp.TCP.submit` runs on that direction's RST; once its FIN has been seen and every octet below the FIN - has arrived; when a SYN, or a segment farther than ``__window__`` from the - buffer, opens a new one on the same four-tuple; on a timeout, if one is + has arrived; when a SYN, or a segment farther than + :data:`~pcapkit.utilities.config.TCP_GAP_LIMIT` from the buffer, opens a + new one on the same four-tuple; on a timeout, if one is set; and, for whatever is still buffered, when the datagrams are read -- ``extraction.reassembly`` flushes every open buffer through :meth:`~pcapkit.foundation.reassembly.reassembly.Reassembly.fetch`. So a diff --git a/pcapkit/foundation/reassembly/data/tcp.py b/pcapkit/foundation/reassembly/data/tcp.py index 886ce25277..31e43f806a 100644 --- a/pcapkit/foundation/reassembly/data/tcp.py +++ b/pcapkit/foundation/reassembly/data/tcp.py @@ -236,9 +236,8 @@ class Fragment(Info): #: merge only ever shrinks or removes one, filling it from the arriving #: segment. The one exception lies wholly *outside* :attr:`raw`: when a #: stream whose SYN was captured is split rather than padded across a gap - #: wider than :attr:`TCP.__window__ - #: `, both halves record - #: the range left between them. Any entry at all makes the fragment + #: wider than :data:`~pcapkit.utilities.config.TCP_GAP_LIMIT`, both halves + #: record the range left between them. Any entry at all makes the fragment #: incomplete. #: #: This is deliberately **not** derived from diff --git a/pcapkit/foundation/reassembly/tcp.py b/pcapkit/foundation/reassembly/tcp.py index 7bd8ed2e55..e2268ddf94 100644 --- a/pcapkit/foundation/reassembly/tcp.py +++ b/pcapkit/foundation/reassembly/tcp.py @@ -18,6 +18,7 @@ DatagramID, Fragment, HoleDescriptor, Packet) from pcapkit.foundation.reassembly.reassembly import ReassemblyBase from pcapkit.protocols.transport.tcp import TCP as TCP_Protocol +from pcapkit.utilities import config if TYPE_CHECKING: from typing import Type @@ -99,6 +100,11 @@ class TCP(ReassemblyBase[Packet, Datagram, BufferID, Buffer]): `; 1.4.x kept the later octets without a record (:issue:`443`). + .. versionremoved:: 1.6.0 + + ``__window__``, now :data:`~pcapkit.utilities.config.TCP_GAP_LIMIT` + (:issue:`2546`). + """ if TYPE_CHECKING: protocol: 'Type[TCP_Protocol]' @@ -129,34 +135,6 @@ class TCP(ReassemblyBase[Packet, Datagram, BufferID, Buffer]): #: ask for one, e.g. ``2 * 120`` for 2·MSL if that is the policy wanted. __timeout__ = math.inf - #: int: Widest gap, in octets, that merging a segment into the payload - #: buffer it is read against may zero-fill: 16 MiB. The gap is measured on - #: whichever side the segment lands, from the buffer's end to the - #: segment's start, or from the segment's end to the buffer's start. - #: - #: A segment can only legitimately open a gap as wide as the data in flight, - #: which the receive window bounds. :rfc:`7323` allows windows up to - #: ``2 ** 30``, but zero-filling a gap that wide costs 1 GiB, and real - #: windows are far smaller -- Linux's default ``net.ipv4.tcp_rmem`` ceiling - #: is 6 MiB -- so 16 MiB clears them with room to spare. A segment that - #: would open a wider gap is not padded to; the buffer held so far is - #: submitted and a fresh one opened for the segment instead. What that - #: means depends on whether the buffer's SYN was captured: - #: - #: * **No SYN** -- the segment is taken to start a new stream on a reused - #: 4-tuple, and the two buffers are reported independently. - #: * **SYN captured** -- the SYN proves both are the same stream with the - #: range in between lost, so that range is recorded in the - #: :attr:`~pcapkit.foundation.reassembly.data.tcp.Fragment.gap` of the - #: payload buffer the segment was read against and of the new one, and - #: both are reported - #: :attr:`~pcapkit.foundation.reassembly.data.data.Completion.PARTIAL` - #: (in strict mode, as a single run each). The new buffer keeps the - #: SYN's header and counts as having seen it. - #: - #: Override in a subclass to change the bound. - __window__ = 1 << 24 - ########################################################################## # Methods. ########################################################################## @@ -164,6 +142,26 @@ class TCP(ReassemblyBase[Packet, Datagram, BufferID, Buffer]): def reassembly(self, info: 'Packet') -> 'None': """Reassembly procedure. + A segment is merged into the payload buffer it is read against by + zero-filling the gap between them, measured on whichever side the + segment lands: from the buffer's end to the segment's start, or from + the segment's end to the buffer's start. A gap wider than + :data:`~pcapkit.utilities.config.TCP_GAP_LIMIT` is not padded to; the + buffer held so far is submitted and a fresh one opened for the segment + instead. What that means depends on whether the buffer's SYN was + captured: + + * **No SYN** -- the segment is taken to start a new stream on a reused + 4-tuple, and the two buffers are reported independently. + * **SYN captured** -- the SYN proves both are the same stream with the + range in between lost, so that range is recorded in the + :attr:`~pcapkit.foundation.reassembly.data.tcp.Fragment.gap` of the + payload buffer the segment was read against and of the new one, and + both are reported + :attr:`~pcapkit.foundation.reassembly.data.data.Completion.PARTIAL` + (in strict mode, as a single run each). The new buffer keeps the + SYN's header and counts as having seen it. + Arguments: info: :term:`info ` dict of packets to be reassembled @@ -203,8 +201,8 @@ def reassembly(self, info: 'Packet') -> 'None': # it most plausibly continues -- this ACK's own payload buffer, or else # the most recently opened one -- per :rfc:`9293#section-3.4`. Plain # integer comparison reads a stream crossing 2**32 as a gap of about - # 4 GiB. A segment landing farther than ``__window__`` outside that - # buffer starts a new stream instead -- see ``__window__``. + # 4 GiB. A segment landing farther than ``config.TCP_GAP_LIMIT`` + # outside that buffer splits the stream instead -- see the docstring. SHIFT = 0 HDR = info.header if SYN else b'' # header of the buffer opened below, if any ANCHORED = SYN # whether that buffer's stream start is known @@ -217,7 +215,8 @@ def reassembly(self, info: 'Packet') -> 'None': END = fragment.isn + fragment.len UNWRAPPED = self._unwrap(PSN, END) # the zero-filled gap merging this segment would open, either side - if max(UNWRAPPED - END, fragment.isn - (UNWRAPPED + info.len)) > self.__window__: + limit = config.TCP_GAP_LIMIT + if limit != -1 and max(UNWRAPPED - END, fragment.isn - (UNWRAPPED + info.len)) > limit: old = self._buffer.pop(BUFID) if self._anchored(old): # The SYN proves this is still the same stream, with the @@ -552,7 +551,7 @@ def _trim_gaps(gap: 'list[tuple[int, int]]', first: 'int', last: 'int') -> 'None """Remove a received range from a fragment's gap list, in place. Only an entry lying outside :attr:`~pcapkit.foundation.reassembly.data.tcp.Fragment.raw` - -- the range a split stream left out, see :attr:`TCP.__window__` -- can + -- the range a split stream left out, see :meth:`TCP.reassembly` -- can still overlap a segment once it is merged; :meth:`_merge_overlap` has already closed every entry inside ``raw`` that the segment covers. @@ -806,7 +805,7 @@ def submit(self, buf: 'Buffer', *, bufid: 'BufferID', # type: ignore[override] # How completely this buffer came out, and why it stopped. Derived # once per buffer, so the two branches cannot disagree about it. # Any gap entry counts, including one lying wholly outside ``raw``: - # that is the range a split stream left out (see ``__window__``). + # that is the range a split stream left out (see ``reassembly``). completion = Completion.COMPLETE if not buffer.gap else ( Completion.TIMEOUT if timeout else Completion.PARTIAL ) diff --git a/pcapkit/utilities/config.py b/pcapkit/utilities/config.py index 9f91206830..7a7248c91e 100644 --- a/pcapkit/utilities/config.py +++ b/pcapkit/utilities/config.py @@ -51,7 +51,7 @@ __all__ = [ # limits 'FRAME_LAYER_LIMIT', 'EXTENSION_HEADER_LIMIT', 'IPV6_TCP_WARNED_LIMIT', - 'FALLBACK_LIMIT', 'DETECT_CACHE_SIZE', 'READ_CHUNK', + 'TCP_GAP_LIMIT', 'FALLBACK_LIMIT', 'DETECT_CACHE_SIZE', 'READ_CHUNK', 'MAX_ZERO_PAD_LENGTH', 'MAX_ZERO_PAD_SHORTFALL', 'ZERO_PAD_BUDGET_RATIO', # readers 'read_limit', 'read_flag', @@ -215,6 +215,31 @@ def read_flag(name: 'str', default: 'bool' = False) -> 'bool': IPV6_TCP_WARNED_LIMIT = read_limit('IPV6_TCP_WARNED_LIMIT', 32, risk=( 'each capture remembered pins one of its packets in memory')) +#: int: Widest gap, in octets, that +#: :meth:`TCP.reassembly ` +#: zero-fills to merge a segment into the payload buffer it is read against, +#: :envvar:`PCAPKIT_TCP_GAP_LIMIT`; ``-1`` always merges. +#: +#: This is a safety limit, not a reassembly rule: no RFC calls for one +#: (:issue:`2546`). A segment can only legitimately open a gap as wide as the +#: data in flight, which the receive window bounds. :rfc:`7323` allows windows +#: up to ``2 ** 30``, but zero-filling a gap that wide costs 1 GiB, and real +#: windows are far smaller -- Linux's default ``net.ipv4.tcp_rmem`` ceiling is +#: 6 MiB -- so the default of 16 MiB clears them with room to spare. Past it, +#: the stream is split rather than padded, as :meth:`TCP.reassembly +#: ` describes. +#: +#: At ``-1`` only the sequence space bounds the gap: a sequence number is read +#: within ``2 ** 31`` of the data buffered (:rfc:`9293#section-3.4`), so one +#: segment can have up to 2 GiB zero-filled. +#: +#: .. versionadded:: 1.6.0 +#: +#: Moved from ``pcapkit.foundation.reassembly.tcp.TCP.__window__`` (:issue:`2546`). +TCP_GAP_LIMIT = read_limit('TCP_GAP_LIMIT', 1 << 24, risk=( + 'merging a segment that far from the data buffered zero-fills the gap in memory, ' + 'up to 2 GiB per segment at -1')) + #: int: Most objects the ``_append_fallback`` method of a dumper #: :func:`~pcapkit.dumpkit.common.make_dumper` builds #: expands for one top-level object, counting an object once per time it is met, diff --git a/tests/foundation/reassembly/test_tcp_reassembly_roundtrip_unit.py b/tests/foundation/reassembly/test_tcp_reassembly_roundtrip_unit.py index a19148b416..98409c0584 100644 --- a/tests/foundation/reassembly/test_tcp_reassembly_roundtrip_unit.py +++ b/tests/foundation/reassembly/test_tcp_reassembly_roundtrip_unit.py @@ -13,6 +13,9 @@ * #1356 -- a FIN that submits the buffer at once splits data reordered behind it off into a buffer of its own. +The widest gap a segment is padded across is ``config.TCP_GAP_LIMIT``, a safety +limit that ``-1`` lifts (#2546). + Every case feeds hand-built segments and reads no capture. Classes are imported inside each test, after :func:`~tests._support.reimport_once_per_class`. @@ -174,14 +177,62 @@ def test_a_segment_far_below_the_buffer_starts_a_new_stream(self) -> None: reasm = self._run(self._seg(0x30000000, b'A' * 5), self._seg(0x10000000, b'B' * 5)) self.assertEqual(sorted(p for (_, _, _, p) in self._summary(reasm)), [b'A' * 5, b'B' * 5]) - def test_a_gap_within_the_window_is_still_padded(self) -> None: - from pcapkit.foundation.reassembly.tcp import TCP + def test_a_gap_at_the_default_limit_is_still_padded(self) -> None: + from pcapkit.utilities import config - far = TCP.__window__ + far = config.TCP_GAP_LIMIT reasm = self._run(self._seg(0, b'A' * 5), self._seg(5 + far, b'B' * 5), strict=False) (completed, _, index, payload), = self._summary(reasm) self.assertEqual((completed, index, len(payload)), ('PARTIAL', (1, 2), 10 + far)) + # #2546 ------------------------------------------------------------------ + + def _gap_limit(self, limit): + """Patch ``config.TCP_GAP_LIMIT`` to ``limit`` for the rest of the test.""" + from unittest import mock + + from pcapkit.utilities import config + + patcher = mock.patch.object(config, 'TCP_GAP_LIMIT', limit) + patcher.start() + self.addCleanup(patcher.stop) + + def test_a_gap_up_to_the_limit_is_padded(self) -> None: + self._gap_limit(100) + # the gap is measured on either side of the buffer + for side, segs, runs in (('after', (self._seg(0, b'A' * 5), self._seg(105, b'B' * 5)), + (b'A' * 5, b'B' * 5)), + ('before', (self._seg(200, b'A' * 5), self._seg(95, b'B' * 5)), + (b'B' * 5, b'A' * 5))): + with self.subTest(side=side): + self.assertEqual(self._summary(self._run(*segs)), [('PARTIAL', 500, (1, 2), runs)]) + + def test_a_gap_past_the_limit_splits_the_stream(self) -> None: + self._gap_limit(100) + for side, seq in (('after', 106), ('before', 94)): + with self.subTest(side=side, syn=False): + reasm = self._run(self._seg(200 if side == 'before' else 0, b'A' * 5), + self._seg(seq, b'B' * 5)) + self.assertEqual(sorted((c, p) for (c, _, _, p) in self._summary(reasm)), + [('COMPLETE', b'A' * 5), ('COMPLETE', b'B' * 5)]) + # a captured SYN keeps both halves one stream, each missing the gap + reasm = self._run(self._seg(0, syn=True), self._seg(1, b'A' * 5), self._seg(107, b'B' * 5)) + self.assertEqual(self._summary(reasm), [('PARTIAL', 500, (3,), (b'B' * 5,)), + ('PARTIAL', 500, (1, 2), (b'A' * 5,))]) + self.assertEqual(reasm._buffer[BUFID].ack[500].gap, [(6, 106)]) + + def test_minus_one_pads_a_gap_past_the_default(self) -> None: + self._gap_limit(-1) + far = (1 << 24) + 1 + start = far + 10 + for side, segs, runs in (('after', (self._seg(0, b'A' * 5), self._seg(5 + far, b'B' * 5)), + (b'A' * 5, b'B' * 5)), + ('before', (self._seg(start, b'A' * 5), + self._seg(start - far - 5, b'B' * 5)), + (b'B' * 5, b'A' * 5))): + with self.subTest(side=side): + self.assertEqual(self._summary(self._run(*segs)), [('PARTIAL', 500, (1, 2), runs)]) + def test_data_reordered_behind_the_fin_joins_the_stream(self) -> None: reasm = self._run(self._seg(100, syn=True, ack=0), self._seg(101, b'AAAAA'), self._seg(111, fin=True), self._seg(106, b'BBBBB')) diff --git a/tests/project/test_rfc_constants_one_home_1739_unit.py b/tests/project/test_rfc_constants_one_home_1739_unit.py index a349ab8595..771ba1c1d7 100644 --- a/tests/project/test_rfc_constants_one_home_1739_unit.py +++ b/tests/project/test_rfc_constants_one_home_1739_unit.py @@ -350,8 +350,6 @@ 'an if_tsoffset option: 4 octets of header and 8 of value', ('pcapkit/foundation/engines/scapy.py', 'options[4:12]'): 'the 8-octet value of an if_tsoffset option', - ('pcapkit/foundation/reassembly/tcp.py', '1 << 24'): - 'a bit width, not a PCAP global header', ('pcapkit/toolkit/dpkt.py', '_fragment(6)'): 'the IP version number', ('pcapkit/toolkit/dpkt.py', 'version == 6'): diff --git a/tests/utilities/test_config_unit.py b/tests/utilities/test_config_unit.py index 0757603752..bb47b41bff 100644 --- a/tests/utilities/test_config_unit.py +++ b/tests/utilities/test_config_unit.py @@ -72,6 +72,7 @@ ('FRAME_LAYER_LIMIT', 64, 0, 'RecursionError (#1610)'), ('EXTENSION_HEADER_LIMIT', 32, 0, 'RecursionError (#1610)'), ('IPV6_TCP_WARNED_LIMIT', 32, 0, 'pins one of its packets in memory'), + ('TCP_GAP_LIMIT', 16_777_216, 0, 'zero-fills the gap in memory'), ('FALLBACK_LIMIT', 10_000, 0, 'a large or endless object graph'), ('DETECT_CACHE_SIZE', 1024, 0, 'never-repeating text'), ('READ_CHUNK', 1_048_576, 1, 'a bogus record length allocates'), @@ -301,6 +302,12 @@ def test_a_moved_name_has_no_re_export(self) -> None: with self.subTest(module=module, name=name): self.assertFalse(hasattr(loaded, name)) + def test_the_tcp_gap_limit_left_no_class_attribute(self) -> None: + # #2546: TCP.__window__ moved here as TCP_GAP_LIMIT. + from pcapkit.foundation.reassembly.tcp import TCP + + self.assertFalse(hasattr(TCP, '__window__')) + def test_every_variable_is_documented(self) -> None: from pcapkit.utilities import config