diff --git a/pcapkit/protocols/schema/misc/pcapng.py b/pcapkit/protocols/schema/misc/pcapng.py index ff61179147..2c7c018ad6 100644 --- a/pcapkit/protocols/schema/misc/pcapng.py +++ b/pcapkit/protocols/schema/misc/pcapng.py @@ -323,6 +323,52 @@ def callback(pkt: 'dict[str, Any]') -> 'int': return callback +def held_body(framing: 'int') -> 'Callable[[dict[str, Any]], int]': + """Bound a block body read as one field to the octets the stream holds. + + Args: + framing: Octets the block's fixed fields span, both copies of the Block + Total Length included. + + Returns: + A callback returning the block's Block Total Length less ``framing``, + never below zero, and never past what is left of the stream. + + The bodies of :class:`UnknownBlock`, :class:`CustomBlock` and + :class:`SystemdJournalExportBlock` are each one + :class:`~pcapkit.corekit.fields.strings.BytesField` spanning the whole + block, so a Block Total Length that runs past the end of the file is one + field declaring that many octets. Unbounded, that field is zero-filled + within :data:`~pcapkit.utilities.config.MAX_ZERO_PAD_LENGTH`, and past it + :meth:`FieldBase.unpack ` + raises :exc:`~pcapkit.utilities.exceptions.FieldValueError` from inside the + block's schema, where :meth:`PCAPNG.read + ` cannot keep the block as + captured, so the whole extraction ends (:issue:`1984`). The other block + types read their bodies field by field, so none of them declares that much + in one field. + + Bounded, the body is the octets the file actually holds, whatever the + block declares, and :meth:`PCAPNG.read + ` reports the truncation as it + does for every other block type. On a well-formed block the stream holds + the body and the trailing Block Total Length, so this is a no-op; on the + packing path :meth:`Schema.pack + ` leaves ``__length__`` at + ``-1`` for "unknown", and the declared span is kept. + + """ + floored = nonnegative(lambda pkt: pkt['length'] - framing) + + def callback(pkt: 'dict[str, Any]') -> 'int': + nominal = floored(pkt) + remaining = pkt.get('__length__') + if not isinstance(remaining, int) or remaining < 0: + return nominal + return min(nominal, remaining) + return callback + + def bounded_option(length: 'Callable[[dict[str, Any]], int]') -> 'Callable[[dict[str, Any]], int]': """Refuse an option or record payload that runs past its area. @@ -903,7 +949,7 @@ class UnknownBlock(BlockType): #: Block total length. length: 'int' = UInt32Field(callback=byteorder_callback) #: Block body (including padding). - body: 'bytes' = BytesField(length=nonnegative(lambda pkt: pkt['length'] - GENERIC_BLOCK_MINIMUM)) + body: 'bytes' = BytesField(length=held_body(GENERIC_BLOCK_MINIMUM)) #: Block total length. length2: 'int' = UInt32Field(callback=byteorder_callback) @@ -2089,7 +2135,7 @@ class SystemdJournalExportBlock(BlockType, code=Enum_BlockType.systemd_Journal_E #: Block total length. length: 'int' = UInt32Field(callback=byteorder_callback) #: Journal entry. - entry: 'bytes' = BytesField(length=nonnegative(lambda pkt: pkt['length'] - GENERIC_BLOCK_MINIMUM)) + entry: 'bytes' = BytesField(length=held_body(GENERIC_BLOCK_MINIMUM)) #: Block total length. length2: 'int' = UInt32Field(callback=byteorder_callback) @@ -2517,8 +2563,8 @@ class CustomBlock(BlockType, code=[Enum_BlockType.Custom_Block_that_rewriters_ca # NOTE: both custom block types span the same fixed fields, so the one # entry stands for both. #: Custom data (incl. padding and options). - data: 'bytes' = BytesField(length=nonnegative( - lambda pkt: pkt['length'] - BLOCK_MINIMUM[Enum_BlockType.Custom_Block_that_rewriters_can_copy_into_new_files])) + data: 'bytes' = BytesField(length=held_body( + BLOCK_MINIMUM[Enum_BlockType.Custom_Block_that_rewriters_can_copy_into_new_files])) #: Block total length. length2: 'int' = UInt32Field(callback=byteorder_callback) diff --git a/tests/protocols/misc/test_pcapng_cut_block_past_eof_1984_unit.py b/tests/protocols/misc/test_pcapng_cut_block_past_eof_1984_unit.py new file mode 100644 index 0000000000..2cbbc29de3 --- /dev/null +++ b/tests/protocols/misc/test_pcapng_cut_block_past_eof_1984_unit.py @@ -0,0 +1,131 @@ +# -*- coding: utf-8 -*- +"""GitHub issue #1984: an unknown, custom or journal block the file cuts short is kept. + +The bodies of an unknown block, a Custom Block and a systemd Journal Export +Block are each one field spanning the whole block. A Block Total Length past +the end of the file made that one field declare the missing octets, and past +the zero-pad ceiling the read raised +:exc:`~pcapkit.utilities.exceptions.FieldValueError` from inside the block's +schema, which ended the whole extraction and lost every earlier frame. The body +is sized from the octets the file holds, at any declared length, so the +block is reported as truncated as every other block type is, and extraction +keeps the frames before it. + +:mod:`pcapkit` is imported inside each test, after +:func:`~tests._support.reimport_once_per_class`. + +""" + +import io +import struct +import unittest +import warnings + +from tests.protocols.misc.test_pcapng_roundtrip2_unit import PCAPNGTestCase + +FRAME = bytes(range(1, 61)) + +#: An unregistered block type, the systemd Journal Export Block, and a Custom Block. +UNKNOWN = 0x77 +JOURNAL = 0x9 +CUSTOM = 0xBAD + +#: Block Total Lengths past and within the zero-pad ceiling. +LENGTHS = (0x7FFF_FFF0, 0x10000) + +#: The octets the file holds of a cut block after its Block Total Length. +TAIL = bytes(range(1, 9)) + +#: The body field of each block type. +FIELD = {UNKNOWN: 'body', JOURNAL: 'entry', CUSTOM: 'data'} + + +def _block(btype: 'int', body: 'bytes') -> 'bytes': + """A little-endian block of ``btype`` around ``body``, padded to 32 bits.""" + body += bytes(-len(body) % 4) + total = 12 + len(body) + return struct.pack(' 'bytes': + """An Enhanced Packet Block on interface 0 holding :data:`FRAME`.""" + return _block(6, struct.pack(' 'bytes': + """A block of ``btype`` declaring ``length`` that the file ends inside.""" + return struct.pack(' 'tuple[list[bytes], list[str]]': + """Extract ``data``, returning its frames' packets and the warnings raised.""" + import pcapkit + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter('always') + extraction = pcapkit.extract(fin=io.BytesIO(data), nofile=True, store=True, + engine='default') + return ([bytes(frame.info.packet) for frame in extraction.frame], + [str(item.message) for item in caught]) + + def test_earlier_frames_survive(self) -> None: + for btype in FIELD: + for length in LENGTHS: + with self.subTest(btype=hex(btype), length=hex(length)): + frames, messages = self._extract(SHB + IDB + _epb() + _cut(btype, length)) + self.assertEqual(frames, [FRAME]) + self.assertIn(f'PCAP-NG: [Block {btype}] block length {length} exceeds the ' + f'{len(TAIL) + 8} octet(s) left in the file; block truncated', + messages) + + def test_body_is_the_octets_held(self) -> None: + for btype, name in FIELD.items(): + for length in LENGTHS: + with self.subTest(btype=hex(btype), length=hex(length)): + octets = _cut(btype, length) + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + parsed = self._parse(octets) + held = TAIL[4:] if btype == CUSTOM else TAIL + self.assertEqual(getattr(parsed.__header__.block, name), held) + self.assertEqual(parsed.info.length, length) + + def test_cut_block_rebuilds_as_captured(self) -> None: + for btype in FIELD: + for length in LENGTHS: + with self.subTest(btype=hex(btype), length=hex(length)): + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + self.assertRebuilds(_cut(btype, length)) + + def test_well_formed_blocks_are_unchanged(self) -> None: + for btype, body in BODY.items(): + with self.subTest(btype=hex(btype)): + octets = _block(btype, body) + frames, messages = self._extract(SHB + IDB + _epb() + octets + _epb()) + self.assertEqual(frames, [FRAME, FRAME]) + self.assertFalse(any('block truncated' in text for text in messages), messages) + self.assertFalse(any('block length mismatch' in text for text in messages), messages) + parsed = self._parse(octets) + held = body[4:] if btype == CUSTOM else body + self.assertEqual(getattr(parsed.__header__.block, FIELD[btype]), + held + bytes(-len(held) % 4)) + self.assertRebuilds(octets) + + +if __name__ == '__main__': + unittest.main()