From 7fbdffdbbab2fba1a23f794228f65d98b2f34453 Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Sun, 1 Nov 2026 22:49:14 -0500 Subject: [PATCH] test: split the conflict-prone count pins and generator tables (#2615) - const: the four sweeps compare against tests/const/census, one module per pcapkit.const package, instead of pinned counts - vendor: the constant-miss sweep compares against a sorted crawler list - dispatch, options: each family moves to dispatch_families/ or option_families/; the census tests derive from them and the tree's maps - rfc constants: NOT_A_COPY and UNRELATED are sorted by module, and pinned so - excusals in enum_get and registry_protocol assert their reason, so the derived totals keep the pins' strength - test-coverage.rst: the suites' sources and the adding table follow - references to TCP_BASE, HIP_BASE and other moved names follow them --- .../conventions/test-coverage.rst | 59 +- examples/generators/dispatch.py | 583 +------ examples/generators/dispatch_families/frad.py | 56 + .../generators/dispatch_families/internet.py | 181 ++ examples/generators/dispatch_families/link.py | 82 + .../generators/dispatch_families/linktype.py | 124 ++ .../generators/dispatch_families/loopback.py | 44 + .../generators/dispatch_families/radiotap.py | 56 + examples/generators/dispatch_families/sctp.py | 47 + examples/generators/dispatch_families/tcp.py | 64 + examples/generators/dispatch_families/udp.py | 63 + examples/generators/option_families/dccp.py | 85 + examples/generators/option_families/hip.py | 136 ++ examples/generators/option_families/httpv2.py | 86 + examples/generators/option_families/icmp.py | 125 ++ examples/generators/option_families/ipv4.py | 85 + examples/generators/option_families/ipv6.py | 168 ++ examples/generators/option_families/mh.py | 189 +++ examples/generators/option_families/pcapng.py | 421 +++++ examples/generators/option_families/sctp.py | 165 ++ examples/generators/option_families/tcp.py | 138 ++ examples/generators/option_families/tls.py | 81 + examples/generators/options.py | 1491 +---------------- tests/const/census/__init__.py | 109 ++ tests/const/census/arp.py | 8 + tests/const/census/dccp.py | 12 + tests/const/census/esp.py | 8 + tests/const/census/ftp.py | 13 + tests/const/census/hip.py | 24 + tests/const/census/http.py | 11 + tests/const/census/icmp.py | 10 + tests/const/census/icmpv6.py | 10 + tests/const/census/ieee802_11.py | 11 + tests/const/census/igmp.py | 8 + tests/const/census/ipv4.py | 18 + tests/const/census/ipv6.py | 15 + tests/const/census/ipx.py | 8 + tests/const/census/l2tp.py | 7 + tests/const/census/linux.py | 7 + tests/const/census/mh.py | 48 + tests/const/census/ngap.py | 8 + tests/const/census/ospf.py | 8 + tests/const/census/pcapng.py | 14 + tests/const/census/reg.py | 15 + tests/const/census/sctp.py | 10 + tests/const/census/sslv2.py | 8 + tests/const/census/tcp.py | 10 + tests/const/census/tls.py | 9 + tests/const/census/vlan.py | 7 + tests/const/test_const_census_2615_unit.py | 76 + tests/const/test_const_enum_builtin_parity.py | 59 +- tests/const/test_const_enum_get.py | 30 +- tests/const/test_const_enum_lookup.py | 26 +- tests/const/test_const_registry_protocol.py | 60 +- tests/project/test_dead_helpers_2318_unit.py | 2 +- .../test_rfc_constants_one_home_1739_unit.py | 343 ++-- .../internet/test_hip_from_data_alias_unit.py | 2 +- .../test_hip_host_id_di_width_unit.py | 2 +- ...t_hip_list_argument_str_names_2394_unit.py | 2 +- .../test_hip_locator_set_length_unit.py | 2 +- .../test_hip_param_data_rebuild_2352_unit.py | 2 +- .../test_hip_r1_counter_width_unit.py | 4 +- .../protocols/link/test_l2tp_version_unit.py | 2 +- .../test_construction_keyword_check_unit.py | 4 +- .../protocols/test_dispatch_registry_unit.py | 136 +- .../test_option_generator_tcp_base_unit.py | 12 +- tests/protocols/test_option_roundtrip_unit.py | 85 +- .../test_tcp_mptcp_capable_length_unit.py | 2 +- .../test_tcp_mptcp_error_message_unit.py | 2 +- .../test_tcp_mptcp_join_flag_ordering_unit.py | 2 +- .../test_tcp_mptcp_length_arithmetic_unit.py | 2 +- .../transport/test_tcp_mptcp_subtype_unit.py | 2 +- tests/vendor/test_vendor_missing_body_unit.py | 105 +- 73 files changed, 3622 insertions(+), 2287 deletions(-) create mode 100644 examples/generators/dispatch_families/frad.py create mode 100644 examples/generators/dispatch_families/internet.py create mode 100644 examples/generators/dispatch_families/link.py create mode 100644 examples/generators/dispatch_families/linktype.py create mode 100644 examples/generators/dispatch_families/loopback.py create mode 100644 examples/generators/dispatch_families/radiotap.py create mode 100644 examples/generators/dispatch_families/sctp.py create mode 100644 examples/generators/dispatch_families/tcp.py create mode 100644 examples/generators/dispatch_families/udp.py create mode 100644 examples/generators/option_families/dccp.py create mode 100644 examples/generators/option_families/hip.py create mode 100644 examples/generators/option_families/httpv2.py create mode 100644 examples/generators/option_families/icmp.py create mode 100644 examples/generators/option_families/ipv4.py create mode 100644 examples/generators/option_families/ipv6.py create mode 100644 examples/generators/option_families/mh.py create mode 100644 examples/generators/option_families/pcapng.py create mode 100644 examples/generators/option_families/sctp.py create mode 100644 examples/generators/option_families/tcp.py create mode 100644 examples/generators/option_families/tls.py create mode 100644 tests/const/census/__init__.py create mode 100644 tests/const/census/arp.py create mode 100644 tests/const/census/dccp.py create mode 100644 tests/const/census/esp.py create mode 100644 tests/const/census/ftp.py create mode 100644 tests/const/census/hip.py create mode 100644 tests/const/census/http.py create mode 100644 tests/const/census/icmp.py create mode 100644 tests/const/census/icmpv6.py create mode 100644 tests/const/census/ieee802_11.py create mode 100644 tests/const/census/igmp.py create mode 100644 tests/const/census/ipv4.py create mode 100644 tests/const/census/ipv6.py create mode 100644 tests/const/census/ipx.py create mode 100644 tests/const/census/l2tp.py create mode 100644 tests/const/census/linux.py create mode 100644 tests/const/census/mh.py create mode 100644 tests/const/census/ngap.py create mode 100644 tests/const/census/ospf.py create mode 100644 tests/const/census/pcapng.py create mode 100644 tests/const/census/reg.py create mode 100644 tests/const/census/sctp.py create mode 100644 tests/const/census/sslv2.py create mode 100644 tests/const/census/tcp.py create mode 100644 tests/const/census/tls.py create mode 100644 tests/const/census/vlan.py create mode 100644 tests/const/test_const_census_2615_unit.py diff --git a/docs/source/contributing/conventions/test-coverage.rst b/docs/source/contributing/conventions/test-coverage.rst index 30fc39d19d..5f697dc722 100644 --- a/docs/source/contributing/conventions/test-coverage.rst +++ b/docs/source/contributing/conventions/test-coverage.rst @@ -32,24 +32,42 @@ known-failure table keep their cases in a generator, beside the captures it also - Cases from - Asserts * - :file:`tests/protocols/test_option_roundtrip_unit.py` - - :file:`examples/generators/options.py` + - :file:`examples/generators/options.py`, from its family modules under + :file:`examples/generators/option_families/`, one per protocol - every option, chunk, parameter, message, frame and block code survives construct -> parse -> construct with identical octets; known failures in ``EXPECTED_FAILURES`` * - :file:`tests/protocols/test_dispatch_registry_unit.py` - - :file:`examples/generators/dispatch.py` - - every ``__proto__`` entry dispatches to the class ``PINNED_TARGETS`` names; - known degrades in ``DispatchRegistryTests.KNOWN_DEGRADED``, which the - generator's own ``KNOWN_DEGRADED`` mirrors for its command-line report - (:issue:`496`) + - :file:`examples/generators/dispatch.py`, from its family modules under + :file:`examples/generators/dispatch_families/`, one per ``__proto__`` table + or per tables registered together + - every ``__proto__`` entry dispatches to the class its family module's + ``PINNED_TARGETS`` names; known degrades in + ``DispatchRegistryTests.KNOWN_DEGRADED``, which the generator's own + ``KNOWN_DEGRADED`` mirrors for its command-line report (:issue:`496`) + +What an unrelated change adds to these suites lands in a file of its own or on a line +of its own, so that concurrent changes do not edit the same lines: a family goes in a +module of its generator's family directory, and a :mod:`pcapkit.const` registry is a +line in its package's module under :file:`tests/const/census/`. Where a table stays one +file, as ``NOT_A_COPY`` and ``UNRELATED`` in +:file:`tests/project/test_rfc_constants_one_home_1739_unit.py` and +``CRAWLERS_WITH_CONSTANT_MISS_LINES`` in +:file:`tests/vendor/test_vendor_missing_body_unit.py` do, it is sorted. A sweep holds +what it finds to these records by name rather than to a pinned count -- split under +:issue:`2615`. The rules those tables carry: * **Every registered code has a case.** ``test_every_registered_code_has_a_case`` - fails otherwise. The option suite's ``SKIP`` table is the only exit, and each of its - entries states why a case would assert nothing: ``Multipath_TCP`` is the - envelope of the ``tcp-mptcp`` family and would duplicate one of its subtypes, and - ``PadN`` under HOPOPT and IPv6-Opts emits the same option-less header as ``Pad1``. + fails otherwise, and so does ``test_every_option_registry_has_a_family`` or + ``test_every_dispatch_table_has_a_family`` on a registry map no family exercises, + found from the tree's own maps rather than from the family modules. The option + suite's ``SKIP`` table, the merge of each family module's own ``SKIP``, is the only + exit, and each of its entries states why a case would assert nothing: + ``Multipath_TCP`` is the envelope of the ``tcp-mptcp`` family and would duplicate + one of its subtypes, and ``PadN`` under HOPOPT and IPv6-Opts emits the same + option-less header as ``Pad1``. A code that fails is never skipped. * **A failing case is an entry, never a changed argument.** An ``EXPECTED_FAILURES`` entry records the status, a failure-detail fragment narrow enough to match one @@ -184,16 +202,25 @@ above: * - Adding - Must bring * - an option-like code - - Registering it gives it a case automatically. Add arguments to the generator's - tables in :file:`examples/generators/options.py` where the ``_make_*`` defaults - are degenerate, and a ``Family`` to ``FAMILIES`` for a new registry. Its cycle - closes before it merges. + - Registering it gives it a case automatically. Add arguments where the + ``_make_*`` defaults are degenerate to the tables of its protocol's module under + :file:`examples/generators/option_families/`. A registry map no family exercises + takes a ``Family`` in that module's ``FAMILIES``, or a module of its own for a + protocol without one, which ``test_every_option_registry_has_a_family`` + enforces. Its cycle closes before it merges. * - a protocol - A unit-tier :file:`tests/protocols//test__unit.py` that builds its own octets, asserts construct -> parse -> construct identity, and checks a parse against the specification's field values. A ``__proto__`` registration also - needs its ``PINNED_TARGETS`` entry in :file:`examples/generators/dispatch.py`, - which ``test_every_case_has_a_pinned_target`` enforces. + needs its ``PINNED_TARGETS`` entry in its family's module under + :file:`examples/generators/dispatch_families/`, which + ``test_every_case_has_a_pinned_target`` enforces, and a ``__proto__`` table no + family probes takes a module of its own there, which + ``test_every_dispatch_table_has_a_family`` enforces. A :mod:`pcapkit.const` + registry it adds is a line in its package's module under + :file:`tests/const/census/`, in sorted order, or a module of its own there for a + package without one; every const sweep compares what it finds against the + census, and names a registry it lacks. * - a dumper - Unit tests under :file:`tests/dumpkit/` asserting the output exactly, and a case in :file:`tests/integration/test_output_formats.py` for the file that lands on diff --git a/examples/generators/dispatch.py b/examples/generators/dispatch.py index 56a50f9da4..6a2dd7488b 100644 --- a/examples/generators/dispatch.py +++ b/examples/generators/dispatch.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """Enumerate the ``__proto__`` dispatch registries, and probe every entry. -:file:`pcapkit/foundation/registry/protocols.py` documents the registries that +:file:`pcapkit/foundation/registry/protocols/` documents the registries that decide *which* :class:`~pcapkit.protocols.protocol.Protocol` subclass parses the next layer. No other module enumerates them: several test modules reference ``__proto__``, and @@ -20,7 +20,10 @@ :file:`examples/generators/options.py` and deliberately mirrors its shape: a :data:`FAMILIES` table drives :func:`cases`, which walks each registry rather than a hardcoded list, so a code registered tomorrow gets a case -- or fails the -harness -- tomorrow. What differs is the *check* itself. The option harness +harness -- tomorrow. Each family lives in a module of its own under +:file:`examples/generators/dispatch_families/`, beside its entries of +:data:`PINNED_TARGETS`, so that an added family is an added file (#2615). +What differs is the *check* itself. The option harness round-trips construct -> parse -> construct and compares octets; dispatch has no such symmetry to exploit, so :func:`probe` instead builds one minimal envelope per code, decodes it the way the library itself would (through @@ -40,9 +43,9 @@ test. Read from there (``target=_resolve(entry)``), the check would be self-referential: dispatch would always "reach" whatever the table said, because that is the very value being checked against. Deleting an entry would -still be caught, by the exact per-family counts in +still be caught, as a pinned target no case reaches, by :meth:`~tests.protocols.test_dispatch_registry_unit.DispatchRegistryTests -.test_cases_cover_every_table_named_in_the_issue`, but *retargeting* one -- +.test_cases_match_the_pinned_targets`, but *retargeting* one -- entry present, registry size unchanged, class wrong -- would pass silently, since nothing independent of the table would say what the table should hold. :data:`PINNED_TARGETS` is that independent record, and :attr:`Case.registered` @@ -71,6 +74,7 @@ """ from __future__ import annotations +import importlib.util import inspect import os import pathlib @@ -80,6 +84,7 @@ from typing import TYPE_CHECKING, NamedTuple if TYPE_CHECKING: + from types import ModuleType from typing import Any, Callable, Optional __all__ = ['cases', 'probe', 'outcomes', 'Case', 'Outcome', 'Family', 'FAMILIES', @@ -191,7 +196,7 @@ def _pcapng_bytes(linktype: 'Any', packet_data: 'bytes') -> 'bytes': Built through :mod:`pcapkit`'s own :class:`~pcapkit.protocols.misc.pcapng.PCAPNG` construction API rather than hand-rolled :mod:`struct` packing, the same way - :func:`examples.generators.options._pcapng_context` does -- GitHub issue + :func:`examples.generators.option_families.pcapng._pcapng_context` does -- GitHub issue #496 judged hand-building a valid SHB+IDB+EPB stream riskier than the check was worth, and this sidesteps that by reusing the library's own ``make`` path, which is exactly what is under test one layer further down @@ -291,407 +296,6 @@ def _linux_sll2(payload: 'bytes') -> 'bytes': return bytes(LinuxSLL2(payload=payload)) -def _wlan_ack() -> 'bytes': - """An 802.11 Ack frame, which carries no body.""" - from pcapkit.protocols.link.ieee802_11 import IEEE802_11 - return bytes(IEEE802_11(type='Control', subtype='ACK', addr1='02:00:00:00:00:01')) - - -def _radiotap(payload: 'bytes') -> 'bytes': - """A minimal radiotap header carrying ``payload``.""" - from pcapkit.protocols.link.radiotap import Radiotap - return bytes(Radiotap(payload=payload)) - - -############################################################################### -# Link -- EtherType. -############################################################################### - - -def _link_registry() -> 'Any': - from pcapkit.protocols.link.link import Link - return Link.__proto__ - - -def _link_payload(code: 'int') -> 'bytes': - """The registered target's own wire bytes for ``code``, as an Ethernet payload.""" - from pcapkit.const.reg.ethertype import EtherType - - if code == EtherType.Address_Resolution_Protocol: - from pcapkit.protocols.link.arp import ARP - return bytes(ARP(oper=1)) - if code == EtherType.Reverse_Address_Resolution_Protocol: - from pcapkit.protocols.application.rarp import RARP - - # ARP/RARP/InARP/DRARP all report their alias from the wire ``oper`` - # field rather than from the dispatching EtherType (``ARP.read`` in - # pcapkit/protocols/link/arp.py, per #496), so ``oper`` has to land in - # the RARP range (3, 4) or this would silently probe ARP instead while - # still "passing". - return bytes(RARP(oper=3)) - if code == EtherType.Customer_VLAN_Tag_Type: - from pcapkit.protocols.link.c_tag import C_Tag - return bytes(C_Tag(vid=200, type=0x0800, payload=_ip4(6, _tcp(9999)))) - if code == EtherType.IEEE_Std_802_1Q_Service_VLAN_tag_identifier: - from pcapkit.protocols.link.s_tag import S_Tag - return bytes(S_Tag(vid=100, type=0x0800, payload=_ip4(6, _tcp(9999)))) - if code == EtherType.Internet_Protocol_version_4: - return _ip4(6, _tcp(9999)) - if code == EtherType.Internet_Protocol_version_6: - return _ip6(6, _tcp(9999)) - if code == 0x8137: # Novell IPX. - from pcapkit.protocols.internet.ipx import IPX - return bytes(IPX()) - raise LookupError(f'link: no payload builder for {code!r}') # pragma: no cover - - -def _link_build(code: 'int') -> 'Any': - frame = _ethernet(int(code), _link_payload(code)) - return _extract_first(_pcap_bytes((frame,), linktype=1), '.pcap') - - -############################################################################### -# Internet -- TransType. -############################################################################### - -#: Codes dispatched over an IPv4 outer header. ICMP (1) is canonically -#: IPv4-only -- its IPv6 equivalent is ICMPv6 (58), dispatched separately -#: below. IGMP (2) is likewise IPv4-only -- multicast's IPv6 equivalent is -#: MLD, which rides in ICMPv6 rather than as its own IP protocol number -#: (#2439). -_INTERNET_OVER_IPV4 = frozenset({1, 2, 4, 6, 17, 50, 51, 89, 111, 132}) - - -def _internet_registry() -> 'Any': - from pcapkit.protocols.internet.internet import Internet - return Internet.__proto__ - - -def _internet_payload(code: 'int') -> 'bytes': - """The registered target's own wire bytes for ``code``.""" - from pcapkit.const.reg.transtype import TransType - - if code == TransType.HOPOPT: - from pcapkit.protocols.internet.hopopt import HOPOPT - return bytes(HOPOPT(options=[], next=6, payload=_tcp(9999))) - if code == TransType.IPv4: # IP-in-IP. - return _ip4(6, _tcp(9999)) - if code == TransType.ICMP: - from pcapkit.protocols.internet.icmpv4 import ICMPv4 - - # NOTE: the checksum is left uncomputed (#2470). - return bytes(ICMPv4(type=8, code=0, - body={'id': 1, 'seq': 1, 'data': b'ping'})) - if code == TransType.IPv6_ICMP: - from pcapkit.protocols.internet.icmpv6 import ICMPv6 - - # NOTE: the checksum is left uncomputed (#2470). - return bytes(ICMPv6(type=128, code=0, - body={'id': 1, 'seq': 1, 'data': b'ping'})) - if code == TransType.TCP: - return _tcp(9999) - if code == TransType.UDP: - return _udp(9999) - if code == TransType.DCCP: - from pcapkit.protocols.transport.dccp import DCCP - return bytes(DCCP(srcport=50000, dstport=9999, seq=1, payload=b'\x01\x02\x03')) - if code == TransType.IPv6: - return _ip6(6, _tcp(9999)) - if code == TransType.IPv6_Route: - from pcapkit.const.ipv6.routing import Routing - from pcapkit.protocols.internet.ipv6_route import IPv6_Route - return bytes(IPv6_Route(type=Routing.Type_2_Routing_Header, data={'ip': '2001:db8::2'}, - next=6, seg_left=0, payload=_tcp(9999))) - if code == TransType.IPv6_Frag: - from pcapkit.protocols.internet.ipv6_frag import IPv6_Frag - return bytes(IPv6_Frag(next=17, id=1, offset=0, mf=False, payload=_udp(9999))) - if code == TransType.ESP: - from pcapkit.protocols.internet.esp import ESP - return bytes(ESP(spi=0x1234, seq=1, payload=b'\x01\x02\x03\x04')) - if code == TransType.AH: - from pcapkit.protocols.internet.ah import AH - return bytes(AH(next=TransType.TCP, spi=0x1234, seq=1, icv=b'\x00' * 12, - payload=_tcp(9999))) - if code == TransType.IGMP: - from pcapkit.const.igmp.type import Type - from pcapkit.protocols.internet.igmpv2 import IGMPv2 - - # A General Query: an IGMPv1/v2 message is the 8-octet header and - # nothing else, so there is no further payload to wrap. A nonzero - # max_resp_time makes it IGMPv2, the class IGMP dispatches it to. - return bytes(IGMPv2(type=Type.IGMP_Membership_Query, max_resp_time=100, group='0.0.0.0')) - if code == TransType.IPv6_NoNxt: - # The registered target for this code *is* Raw -- there is no - # dissector to hand it, "no next header" is the whole point -- so any - # non-empty payload demonstrates the same thing an empty one would. - return b'\x01\x02\x03' - if code == TransType.IPv6_Opts: - from pcapkit.protocols.internet.ipv6_opts import IPv6_Opts - return bytes(IPv6_Opts(options=[], next=6, payload=_tcp(9999))) - if code == TransType.IPX_in_IP: - from pcapkit.protocols.internet.ipx import IPX - return bytes(IPX()) - if code == TransType.SCTP: - from pcapkit.const.sctp.chunk import Chunk - from pcapkit.protocols.transport.sctp import SCTP - return bytes(SCTP(srcport=50000, dstport=80, vtag=0x11223344, chksum=b'\x00' * 4, - chunks=[(Chunk.Payload_Data, {'data': b'\x01\x02\x03'})])) - if code == TransType.Mobility_Header: - from pcapkit.const.mh.packet import Packet - from pcapkit.protocols.internet.mh import MH - return bytes(MH(next=6, chksum=b'\x00\x00', type=Packet.Binding_Refresh_Request, - data={'options': []}, payload=_tcp(9999))) - if code == TransType.HIP: - from pcapkit.const.hip.parameter import Parameter - from pcapkit.protocols.internet.hip import HIP - - # Only the dispatch into HIP is under test here, so which parameter - # travels inside it does not matter. - return bytes(HIP(parameters=[(Parameter.SEQ, {})], extension=True, - next=6, packet=1, version=2, checksum=b'\x00\x00', - controls_anonymous=False, shit=0, rhit=0, payload=_tcp(9999))) - if code == TransType.OSPFIGP: - from pcapkit.protocols.application.ospf import OSPF - return bytes(OSPF()) - if code == TransType.Shim6: - # #904: no dedicated dissector exists for Shim6 -- IPv6_Ext - # parses only the two octets RFC 6564 §4 guarantees (next header, - # Hdr Ext Len), so this is hand-built rather than constructed - # through a class: next=TCP(6), Hdr Ext Len=0 -> an 8-octet header, - # six of them padding. - return bytes([int(TransType.TCP), 0]) + b'\x00' * 6 + _tcp(9999) - raise LookupError(f'internet: no payload builder for {code!r}') # pragma: no cover - - -def _internet_build(code: 'int') -> 'Any': - payload = _internet_payload(code) - if int(code) in _INTERNET_OVER_IPV4: - frame = _ethernet(0x0800, _ip4(int(code), payload)) - else: - frame = _ethernet(0x86DD, _ip6(int(code), payload)) - return _extract_first(_pcap_bytes((frame,), linktype=1), '.pcap') - - -############################################################################### -# TCP / UDP -- port. -############################################################################### - -#: A minimal, genuine TLS record: ``application_data`` (23), legacy version -#: 3.3, and a zero-length fragment -- content type and legacy version both in -#: range, and the (zero) declared length fits the (zero) remaining buffer, so -#: :meth:`SSL._validate_tls_record ` -#: accepts it rather than degrading it to :class:`Raw`. -_TLS_RECORD = b'\x17\x03\x03\x00\x00' - -#: The DTLS equivalent of :data:`_TLS_RECORD`: ``application_data``, legacy -#: version 0xFEFD, epoch 0, sequence number 1, zero-length fragment. -_DTLS_RECORD = b'\x17\xfe\xfd\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00' - -_TCP_PORT_PAYLOAD = { - 20: b'binary-file-contents', - 21: b'USER anonymous\r\n', - 80: b'GET / HTTP/1.1\r\nHost: probe.invalid\r\n\r\n', - 443: _TLS_RECORD, - 8080: b'GET / HTTP/1.1\r\nHost: probe.invalid\r\n\r\n', -} - -#: An L2TPv2 data message with every optional field absent, matching -#: :func:`tests.protocols.test_dispatch_bindings_unit.l2tp_data`. -_L2TP_DATA = struct.pack('!HHH', 0x0002, 0x1234, 0x5678) + b'\xff\x03\x00\x21PPP' - -_UDP_PORT_PAYLOAD = { - 80: b'GET / HTTP/1.1\r\nHost: probe.invalid\r\n\r\n', - 1701: _L2TP_DATA, - 5684: _DTLS_RECORD, - 8080: b'GET / HTTP/1.1\r\nHost: probe.invalid\r\n\r\n', -} - - -def _tcp_registry() -> 'Any': - from pcapkit.protocols.transport.tcp import TCP - return TCP.__proto__ - - -def _tcp_build(port: 'int') -> 'Any': - payload = _TCP_PORT_PAYLOAD.get(port) - if payload is None: - raise LookupError(f'tcp: no payload builder for port {port!r}') - frame = _ethernet(0x0800, _ip4(6, _tcp(port, payload))) - return _extract_first(_pcap_bytes((frame,), linktype=1), '.pcap') - - -def _udp_registry() -> 'Any': - from pcapkit.protocols.transport.udp import UDP - return UDP.__proto__ - - -def _udp_build(port: 'int') -> 'Any': - payload = _UDP_PORT_PAYLOAD.get(port) - if payload is None: - raise LookupError(f'udp: no payload builder for port {port!r}') - frame = _ethernet(0x0800, _ip4(17, _udp(port, payload))) - return _extract_first(_pcap_bytes((frame,), linktype=1), '.pcap') - - -############################################################################### -# SCTP -- payload protocol identifier. -############################################################################### - - -def _sctp_registry() -> 'Any': - from pcapkit.protocols.transport.sctp import SCTP - return SCTP.__proto__ - - -def _sctp_build(ppid: 'Any') -> 'Any': - from pcapkit.const.sctp.chunk import Chunk - from pcapkit.protocols.transport.sctp import SCTP - - sctp = bytes(SCTP(srcport=50000, dstport=80, vtag=0x11223344, chksum=b'\x00' * 4, - chunks=[(Chunk.Payload_Data, {'ppid': ppid, 'data': b'ngap-pdu'})])) - frame = _ethernet(0x0800, _ip4(132, sctp)) - return _extract_first(_pcap_bytes((frame,), linktype=1), '.pcap') - - -############################################################################### -# Frame (PCAP) / PCAPNG -- LinkType. -############################################################################### - - -def _pcap_frame_registry() -> 'Any': - from pcapkit.protocols.misc.pcap.frame import Frame - return Frame.__proto__ - - -def _linktype_payload(code: 'Any') -> 'tuple[bytes, bool]': - """The bytes to put in one capture record for LinkType ``code``. - - Args: - code: The ``LinkType`` under test. - - Returns: - The record's octets, and whether they still need an Ethernet wrapper - (true for :attr:`~pcapkit.const.reg.linktype.LinkType.ETHERNET`, false - for the raw-IP link types, which carry no link layer at all, and - for the loopback link types, whose header is built here). - - """ - from pcapkit.const.reg.linktype import LinkType - - if code == LinkType.ETHERNET: - return _ip4(6, _tcp(9999)), True - if code == LinkType.IPV4: - return _ip4(6, _tcp(9999)), False - if code == LinkType.IPV6: - return _ip6(6, _tcp(9999)), False - if code == LinkType.NULL: # in the capturing host's byte order (#1574) - return _loopback(2, _ip4(6, _tcp(9999))), False - if code == LinkType.LOOP: # in network byte order (#1574) - return _loopback(2, _ip4(6, _tcp(9999)), byteorder='big'), False - if code == LinkType.LINUX_SLL: - return _linux_sll(_ip4(6, _tcp(9999))), False - if code == LinkType.LINUX_SLL2: - return _linux_sll2(_ip4(6, _tcp(9999))), False - if code == LinkType.IEEE802_11: - return _wlan_ack(), False - if code == LinkType.IEEE802_11_RADIOTAP: - return _radiotap(_wlan_ack()), False - raise LookupError(f'linktype: no payload builder for {code!r}') # pragma: no cover - - -def _pcap_frame_build(code: 'Any') -> 'Any': - payload, wrap = _linktype_payload(code) - record = _ethernet(0x0800, payload) if wrap else payload - return _extract_first(_pcap_bytes((record,), linktype=int(code)), '.pcap') - - -def _pcapng_registry() -> 'Any': - from pcapkit.protocols.misc.pcapng import PCAPNG - return PCAPNG.__proto__ - - -def _pcapng_build(code: 'Any') -> 'Any': - payload, wrap = _linktype_payload(code) - record = _ethernet(0x0800, payload) if wrap else payload - return _extract_first(_pcapng_bytes(code, record), '.pcapng') - - -############################################################################### -# Loopback -- address family. -############################################################################### - - -def _loopback_registry() -> 'Any': - from pcapkit.protocols.link.loopback import Loopback - return Loopback.__proto__ - - -def _loopback_build(family: 'int') -> 'Any': - payload = _ip4(6, _tcp(9999)) if family == 2 else _ip6(6, _tcp(9999)) - return _extract_first(_pcap_bytes((_loopback(family, payload),), linktype=0), '.pcap') - - -############################################################################### -# FRAD -- NLPID. -############################################################################### - - -def _frad_registry() -> 'Any': - from pcapkit.protocols.link.frad import FRAD - return FRAD.__proto__ - - -def _frad_payload(nlpid: 'int') -> 'bytes': - """The registered target's own wire bytes for ``nlpid``.""" - if nlpid == 0xCC: - return _ip4(6, _tcp(9999)) - if nlpid == 0x8E: - return _ip6(6, _tcp(9999)) - raise LookupError(f'frad: no payload builder for NLPID {nlpid!r}') # pragma: no cover - - -def _frad_build(nlpid: 'int') -> 'Any': - """A direct-NLPID FRAD frame, reached as LinuxSLL dispatches it in a capture.""" - from pcapkit.const.linux.arphrd import ARPHRD - from pcapkit.protocols.link.frad import FRAD - from pcapkit.protocols.link.linux_sll1 import LinuxSLL1 - - frad = bytes(FRAD(nlpid=int(nlpid), payload=_frad_payload(nlpid))) - record = bytes(LinuxSLL1(arphrd_type=ARPHRD.FRAD, protocol_type=0, payload=frad)) - return _extract_first(_pcap_bytes((record,), linktype=113), '.pcap') - - -############################################################################### -# Radiotap -- link type of the frame behind the header. -############################################################################### - - -def _radiotap_registry() -> 'Any': - from pcapkit.protocols.link.radiotap import Radiotap - return Radiotap.__proto__ - - -def _radiotap_build(code: 'Any') -> 'Any': - """An 802.11 Ack behind radiotap, on a ``LINUX_SLL`` capture. - - :meth:`Radiotap.read ` - consults ``LinkType.IEEE802_11`` alone, so that is the only code with a - case. - - """ - from pcapkit.const.linux.arphrd import ARPHRD - from pcapkit.const.reg.linktype import LinkType - from pcapkit.protocols.link.ieee802_11 import IEEE802_11 - from pcapkit.protocols.link.linux_sll1 import LinuxSLL1 - from pcapkit.protocols.link.radiotap import Radiotap - - if code != LinkType.IEEE802_11: - raise ValueError(f'no radiotap case for link type {code!r}') - ack = bytes(IEEE802_11(type='Control', subtype='ACK', addr1='02:00:00:00:00:01')) - frame = bytes(LinuxSLL1(arphrd_type=ARPHRD.IEEE80211_RADIOTAP, payload=bytes(Radiotap(payload=ack)))) - return _extract_first(_pcap_bytes((frame,), linktype=int(LinkType.LINUX_SLL)), '.pcap') - - ############################################################################### # The families, the cases, and the probe. ############################################################################### @@ -713,26 +317,56 @@ class Family(NamedTuple): enum: 'Optional[Any]' = None -def _link_enum() -> 'Any': - from pcapkit.const.reg.ethertype import EtherType - return EtherType +#: The directory holding the family modules: one file per ``__proto__`` table, +#: or per tables registered together, so that a family added in one change and +#: a family added in another touch different files (#2615). +FAMILY_DIR = ROOT / 'examples' / 'generators' / 'dispatch_families' -#: Every family this module exercises, one per ``__proto__`` table named in -#: GitHub issue #496, the loopback table #1574 added, FRAD's NLPID table -#: (#2560), and the radiotap table of #2575. -FAMILIES = ( - Family('link', _link_registry, _link_build, _link_enum), - Family('internet', _internet_registry, _internet_build), - Family('tcp', _tcp_registry, _tcp_build), - Family('udp', _udp_registry, _udp_build), - Family('sctp', _sctp_registry, _sctp_build), - Family('pcap-frame', _pcap_frame_registry, _pcap_frame_build), - Family('pcapng-frame', _pcapng_registry, _pcapng_build), - Family('loopback', _loopback_registry, _loopback_build), - Family('frad', _frad_registry, _frad_build), - Family('radiotap', _radiotap_registry, _radiotap_build), -) +class _Generator: + """This module as its family modules read it: its globals, looked up live.""" + + def __getattr__(self, name: 'str') -> 'Any': + try: + return globals()[name] + except KeyError: + raise AttributeError(f'{__name__!r} has no attribute {name!r}') from None + + +def _family_modules() -> 'dict[str, ModuleType]': + """Load every module of :data:`FAMILY_DIR`, in file name order, keyed by stem. + + Each is loaded by path, as this module is, with ``generator`` bound before + its body runs, so that it builds its :class:`Family` entries and reaches + the shared plumbing above as ``generator.``. Each declares + ``FAMILIES``, the families it adds, and ``PINNED_TARGETS``, their entries + of :data:`PINNED_TARGETS`. + + Returns: + The loaded modules. + + Raises: + RuntimeError: If a module cannot be loaded. + + """ + modules = {} # type: dict[str, ModuleType] + for path in sorted(FAMILY_DIR.glob('*.py')): + spec = importlib.util.spec_from_file_location(f'{__name__}_{path.stem}', path) + if spec is None or spec.loader is None: # pragma: no cover + raise RuntimeError(f'cannot load the dispatch family module {path}') + module = importlib.util.module_from_spec(spec) + setattr(module, 'generator', _Generator()) + spec.loader.exec_module(module) + modules[path.stem] = module + return modules + + +#: Each family module, keyed by its file stem. +FAMILY_MODULES = _family_modules() + +#: Every family this module exercises: those of each family module, in file +#: name order. Each module's docstring names the issue its table came from. +FAMILIES = tuple(family for module in FAMILY_MODULES.values() for family in module.FAMILIES) #: The families keyed by label, for a caller -- :func:`probe` -- that wants #: just one. @@ -740,15 +374,16 @@ def _link_enum() -> 'Any': #: The class each case is *supposed* to dispatch to, pinned independently of -#: whatever the live registry currently holds. +#: whatever the live registry holds: the ``PINNED_TARGETS`` of each family +#: module, merged. #: #: With the expected target read straight out of the table under test #: (``target=_resolve(entry)``), a *mis-pointed* entry would pass silently -- #: dispatch would always "reach" whatever the corrupted table said, because #: that is the very value being checked against. Deleting an entry would still -#: be caught, by the count guard in +#: be caught, as a pinned target no case reaches, by #: :meth:`~tests.protocols.test_dispatch_registry_unit.DispatchRegistryTests -#: .test_cases_cover_every_table_named_in_the_issue`, but retargeting one -- +#: .test_cases_match_the_pinned_targets`, but retargeting one -- #: entry present, registry size unchanged, class wrong -- would not be caught #: by anything. Recording the expected ``(module, name)`` here, by hand, the #: same way :data:`KNOWN_DEGRADED` records degraded cases by hand, means a @@ -762,98 +397,8 @@ def _link_enum() -> 'Any': #: unpinned" guard this table gives for free, the same way #: :data:`examples.generators.options.FAMILIES` gives #: ``test_every_registered_code_has_a_case``. -PINNED_TARGETS = { - # -- Link.__proto__ (EtherType) ------------------------------------------- - 'link/Address_Resolution_Protocol': ('pcapkit.protocols.link.arp', 'ARP'), - 'link/Reverse_Address_Resolution_Protocol': ('pcapkit.protocols.application.rarp', 'RARP'), - 'link/Customer_VLAN_Tag_Type': ('pcapkit.protocols.link.c_tag', 'C_Tag'), - 'link/IEEE_Std_802_1Q_Service_VLAN_tag_identifier': ('pcapkit.protocols.link.s_tag', 'S_Tag'), - 'link/Internet_Protocol_version_4': ('pcapkit.protocols.internet.ipv4', 'IPv4'), - 'link/Internet_Protocol_version_6': ('pcapkit.protocols.internet.ipv6', 'IPv6'), - 'link/Novell_Inc_0x8137': ('pcapkit.protocols.internet.ipx', 'IPX'), - - # -- Internet.__proto__ (TransType) --------------------------------------- - 'internet/HOPOPT': ('pcapkit.protocols.internet.hopopt', 'HOPOPT'), - 'internet/IPv4': ('pcapkit.protocols.internet.ipv4', 'IPv4'), - 'internet/ICMP': ('pcapkit.protocols.internet.icmpv4', 'ICMPv4'), - 'internet/TCP': ('pcapkit.protocols.transport.tcp', 'TCP'), - 'internet/UDP': ('pcapkit.protocols.transport.udp', 'UDP'), - 'internet/DCCP': ('pcapkit.protocols.transport.dccp', 'DCCP'), - 'internet/IPv6': ('pcapkit.protocols.internet.ipv6', 'IPv6'), - 'internet/IPv6_ICMP': ('pcapkit.protocols.internet.icmpv6', 'ICMPv6'), - 'internet/IPv6_Route': ('pcapkit.protocols.internet.ipv6_route', 'IPv6_Route'), - 'internet/IPv6_Frag': ('pcapkit.protocols.internet.ipv6_frag', 'IPv6_Frag'), - 'internet/ESP': ('pcapkit.protocols.internet.esp', 'ESP'), - 'internet/AH': ('pcapkit.protocols.internet.ah', 'AH'), - 'internet/IGMP': ('pcapkit.protocols.internet.igmp', 'IGMP'), - 'internet/IPv6_NoNxt': ('pcapkit.protocols.misc.raw', 'Raw'), - 'internet/IPv6_Opts': ('pcapkit.protocols.internet.ipv6_opts', 'IPv6_Opts'), - 'internet/IPX_in_IP': ('pcapkit.protocols.internet.ipx', 'IPX'), - 'internet/Mobility_Header': ('pcapkit.protocols.internet.mh', 'MH'), - 'internet/HIP': ('pcapkit.protocols.internet.hip', 'HIP'), - 'internet/SCTP': ('pcapkit.protocols.transport.sctp', 'SCTP'), - 'internet/OSPFIGP': ('pcapkit.protocols.application.ospf', 'OSPF'), - # #904: Shim6 (140) is registered directly at IPv6_Ext, which parses - # the RFC 6564 §4 generic layout, as Shim6 has no dedicated dissector. - # Without the entry the default factory would resolve it to Raw. - 'internet/Shim6': ('pcapkit.protocols.internet.ipv6_ext', 'IPv6_Ext'), - - # -- TCP.__proto__ (port) -------------------------------------------------- - 'tcp/20': ('pcapkit.protocols.application.ftp_data', 'FTP_DATA'), - 'tcp/21': ('pcapkit.protocols.application.ftp', 'FTP'), - # The version-identifying proxy (#682), as for ``udp/80`` and ``udp/8080`` - # below; it dispatches to HTTPv1 or HTTPv2 (#2524). - 'tcp/80': ('pcapkit.protocols.application.http', 'HTTP'), - 'tcp/443': ('pcapkit.protocols.application.tls', 'TLS'), - 'tcp/8080': ('pcapkit.protocols.application.http', 'HTTP'), - - # -- UDP.__proto__ (port) -------------------------------------------------- - 'udp/80': ('pcapkit.protocols.application.http', 'HTTP'), - 'udp/1701': ('pcapkit.protocols.link.l2tpv2', 'L2TPv2'), - 'udp/5684': ('pcapkit.protocols.application.dtls', 'DTLS'), - 'udp/8080': ('pcapkit.protocols.application.http', 'HTTP'), - - # -- SCTP.__proto__ (payload protocol identifier) -------------------------- - 'sctp/PayloadProtocolIdentifier_3GPP_NG_Application_Protocol': - ('pcapkit.protocols.application.ngap', 'NGAP'), - 'sctp/PayloadProtocolIdentifier_3GPP_NGAP_over_DTLS_over_SCTP': - ('pcapkit.protocols.application.ngap', 'NGAP'), - - # -- Frame.__proto__ (pcap LinkType) --------------------------------------- - 'pcap-frame/NULL': ('pcapkit.protocols.link', 'Loopback'), - 'pcap-frame/ETHERNET': ('pcapkit.protocols.link', 'Ethernet'), - 'pcap-frame/LOOP': ('pcapkit.protocols.link', 'Loopback'), - 'pcap-frame/IPV4': ('pcapkit.protocols.internet', 'IPv4'), - 'pcap-frame/IPV6': ('pcapkit.protocols.internet', 'IPv6'), - 'pcap-frame/LINUX_SLL': ('pcapkit.protocols.link', 'LinuxSLL1'), - 'pcap-frame/LINUX_SLL2': ('pcapkit.protocols.link', 'LinuxSLL2'), - 'pcap-frame/IEEE802_11': ('pcapkit.protocols.link', 'IEEE802_11'), - 'pcap-frame/IEEE802_11_RADIOTAP': ('pcapkit.protocols.link', 'Radiotap'), - - # -- PCAPNG.__proto__ (LinkType) -------------------------------------------- - 'pcapng-frame/NULL': ('pcapkit.protocols.link', 'Loopback'), - 'pcapng-frame/ETHERNET': ('pcapkit.protocols.link', 'Ethernet'), - 'pcapng-frame/LOOP': ('pcapkit.protocols.link', 'Loopback'), - 'pcapng-frame/IPV4': ('pcapkit.protocols.internet', 'IPv4'), - 'pcapng-frame/IPV6': ('pcapkit.protocols.internet', 'IPv6'), - 'pcapng-frame/LINUX_SLL': ('pcapkit.protocols.link', 'LinuxSLL1'), - 'pcapng-frame/LINUX_SLL2': ('pcapkit.protocols.link', 'LinuxSLL2'), - 'pcapng-frame/IEEE802_11': ('pcapkit.protocols.link', 'IEEE802_11'), - 'pcapng-frame/IEEE802_11_RADIOTAP': ('pcapkit.protocols.link', 'Radiotap'), - - # -- Loopback.__proto__ (address family) ------------------------------------ - 'loopback/2': ('pcapkit.protocols.internet.ipv4', 'IPv4'), - 'loopback/24': ('pcapkit.protocols.internet.ipv6', 'IPv6'), - 'loopback/28': ('pcapkit.protocols.internet.ipv6', 'IPv6'), - 'loopback/30': ('pcapkit.protocols.internet.ipv6', 'IPv6'), - - # -- FRAD.__proto__ (NLPID, keyed in decimal: 0x8E, 0xCC) ------------------ - 'frad/142': ('pcapkit.protocols.internet.ipv6', 'IPv6'), - 'frad/204': ('pcapkit.protocols.internet.ipv4', 'IPv4'), - - # -- Radiotap.__proto__ (LinkType of the frame behind it) -------------------- - 'radiotap/IEEE802_11': ('pcapkit.protocols.link.ieee802_11', 'IEEE802_11'), -} +PINNED_TARGETS = {label: target for module in FAMILY_MODULES.values() + for label, target in module.PINNED_TARGETS.items()} def _pinned_class(label: 'str') -> 'Optional[Any]': diff --git a/examples/generators/dispatch_families/frad.py b/examples/generators/dispatch_families/frad.py new file mode 100644 index 0000000000..7858d445f7 --- /dev/null +++ b/examples/generators/dispatch_families/frad.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +"""The ``frad`` family: ``FRAD.__proto__``, keyed by NLPID (#2560). + +A family module of :file:`examples/generators/dispatch.py`, which loads it by +path and binds ``generator`` to itself before this body runs (#2615). + +""" +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any + +#: :file:`examples/generators/dispatch.py`, which its loader binds here +#: before this body runs. +generator = globals()['generator'] # type: Any + + +def _registry() -> 'Any': + from pcapkit.protocols.link.frad import FRAD + return FRAD.__proto__ + + +def _payload(nlpid: 'int') -> 'bytes': + """The registered target's own wire bytes for ``nlpid``.""" + if nlpid == 0xCC: + return generator._ip4(6, generator._tcp(9999)) + if nlpid == 0x8E: + return generator._ip6(6, generator._tcp(9999)) + raise LookupError(f'frad: no payload builder for NLPID {nlpid!r}') # pragma: no cover + + +def _build(nlpid: 'int') -> 'Any': + """A direct-NLPID FRAD frame, reached as LinuxSLL dispatches it in a capture.""" + from pcapkit.const.linux.arphrd import ARPHRD + from pcapkit.protocols.link.frad import FRAD + from pcapkit.protocols.link.linux_sll1 import LinuxSLL1 + + frad = bytes(FRAD(nlpid=int(nlpid), payload=_payload(nlpid))) + record = bytes(LinuxSLL1(arphrd_type=ARPHRD.FRAD, protocol_type=0, payload=frad)) + return generator._extract_first(generator._pcap_bytes((record,), linktype=113), '.pcap') + + +#: The families this module adds. +FAMILIES = ( + generator.Family('frad', _registry, _build), +) + +#: The class each case of :data:`FAMILIES` is pinned to, as +#: :data:`examples.generators.dispatch.PINNED_TARGETS` describes. Keyed in +#: decimal: 0x8E, 0xCC. +PINNED_TARGETS = { + 'frad/142': ('pcapkit.protocols.internet.ipv6', 'IPv6'), + 'frad/204': ('pcapkit.protocols.internet.ipv4', 'IPv4'), +} diff --git a/examples/generators/dispatch_families/internet.py b/examples/generators/dispatch_families/internet.py new file mode 100644 index 0000000000..d55d7ade8f --- /dev/null +++ b/examples/generators/dispatch_families/internet.py @@ -0,0 +1,181 @@ +# -*- coding: utf-8 -*- +"""The ``internet`` family: ``Internet.__proto__``, keyed by TransType (#496). + +A family module of :file:`examples/generators/dispatch.py`, which loads it by +path and binds ``generator`` to itself before this body runs (#2615). + +""" +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any + +#: :file:`examples/generators/dispatch.py`, which its loader binds here +#: before this body runs. +generator = globals()['generator'] # type: Any + +#: Codes dispatched over an IPv4 outer header, one per line. ICMP (1) is +#: canonically IPv4-only -- its IPv6 equivalent is ICMPv6 (58), dispatched +#: separately below. IGMP (2) is likewise IPv4-only -- multicast's IPv6 +#: equivalent is MLD, which rides in ICMPv6 rather than as its own IP protocol +#: number (#2439). +_OVER_IPV4 = frozenset({ + 1, # ICMP + 2, # IGMP + 4, # IPv4 + 6, # TCP + 17, # UDP + 50, # ESP + 51, # AH + 89, # OSPFIGP + 111, # IPX_in_IP + 132, # SCTP +}) + + +def _registry() -> 'Any': + from pcapkit.protocols.internet.internet import Internet + return Internet.__proto__ + + +def _payload(code: 'int') -> 'bytes': + """The registered target's own wire bytes for ``code``.""" + from pcapkit.const.reg.transtype import TransType + + _ip4, _ip6, _tcp, _udp = generator._ip4, generator._ip6, generator._tcp, generator._udp + if code == TransType.HOPOPT: + from pcapkit.protocols.internet.hopopt import HOPOPT + return bytes(HOPOPT(options=[], next=6, payload=_tcp(9999))) + if code == TransType.IPv4: # IP-in-IP. + return _ip4(6, _tcp(9999)) + if code == TransType.ICMP: + from pcapkit.protocols.internet.icmpv4 import ICMPv4 + + # NOTE: the checksum is left uncomputed (#2470). + return bytes(ICMPv4(type=8, code=0, + body={'id': 1, 'seq': 1, 'data': b'ping'})) + if code == TransType.IPv6_ICMP: + from pcapkit.protocols.internet.icmpv6 import ICMPv6 + + # NOTE: the checksum is left uncomputed (#2470). + return bytes(ICMPv6(type=128, code=0, + body={'id': 1, 'seq': 1, 'data': b'ping'})) + if code == TransType.TCP: + return _tcp(9999) + if code == TransType.UDP: + return _udp(9999) + if code == TransType.DCCP: + from pcapkit.protocols.transport.dccp import DCCP + return bytes(DCCP(srcport=50000, dstport=9999, seq=1, payload=b'\x01\x02\x03')) + if code == TransType.IPv6: + return _ip6(6, _tcp(9999)) + if code == TransType.IPv6_Route: + from pcapkit.const.ipv6.routing import Routing + from pcapkit.protocols.internet.ipv6_route import IPv6_Route + return bytes(IPv6_Route(type=Routing.Type_2_Routing_Header, data={'ip': '2001:db8::2'}, + next=6, seg_left=0, payload=_tcp(9999))) + if code == TransType.IPv6_Frag: + from pcapkit.protocols.internet.ipv6_frag import IPv6_Frag + return bytes(IPv6_Frag(next=17, id=1, offset=0, mf=False, payload=_udp(9999))) + if code == TransType.ESP: + from pcapkit.protocols.internet.esp import ESP + return bytes(ESP(spi=0x1234, seq=1, payload=b'\x01\x02\x03\x04')) + if code == TransType.AH: + from pcapkit.protocols.internet.ah import AH + return bytes(AH(next=TransType.TCP, spi=0x1234, seq=1, icv=b'\x00' * 12, + payload=_tcp(9999))) + if code == TransType.IGMP: + from pcapkit.const.igmp.type import Type + from pcapkit.protocols.internet.igmpv2 import IGMPv2 + + # A General Query: an IGMPv1/v2 message is the 8-octet header and + # nothing else, so there is no further payload to wrap. A nonzero + # max_resp_time makes it IGMPv2, the class IGMP dispatches it to. + return bytes(IGMPv2(type=Type.IGMP_Membership_Query, max_resp_time=100, group='0.0.0.0')) + if code == TransType.IPv6_NoNxt: + # The registered target for this code *is* Raw -- there is no + # dissector to hand it, "no next header" is the whole point -- so any + # non-empty payload demonstrates the same thing an empty one would. + return b'\x01\x02\x03' + if code == TransType.IPv6_Opts: + from pcapkit.protocols.internet.ipv6_opts import IPv6_Opts + return bytes(IPv6_Opts(options=[], next=6, payload=_tcp(9999))) + if code == TransType.IPX_in_IP: + from pcapkit.protocols.internet.ipx import IPX + return bytes(IPX()) + if code == TransType.SCTP: + from pcapkit.const.sctp.chunk import Chunk + from pcapkit.protocols.transport.sctp import SCTP + return bytes(SCTP(srcport=50000, dstport=80, vtag=0x11223344, chksum=b'\x00' * 4, + chunks=[(Chunk.Payload_Data, {'data': b'\x01\x02\x03'})])) + if code == TransType.Mobility_Header: + from pcapkit.const.mh.packet import Packet + from pcapkit.protocols.internet.mh import MH + return bytes(MH(next=6, chksum=b'\x00\x00', type=Packet.Binding_Refresh_Request, + data={'options': []}, payload=_tcp(9999))) + if code == TransType.HIP: + from pcapkit.const.hip.parameter import Parameter + from pcapkit.protocols.internet.hip import HIP + + # Only the dispatch into HIP is under test here, so which parameter + # travels inside it does not matter. + return bytes(HIP(parameters=[(Parameter.SEQ, {})], extension=True, + next=6, packet=1, version=2, checksum=b'\x00\x00', + controls_anonymous=False, shit=0, rhit=0, payload=_tcp(9999))) + if code == TransType.OSPFIGP: + from pcapkit.protocols.application.ospf import OSPF + return bytes(OSPF()) + if code == TransType.Shim6: + # #904: no dedicated dissector exists for Shim6 -- IPv6_Ext + # parses only the two octets RFC 6564 §4 guarantees (next header, + # Hdr Ext Len), so this is hand-built rather than constructed + # through a class: next=TCP(6), Hdr Ext Len=0 -> an 8-octet header, + # six of them padding. + return bytes([int(TransType.TCP), 0]) + b'\x00' * 6 + _tcp(9999) + raise LookupError(f'internet: no payload builder for {code!r}') # pragma: no cover + + +def _build(code: 'int') -> 'Any': + payload = _payload(code) + if int(code) in _OVER_IPV4: + frame = generator._ethernet(0x0800, generator._ip4(int(code), payload)) + else: + frame = generator._ethernet(0x86DD, generator._ip6(int(code), payload)) + return generator._extract_first(generator._pcap_bytes((frame,), linktype=1), '.pcap') + + +#: The families this module adds. +FAMILIES = ( + generator.Family('internet', _registry, _build), +) + +#: The class each case of :data:`FAMILIES` is pinned to, as +#: :data:`examples.generators.dispatch.PINNED_TARGETS` describes. +PINNED_TARGETS = { + 'internet/HOPOPT': ('pcapkit.protocols.internet.hopopt', 'HOPOPT'), + 'internet/IPv4': ('pcapkit.protocols.internet.ipv4', 'IPv4'), + 'internet/ICMP': ('pcapkit.protocols.internet.icmpv4', 'ICMPv4'), + 'internet/TCP': ('pcapkit.protocols.transport.tcp', 'TCP'), + 'internet/UDP': ('pcapkit.protocols.transport.udp', 'UDP'), + 'internet/DCCP': ('pcapkit.protocols.transport.dccp', 'DCCP'), + 'internet/IPv6': ('pcapkit.protocols.internet.ipv6', 'IPv6'), + 'internet/IPv6_ICMP': ('pcapkit.protocols.internet.icmpv6', 'ICMPv6'), + 'internet/IPv6_Route': ('pcapkit.protocols.internet.ipv6_route', 'IPv6_Route'), + 'internet/IPv6_Frag': ('pcapkit.protocols.internet.ipv6_frag', 'IPv6_Frag'), + 'internet/ESP': ('pcapkit.protocols.internet.esp', 'ESP'), + 'internet/AH': ('pcapkit.protocols.internet.ah', 'AH'), + 'internet/IGMP': ('pcapkit.protocols.internet.igmp', 'IGMP'), + 'internet/IPv6_NoNxt': ('pcapkit.protocols.misc.raw', 'Raw'), + 'internet/IPv6_Opts': ('pcapkit.protocols.internet.ipv6_opts', 'IPv6_Opts'), + 'internet/IPX_in_IP': ('pcapkit.protocols.internet.ipx', 'IPX'), + 'internet/Mobility_Header': ('pcapkit.protocols.internet.mh', 'MH'), + 'internet/HIP': ('pcapkit.protocols.internet.hip', 'HIP'), + 'internet/SCTP': ('pcapkit.protocols.transport.sctp', 'SCTP'), + 'internet/OSPFIGP': ('pcapkit.protocols.application.ospf', 'OSPF'), + # #904: Shim6 (140) is registered directly at IPv6_Ext, which parses + # the RFC 6564 §4 generic layout, as Shim6 has no dedicated dissector. + # Without the entry the default factory would resolve it to Raw. + 'internet/Shim6': ('pcapkit.protocols.internet.ipv6_ext', 'IPv6_Ext'), +} diff --git a/examples/generators/dispatch_families/link.py b/examples/generators/dispatch_families/link.py new file mode 100644 index 0000000000..f66c651d33 --- /dev/null +++ b/examples/generators/dispatch_families/link.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- +"""The ``link`` family: ``Link.__proto__``, keyed by EtherType (#496). + +A family module of :file:`examples/generators/dispatch.py`, which loads it by +path and binds ``generator`` to itself before this body runs (#2615). + +""" +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any + +#: :file:`examples/generators/dispatch.py`, which its loader binds here +#: before this body runs. +generator = globals()['generator'] # type: Any + + +def _registry() -> 'Any': + from pcapkit.protocols.link.link import Link + return Link.__proto__ + + +def _payload(code: 'int') -> 'bytes': + """The registered target's own wire bytes for ``code``, as an Ethernet payload.""" + from pcapkit.const.reg.ethertype import EtherType + + if code == EtherType.Address_Resolution_Protocol: + from pcapkit.protocols.link.arp import ARP + return bytes(ARP(oper=1)) + if code == EtherType.Reverse_Address_Resolution_Protocol: + from pcapkit.protocols.application.rarp import RARP + + # ARP/RARP/InARP/DRARP all report their alias from the wire ``oper`` + # field rather than from the dispatching EtherType (``ARP.read`` in + # pcapkit/protocols/link/arp.py, per #496), so ``oper`` has to land in + # the RARP range (3, 4) or this would silently probe ARP instead while + # still "passing". + return bytes(RARP(oper=3)) + if code == EtherType.Customer_VLAN_Tag_Type: + from pcapkit.protocols.link.c_tag import C_Tag + return bytes(C_Tag(vid=200, type=0x0800, payload=generator._ip4(6, generator._tcp(9999)))) + if code == EtherType.IEEE_Std_802_1Q_Service_VLAN_tag_identifier: + from pcapkit.protocols.link.s_tag import S_Tag + return bytes(S_Tag(vid=100, type=0x0800, payload=generator._ip4(6, generator._tcp(9999)))) + if code == EtherType.Internet_Protocol_version_4: + return generator._ip4(6, generator._tcp(9999)) + if code == EtherType.Internet_Protocol_version_6: + return generator._ip6(6, generator._tcp(9999)) + if code == 0x8137: # Novell IPX. + from pcapkit.protocols.internet.ipx import IPX + return bytes(IPX()) + raise LookupError(f'link: no payload builder for {code!r}') # pragma: no cover + + +def _build(code: 'int') -> 'Any': + frame = generator._ethernet(int(code), _payload(code)) + return generator._extract_first(generator._pcap_bytes((frame,), linktype=1), '.pcap') + + +def _enum() -> 'Any': + from pcapkit.const.reg.ethertype import EtherType + return EtherType + + +#: The families this module adds. +FAMILIES = ( + generator.Family('link', _registry, _build, _enum), +) + +#: The class each case of :data:`FAMILIES` is pinned to, as +#: :data:`examples.generators.dispatch.PINNED_TARGETS` describes. +PINNED_TARGETS = { + 'link/Address_Resolution_Protocol': ('pcapkit.protocols.link.arp', 'ARP'), + 'link/Reverse_Address_Resolution_Protocol': ('pcapkit.protocols.application.rarp', 'RARP'), + 'link/Customer_VLAN_Tag_Type': ('pcapkit.protocols.link.c_tag', 'C_Tag'), + 'link/IEEE_Std_802_1Q_Service_VLAN_tag_identifier': ('pcapkit.protocols.link.s_tag', 'S_Tag'), + 'link/Internet_Protocol_version_4': ('pcapkit.protocols.internet.ipv4', 'IPv4'), + 'link/Internet_Protocol_version_6': ('pcapkit.protocols.internet.ipv6', 'IPv6'), + 'link/Novell_Inc_0x8137': ('pcapkit.protocols.internet.ipx', 'IPX'), +} diff --git a/examples/generators/dispatch_families/linktype.py b/examples/generators/dispatch_families/linktype.py new file mode 100644 index 0000000000..d8e967a35e --- /dev/null +++ b/examples/generators/dispatch_families/linktype.py @@ -0,0 +1,124 @@ +# -*- coding: utf-8 -*- +"""The ``pcap-frame`` and ``pcapng-frame`` families: the LinkType tables (#496, #1574). + +``Frame.__proto__`` and ``PCAPNG.__proto__`` are both keyed by LinkType, and a +link type is registered in both, so the two families share this module and its +payload builder. A family module of :file:`examples/generators/dispatch.py`, +which loads it by path and binds ``generator`` to itself before this body runs +(#2615). + +""" +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any + +#: :file:`examples/generators/dispatch.py`, which its loader binds here +#: before this body runs. +generator = globals()['generator'] # type: Any + + +def _pcap_frame_registry() -> 'Any': + from pcapkit.protocols.misc.pcap.frame import Frame + return Frame.__proto__ + + +def _payload(code: 'Any') -> 'tuple[bytes, bool]': + """The bytes to put in one capture record for LinkType ``code``. + + Args: + code: The ``LinkType`` under test. + + Returns: + The record's octets, and whether they still need an Ethernet wrapper + (true for :attr:`~pcapkit.const.reg.linktype.LinkType.ETHERNET`, false + for the raw-IP link types, which carry no link layer at all, and + for the loopback link types, whose header is built here). + + """ + from pcapkit.const.reg.linktype import LinkType + + _ip4, _ip6, _tcp = generator._ip4, generator._ip6, generator._tcp + if code == LinkType.ETHERNET: + return _ip4(6, _tcp(9999)), True + if code == LinkType.IPV4: + return _ip4(6, _tcp(9999)), False + if code == LinkType.IPV6: + return _ip6(6, _tcp(9999)), False + if code == LinkType.NULL: # in the capturing host's byte order (#1574) + return generator._loopback(2, _ip4(6, _tcp(9999))), False + if code == LinkType.LOOP: # in network byte order (#1574) + return generator._loopback(2, _ip4(6, _tcp(9999)), byteorder='big'), False + if code == LinkType.LINUX_SLL: + return generator._linux_sll(_ip4(6, _tcp(9999))), False + if code == LinkType.LINUX_SLL2: + return generator._linux_sll2(_ip4(6, _tcp(9999))), False + if code == LinkType.IEEE802_11: + return _wlan_ack(), False + if code == LinkType.IEEE802_11_RADIOTAP: + return _radiotap(_wlan_ack()), False + raise LookupError(f'linktype: no payload builder for {code!r}') # pragma: no cover + + +def _wlan_ack() -> 'bytes': + """An 802.11 Ack frame, which carries no body.""" + from pcapkit.protocols.link.ieee802_11 import IEEE802_11 + return bytes(IEEE802_11(type='Control', subtype='ACK', addr1='02:00:00:00:00:01')) + + +def _radiotap(payload: 'bytes') -> 'bytes': + """A minimal radiotap header carrying ``payload``.""" + from pcapkit.protocols.link.radiotap import Radiotap + return bytes(Radiotap(payload=payload)) + + +def _pcap_frame_build(code: 'Any') -> 'Any': + payload, wrap = _payload(code) + record = generator._ethernet(0x0800, payload) if wrap else payload + return generator._extract_first(generator._pcap_bytes((record,), linktype=int(code)), '.pcap') + + +def _pcapng_registry() -> 'Any': + from pcapkit.protocols.misc.pcapng import PCAPNG + return PCAPNG.__proto__ + + +def _pcapng_build(code: 'Any') -> 'Any': + payload, wrap = _payload(code) + record = generator._ethernet(0x0800, payload) if wrap else payload + return generator._extract_first(generator._pcapng_bytes(code, record), '.pcapng') + + +#: The families this module adds. +FAMILIES = ( + generator.Family('pcap-frame', _pcap_frame_registry, _pcap_frame_build), + generator.Family('pcapng-frame', _pcapng_registry, _pcapng_build), +) + +#: The class each case of :data:`FAMILIES` is pinned to, as +#: :data:`examples.generators.dispatch.PINNED_TARGETS` describes. +PINNED_TARGETS = { + # -- Frame.__proto__ (pcap LinkType) --------------------------------------- + 'pcap-frame/NULL': ('pcapkit.protocols.link', 'Loopback'), + 'pcap-frame/ETHERNET': ('pcapkit.protocols.link', 'Ethernet'), + 'pcap-frame/LOOP': ('pcapkit.protocols.link', 'Loopback'), + 'pcap-frame/IPV4': ('pcapkit.protocols.internet', 'IPv4'), + 'pcap-frame/IPV6': ('pcapkit.protocols.internet', 'IPv6'), + 'pcap-frame/LINUX_SLL': ('pcapkit.protocols.link', 'LinuxSLL1'), + 'pcap-frame/LINUX_SLL2': ('pcapkit.protocols.link', 'LinuxSLL2'), + 'pcap-frame/IEEE802_11': ('pcapkit.protocols.link', 'IEEE802_11'), + 'pcap-frame/IEEE802_11_RADIOTAP': ('pcapkit.protocols.link', 'Radiotap'), + + # -- PCAPNG.__proto__ (LinkType) -------------------------------------------- + 'pcapng-frame/NULL': ('pcapkit.protocols.link', 'Loopback'), + 'pcapng-frame/ETHERNET': ('pcapkit.protocols.link', 'Ethernet'), + 'pcapng-frame/LOOP': ('pcapkit.protocols.link', 'Loopback'), + 'pcapng-frame/IPV4': ('pcapkit.protocols.internet', 'IPv4'), + 'pcapng-frame/IPV6': ('pcapkit.protocols.internet', 'IPv6'), + 'pcapng-frame/LINUX_SLL': ('pcapkit.protocols.link', 'LinuxSLL1'), + 'pcapng-frame/LINUX_SLL2': ('pcapkit.protocols.link', 'LinuxSLL2'), + 'pcapng-frame/IEEE802_11': ('pcapkit.protocols.link', 'IEEE802_11'), + 'pcapng-frame/IEEE802_11_RADIOTAP': ('pcapkit.protocols.link', 'Radiotap'), +} diff --git a/examples/generators/dispatch_families/loopback.py b/examples/generators/dispatch_families/loopback.py new file mode 100644 index 0000000000..2524d65134 --- /dev/null +++ b/examples/generators/dispatch_families/loopback.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +"""The ``loopback`` family: ``Loopback.__proto__``, keyed by address family (#1574). + +A family module of :file:`examples/generators/dispatch.py`, which loads it by +path and binds ``generator`` to itself before this body runs (#2615). + +""" +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any + +#: :file:`examples/generators/dispatch.py`, which its loader binds here +#: before this body runs. +generator = globals()['generator'] # type: Any + + +def _registry() -> 'Any': + from pcapkit.protocols.link.loopback import Loopback + return Loopback.__proto__ + + +def _build(family: 'int') -> 'Any': + _ip4, _ip6, _tcp = generator._ip4, generator._ip6, generator._tcp + payload = _ip4(6, _tcp(9999)) if family == 2 else _ip6(6, _tcp(9999)) + frame = generator._loopback(family, payload) + return generator._extract_first(generator._pcap_bytes((frame,), linktype=0), '.pcap') + + +#: The families this module adds. +FAMILIES = ( + generator.Family('loopback', _registry, _build), +) + +#: The class each case of :data:`FAMILIES` is pinned to, as +#: :data:`examples.generators.dispatch.PINNED_TARGETS` describes. +PINNED_TARGETS = { + 'loopback/2': ('pcapkit.protocols.internet.ipv4', 'IPv4'), + 'loopback/24': ('pcapkit.protocols.internet.ipv6', 'IPv6'), + 'loopback/28': ('pcapkit.protocols.internet.ipv6', 'IPv6'), + 'loopback/30': ('pcapkit.protocols.internet.ipv6', 'IPv6'), +} diff --git a/examples/generators/dispatch_families/radiotap.py b/examples/generators/dispatch_families/radiotap.py new file mode 100644 index 0000000000..2cdfbdc752 --- /dev/null +++ b/examples/generators/dispatch_families/radiotap.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +"""The ``radiotap`` family: ``Radiotap.__proto__``, keyed by the link type behind it (#2575). + +A family module of :file:`examples/generators/dispatch.py`, which loads it by +path and binds ``generator`` to itself before this body runs (#2615). + +""" +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any + +#: :file:`examples/generators/dispatch.py`, which its loader binds here +#: before this body runs. +generator = globals()['generator'] # type: Any + + +def _registry() -> 'Any': + from pcapkit.protocols.link.radiotap import Radiotap + return Radiotap.__proto__ + + +def _build(code: 'Any') -> 'Any': + """An 802.11 Ack behind radiotap, on a ``LINUX_SLL`` capture. + + :meth:`Radiotap.read ` + consults ``LinkType.IEEE802_11`` alone, so that is the only code with a + case. + + """ + from pcapkit.const.linux.arphrd import ARPHRD + from pcapkit.const.reg.linktype import LinkType + from pcapkit.protocols.link.ieee802_11 import IEEE802_11 + from pcapkit.protocols.link.linux_sll1 import LinuxSLL1 + from pcapkit.protocols.link.radiotap import Radiotap + + if code != LinkType.IEEE802_11: + raise ValueError(f'no radiotap case for link type {code!r}') + ack = bytes(IEEE802_11(type='Control', subtype='ACK', addr1='02:00:00:00:00:01')) + frame = bytes(LinuxSLL1(arphrd_type=ARPHRD.IEEE80211_RADIOTAP, payload=bytes(Radiotap(payload=ack)))) + return generator._extract_first(generator._pcap_bytes((frame,), linktype=int(LinkType.LINUX_SLL)), + '.pcap') + + +#: The families this module adds. +FAMILIES = ( + generator.Family('radiotap', _registry, _build), +) + +#: The class each case of :data:`FAMILIES` is pinned to, as +#: :data:`examples.generators.dispatch.PINNED_TARGETS` describes. +PINNED_TARGETS = { + 'radiotap/IEEE802_11': ('pcapkit.protocols.link.ieee802_11', 'IEEE802_11'), +} diff --git a/examples/generators/dispatch_families/sctp.py b/examples/generators/dispatch_families/sctp.py new file mode 100644 index 0000000000..89fa900cc1 --- /dev/null +++ b/examples/generators/dispatch_families/sctp.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- +"""The ``sctp`` family: ``SCTP.__proto__``, keyed by payload protocol identifier (#496). + +A family module of :file:`examples/generators/dispatch.py`, which loads it by +path and binds ``generator`` to itself before this body runs (#2615). + +""" +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any + +#: :file:`examples/generators/dispatch.py`, which its loader binds here +#: before this body runs. +generator = globals()['generator'] # type: Any + + +def _registry() -> 'Any': + from pcapkit.protocols.transport.sctp import SCTP + return SCTP.__proto__ + + +def _build(ppid: 'Any') -> 'Any': + from pcapkit.const.sctp.chunk import Chunk + from pcapkit.protocols.transport.sctp import SCTP + + sctp = bytes(SCTP(srcport=50000, dstport=80, vtag=0x11223344, chksum=b'\x00' * 4, + chunks=[(Chunk.Payload_Data, {'ppid': ppid, 'data': b'ngap-pdu'})])) + frame = generator._ethernet(0x0800, generator._ip4(132, sctp)) + return generator._extract_first(generator._pcap_bytes((frame,), linktype=1), '.pcap') + + +#: The families this module adds. +FAMILIES = ( + generator.Family('sctp', _registry, _build), +) + +#: The class each case of :data:`FAMILIES` is pinned to, as +#: :data:`examples.generators.dispatch.PINNED_TARGETS` describes. +PINNED_TARGETS = { + 'sctp/PayloadProtocolIdentifier_3GPP_NG_Application_Protocol': + ('pcapkit.protocols.application.ngap', 'NGAP'), + 'sctp/PayloadProtocolIdentifier_3GPP_NGAP_over_DTLS_over_SCTP': + ('pcapkit.protocols.application.ngap', 'NGAP'), +} diff --git a/examples/generators/dispatch_families/tcp.py b/examples/generators/dispatch_families/tcp.py new file mode 100644 index 0000000000..c12f24f88a --- /dev/null +++ b/examples/generators/dispatch_families/tcp.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -*- +"""The ``tcp`` family: ``TCP.__proto__``, keyed by port (#496). + +A family module of :file:`examples/generators/dispatch.py`, which loads it by +path and binds ``generator`` to itself before this body runs (#2615). + +""" +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any + +#: :file:`examples/generators/dispatch.py`, which its loader binds here +#: before this body runs. +generator = globals()['generator'] # type: Any + +#: A minimal, genuine TLS record: ``application_data`` (23), legacy version +#: 3.3, and a zero-length fragment -- content type and legacy version both in +#: range, and the (zero) declared length fits the (zero) remaining buffer, so +#: :meth:`SSL._validate_tls_record ` +#: accepts it rather than degrading it to :class:`Raw`. +_TLS_RECORD = b'\x17\x03\x03\x00\x00' + +#: The payload each registered port is probed with. +_PORT_PAYLOAD = { + 20: b'binary-file-contents', + 21: b'USER anonymous\r\n', + 80: b'GET / HTTP/1.1\r\nHost: probe.invalid\r\n\r\n', + 443: _TLS_RECORD, + 8080: b'GET / HTTP/1.1\r\nHost: probe.invalid\r\n\r\n', +} + + +def _registry() -> 'Any': + from pcapkit.protocols.transport.tcp import TCP + return TCP.__proto__ + + +def _build(port: 'int') -> 'Any': + payload = _PORT_PAYLOAD.get(port) + if payload is None: + raise LookupError(f'tcp: no payload builder for port {port!r}') + frame = generator._ethernet(0x0800, generator._ip4(6, generator._tcp(port, payload))) + return generator._extract_first(generator._pcap_bytes((frame,), linktype=1), '.pcap') + + +#: The families this module adds. +FAMILIES = ( + generator.Family('tcp', _registry, _build), +) + +#: The class each case of :data:`FAMILIES` is pinned to, as +#: :data:`examples.generators.dispatch.PINNED_TARGETS` describes. +PINNED_TARGETS = { + 'tcp/20': ('pcapkit.protocols.application.ftp_data', 'FTP_DATA'), + 'tcp/21': ('pcapkit.protocols.application.ftp', 'FTP'), + # The version-identifying proxy (#682), as for ``udp/80`` and ``udp/8080``; + # it dispatches to HTTPv1 or HTTPv2 (#2524). + 'tcp/80': ('pcapkit.protocols.application.http', 'HTTP'), + 'tcp/443': ('pcapkit.protocols.application.tls', 'TLS'), + 'tcp/8080': ('pcapkit.protocols.application.http', 'HTTP'), +} diff --git a/examples/generators/dispatch_families/udp.py b/examples/generators/dispatch_families/udp.py new file mode 100644 index 0000000000..c27c441b1c --- /dev/null +++ b/examples/generators/dispatch_families/udp.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +"""The ``udp`` family: ``UDP.__proto__``, keyed by port (#496). + +A family module of :file:`examples/generators/dispatch.py`, which loads it by +path and binds ``generator`` to itself before this body runs (#2615). + +""" +from __future__ import annotations + +import struct +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any + +#: :file:`examples/generators/dispatch.py`, which its loader binds here +#: before this body runs. +generator = globals()['generator'] # type: Any + +#: The DTLS equivalent of the TLS record the ``tcp`` family probes port 443 +#: with: ``application_data``, legacy version 0xFEFD, epoch 0, sequence number +#: 1, zero-length fragment. +_DTLS_RECORD = b'\x17\xfe\xfd\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00' + +#: An L2TPv2 data message with every optional field absent, matching +#: :func:`tests.protocols.test_dispatch_bindings_unit.l2tp_data`. +_L2TP_DATA = struct.pack('!HHH', 0x0002, 0x1234, 0x5678) + b'\xff\x03\x00\x21PPP' + +#: The payload each registered port is probed with. +_PORT_PAYLOAD = { + 80: b'GET / HTTP/1.1\r\nHost: probe.invalid\r\n\r\n', + 1701: _L2TP_DATA, + 5684: _DTLS_RECORD, + 8080: b'GET / HTTP/1.1\r\nHost: probe.invalid\r\n\r\n', +} + + +def _registry() -> 'Any': + from pcapkit.protocols.transport.udp import UDP + return UDP.__proto__ + + +def _build(port: 'int') -> 'Any': + payload = _PORT_PAYLOAD.get(port) + if payload is None: + raise LookupError(f'udp: no payload builder for port {port!r}') + frame = generator._ethernet(0x0800, generator._ip4(17, generator._udp(port, payload))) + return generator._extract_first(generator._pcap_bytes((frame,), linktype=1), '.pcap') + + +#: The families this module adds. +FAMILIES = ( + generator.Family('udp', _registry, _build), +) + +#: The class each case of :data:`FAMILIES` is pinned to, as +#: :data:`examples.generators.dispatch.PINNED_TARGETS` describes. +PINNED_TARGETS = { + 'udp/80': ('pcapkit.protocols.application.http', 'HTTP'), + 'udp/1701': ('pcapkit.protocols.link.l2tpv2', 'L2TPv2'), + 'udp/5684': ('pcapkit.protocols.application.dtls', 'DTLS'), + 'udp/8080': ('pcapkit.protocols.application.http', 'HTTP'), +} diff --git a/examples/generators/option_families/dccp.py b/examples/generators/option_families/dccp.py new file mode 100644 index 0000000000..fc05d1e038 --- /dev/null +++ b/examples/generators/option_families/dccp.py @@ -0,0 +1,85 @@ +# -*- coding: utf-8 -*- +"""The ``dccp-option`` family: DCCP options. + +A family module of :file:`examples/generators/options.py`, which loads it by +path and binds ``generator`` to itself before this body runs (#2615). + +""" +# Deferring these is the design, as in options.py -- see its module docstring. +# pylint: disable=import-outside-toplevel + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any + +#: :file:`examples/generators/options.py`, which its loader binds here +#: before this body runs. +generator = globals()['generator'] # type: Any + +#: Header fields shared by every constructed DCCP packet: a DCCP-Ack (type 3), +#: whose Acknowledgement Number the Ack Vector and Data Dropped options read +#: against (RFC 4340 sections 11.4 and 11.7). ``checksum`` is pinned, so a +#: mismatch points at the option under test and not at a recomputed checksum. +DCCP_BASE = { + 'srcport': 50000, 'dstport': 5001, 'type': 3, 'x': True, 'seq': 1, 'ack': 1, + 'checksum': b'\x00\x00', 'payload': b'', +} + + +def _dccp_registry() -> 'Any': + from pcapkit.protocols.transport.dccp import DCCP + return DCCP.__option__ + + +def _dccp_overrides() -> 'dict[Any, dict[str, Any]]': + from pcapkit.const.dccp.feature import Feature as Enum_Feature + from pcapkit.const.dccp.option import Option as Enum_Option + + # Each default is an empty or zero field; these give every variable part + # at least one octet, and the Elapsed Time its four-octet form. + vector = [(0, 3), (1, 0), (3, 63)] + return { + Enum_Option.Change_L: {'feature': Enum_Feature.Congestion_Control_ID, 'value': b'\x02\x03'}, + Enum_Option.Confirm_L: {'feature': Enum_Feature.Congestion_Control_ID, 'value': b'\x02\x02\x03'}, + Enum_Option.Change_R: {'feature': Enum_Feature.Send_Ack_Vector, 'value': b'\x01'}, + Enum_Option.Confirm_R: {'feature': Enum_Feature.Send_Ack_Vector, 'value': b'\x01'}, + Enum_Option.Init_Cookie: {'cookie': b'\x01\x02\x03'}, + Enum_Option.NDP_Count: {'count': 0x0102}, + Enum_Option.Ack_Vector_Nonce_0: {'vector': vector}, + Enum_Option.Ack_Vector_Nonce_1: {'vector': vector}, + Enum_Option.Data_Dropped: {'vector': [(None, 5), (3, 2)]}, + Enum_Option.Timestamp: {'timestamp': 0x11223344}, + Enum_Option.Timestamp_Echo: {'timestamp': 0x11223344, 'elapsed': 0x1234}, + Enum_Option.Elapsed_Time: {'elapsed': 0x123456}, + Enum_Option.Data_Checksum: {'checksum': b'\x01\x02\x03\x04'}, + } + + +def _dccp_build(code: 'Any', kwargs: 'dict[str, Any]') -> 'Any': + from pcapkit.protocols.transport.dccp import DCCP + return DCCP(options=[(code, kwargs)], **DCCP_BASE) + + +def _dccp_parse(octets: 'bytes') -> 'Any': + from pcapkit.protocols.transport.dccp import DCCP + return DCCP(octets, len(octets)) + + +def _dccp_extract(parsed: 'Any') -> 'Any': + return parsed.info.options + + +def _dccp_rebuild(options: 'Any') -> 'Any': + from pcapkit.protocols.transport.dccp import DCCP + return DCCP(options=options, **DCCP_BASE) + + +#: The families this module adds. +FAMILIES = ( + generator.Family('dccp-option', _dccp_registry, _dccp_overrides, + _dccp_build, _dccp_parse, _dccp_extract, _dccp_rebuild, + capture='options-transport.pcap', envelope='ipv4', proto=33), +) diff --git a/examples/generators/option_families/hip.py b/examples/generators/option_families/hip.py new file mode 100644 index 0000000000..67a47293fa --- /dev/null +++ b/examples/generators/option_families/hip.py @@ -0,0 +1,136 @@ +# -*- coding: utf-8 -*- +"""The ``hip-parameter`` family: HIP parameters. + +A family module of :file:`examples/generators/options.py`, which loads it by +path and binds ``generator`` to itself before this body runs (#2615). + +""" +# Deferring these is the design, as in options.py -- see its module docstring. +# pylint: disable=import-outside-toplevel + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any + +#: :file:`examples/generators/options.py`, which its loader binds here +#: before this body runs. +generator = globals()['generator'] # type: Any + +#: Header fields shared by every constructed HIP packet. +HIP_BASE = { + 'next': 6, 'packet': 1, 'version': 2, 'checksum': b'\x00\x00', + 'controls_anonymous': False, 'shit': 0, 'rhit': 0, 'payload': b'', +} + +#: Codes tied to one HIP version, and the version their constructor demands: +#: ``R1_Counter`` (128) and ``HIP_TRANSFORM`` (577) are :rfc:`5201` only. +HIP_VERSION = {129: 2, 128: 1, 577: 1} + +#: How many copies of the parameter under test go in one packet. +#: +#: One (#689), because a pair hides the length defects a single parameter +#: exposes. ``HIP.make`` computes the header's ``len`` field as +#: ``total_length // 8 + 4``, which is lossless only when the parameter octets +#: are a multiple of eight, and each parameter is padded to :rfc:`7401` +#: Section 5.2.1's ``Total Length = 11 + Length - (Length + 3) % 8`` so that it +#: is one on its own (#651). A record that is ``4 (mod 8)`` -- from padding that +#: aligns only the contents and ignores the four-octet type-and-length header +#: (#651), or from a packed length that disagrees with the declared ``Length`` +#: (#672, #679) -- makes the floor division drop four octets, and +#: ``_read_hip_param``, which compares the recovered length exactly, rejects the +#: packet. Two such records sum to a multiple of eight, so a pair makes the +#: arithmetic come out exact and the defect invisible. +#: +#: ``HIP_TRANSFORM`` is HIPv1-only -- ``_read_param_hip_transform`` raises for +#: any other version -- and is built at version 1 through :data:`HIP_VERSION`. +#: ``R1_Counter`` (128) parses through ``R1CounterParameter``, which the schema +#: registry holds for both counter codes (#690). Every HIP code round-trips at +#: one copy, so :data:`tests.protocols.test_option_roundtrip_unit.EXPECTED_FAILURES` +#: has no HIP entries. +#: +#: The single-parameter case is asserted directly, and positively, by +#: ``test_a_hip_packet_carrying_one_parameter_round_trips`` in +#: :mod:`tests.protocols.test_option_roundtrip_unit`, which is also where the +#: two-copy shape is exercised, since this constant does not produce it. +HIP_COPIES = 1 + + +def _hip_registry() -> 'Any': + from pcapkit.const.hip.parameter import Parameter + from pcapkit.protocols.internet.hip import HIP + return generator._named_registry(HIP, Parameter, '_make_param_', '__parameter__') + + +def _hip_overrides() -> 'dict[Any, dict[str, Any]]': + from pcapkit.const.hip.ecdsa_curve import ECDSACurve + from pcapkit.const.hip.parameter import Parameter + return { + # ``lifetime=0`` reaches ``math.log2(0)``. + Parameter.PUZZLE: {'lifetime': 1}, + Parameter.SOLUTION: {'lifetime': 1}, + # ``hi_curve=None`` falls through to an explicit raise. + Parameter.HOST_ID: {'hi_curve': ECDSACurve.NIST_P_256}, + # These two are not here because the default fails -- ``counter=0`` + # constructs and parses perfectly well. They are here because a + # zero-valued field cannot discriminate a width defect (#672) from a + # correct width. An RFC-only walk over ``options-internet.pcap``, which + # takes its stride solely from :rfc:`7401` Section 5.2.1's ``Total + # Length = 11 + Length - (Length + 3) % 8``, mis-strides over a counter + # packed at the wrong width; where every octet it mis-reads is zero, its + # zero-padding check passes and it reports no violation. With a non-zero + # counter the fixture cannot conceal a mis-stride here, whether or not + # the width is right. + Parameter.R1_COUNTER: {'counter': 0xaabbccdd}, + Parameter.R1_Counter: {'counter': 0xaabbccdd}, + } + + +def _hip_base(code: 'Any') -> 'dict[str, Any]': + base = dict(HIP_BASE) + base['version'] = HIP_VERSION.get(int(code), 2) + return base + + +def _hip_build(code: 'Any', kwargs: 'dict[str, Any]') -> 'Any': + from pcapkit.protocols.internet.hip import HIP + + # ``extension=True`` is not decoration. ``HIP.alias`` reads ``self._info``, + # which ``__post_init__`` only assigns after ``read`` returns, and the sole + # path that returns before touching it is the ``if extension`` early exit. + # Without it every HIP code fails with ``AttributeError: 'HIP' + # object has no attribute '_info'``. + return HIP(parameters=[(code, kwargs)] * HIP_COPIES, + extension=True, **_hip_base(code)) + + +def _hip_parse(octets: 'bytes') -> 'Any': + from pcapkit.protocols.internet.hip import HIP + return HIP(octets, len(octets), extension=True) + + +def _hip_extract(parsed: 'Any') -> 'Any': + from pcapkit.corekit.multidict import OrderedMultiDict + return getattr(parsed.info, 'parameters', OrderedMultiDict()) + + +def _hip_rebuild(parameters: 'Any') -> 'Any': + from pcapkit.protocols.internet.hip import HIP + + # ``parameters`` is the parsed collection, so the version has to come off + # one of its members rather than from the code being tested. + first = next(iter(parameters), None) + base = dict(HIP_BASE) + if first is not None: + base['version'] = HIP_VERSION.get(int(first), 2) + return HIP(parameters=parameters, extension=True, **base) + + +#: The families this module adds. +FAMILIES = ( + generator.Family('hip-parameter', _hip_registry, _hip_overrides, + _hip_build, _hip_parse, _hip_extract, _hip_rebuild, + capture='options-internet.pcap', envelope='ipv6', proto=139), +) diff --git a/examples/generators/option_families/httpv2.py b/examples/generators/option_families/httpv2.py new file mode 100644 index 0000000000..f33f99ab0a --- /dev/null +++ b/examples/generators/option_families/httpv2.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- +"""The ``httpv2-frame`` family: HTTP/2 frames. + +A family module of :file:`examples/generators/options.py`, which loads it by +path and binds ``generator`` to itself before this body runs (#2615). + +""" +# Deferring these is the design, as in options.py -- see its module docstring. +# pylint: disable=import-outside-toplevel + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any + +#: :file:`examples/generators/options.py`, which its loader binds here +#: before this body runs. +generator = globals()['generator'] # type: Any + + +def _httpv2_registry() -> 'Any': + from pcapkit.protocols.application.httpv2 import HTTPv2 + return HTTPv2.__frame__ + + +def _httpv2_overrides() -> 'dict[Any, dict[str, Any]]': + from pcapkit.const.http.frame import Frame as Enum_Frame + from pcapkit.const.http.setting import Setting as Enum_Setting + return { + # ``settings=None``, the default, matches none of the accepted forms. + Enum_Frame.SETTINGS: {'settings': [(Enum_Setting.HEADER_TABLE_SIZE, 4096)]}, + Enum_Frame.GOAWAY: {'debug_data': b'\xde\xad'}, + # A server-initiated stream is even and non-zero (RFC 9113 Section 5.1.1). + Enum_Frame.PUSH_PROMISE: {'promised_sid': 2}, + } + + +#: HTTP/2 frame types that apply to the connection as a whole and so are sent +#: on stream 0 (RFC 9113 Sections 6.5, 6.7 and 6.8). Every other frame type is +#: sent on :data:`HTTPV2_STREAM`. +HTTPV2_CONNECTION_FRAMES = (4, 6, 7) # SETTINGS, PING, GOAWAY + +#: Stream identifier of every stream-level HTTP/2 frame: the first +#: client-initiated stream (RFC 9113 Section 5.1.1). ``DATA``, ``HEADERS``, +#: ``PRIORITY``, ``RST_STREAM``, ``PUSH_PROMISE`` and ``CONTINUATION`` are a +#: connection error on stream 0, and ``WINDOW_UPDATE`` may use either. +HTTPV2_STREAM = 1 + + +def _httpv2_build(code: 'Any', kwargs: 'dict[str, Any]') -> 'Any': + from pcapkit.protocols.application.httpv2 import HTTPv2 + + sid = 0 if int(code) in HTTPV2_CONNECTION_FRAMES else HTTPV2_STREAM + return HTTPv2(type=code, sid=sid, frame=dict(kwargs)) + + +def _httpv2_parse(octets: 'bytes') -> 'Any': + from pcapkit.protocols.application.httpv2 import HTTPv2 + return HTTPv2(octets, len(octets)) + + +def _httpv2_extract(parsed: 'Any') -> 'Any': + return parsed.info + + +def _httpv2_rebuild(frame: 'Any') -> 'Any': + from pcapkit.protocols.application.httpv2 import HTTPv2 + + # ``HTTP.make`` ORs ``flags`` into the bits the ``_make_http_*`` constructor + # derives from ``frame``, and that constructor knows only the flags its + # frame type defines. So the raw flags octet is passed on, as + # ``HTTP._make_data`` does, or a bit the type leaves undefined is dropped. + # ``reserved`` is passed on for the same reason: ``make`` defaults it to 0. + return HTTPv2(type=frame.type, sid=frame.sid, reserved=frame.reserved, + flags=frame.flags.__value__ if frame.flags is not None else 0, + frame=frame) + + +#: The families this module adds. +FAMILIES = ( + generator.Family('httpv2-frame', _httpv2_registry, _httpv2_overrides, + _httpv2_build, _httpv2_parse, _httpv2_extract, _httpv2_rebuild, + capture='options-transport.pcap', envelope='tcp'), +) diff --git a/examples/generators/option_families/icmp.py b/examples/generators/option_families/icmp.py new file mode 100644 index 0000000000..b1f925e938 --- /dev/null +++ b/examples/generators/option_families/icmp.py @@ -0,0 +1,125 @@ +# -*- coding: utf-8 -*- +"""The ``icmpv4-type`` and ``icmpv6-type`` families: ICMP and ICMPv6 message types. + +A family module of :file:`examples/generators/options.py`, which loads it by +path and binds ``generator`` to itself before this body runs (#2615). + +""" +# Deferring these is the design, as in options.py -- see its module docstring. +# pylint: disable=import-outside-toplevel + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any + +#: :file:`examples/generators/options.py`, which its loader binds here +#: before this body runs. +generator = globals()['generator'] # type: Any + +#: The invoking packet an ICMP error message quotes: an IPv4 header and the +#: first eight octets of its UDP datagram [:rfc:`792`]. +ICMPV4_QUOTED = (bytes.fromhex('4500001c00010000401100000a0000010a000002') + + bytes.fromhex('3039003500080000')) + +#: The invoking packet an ICMPv6 error message quotes: an IPv6 header and its +#: UDP header [:rfc:`4443#section-3`]. +ICMPV6_QUOTED = (bytes.fromhex('6000000000081140') + bytes(15) + b'\x01' + bytes(15) + b'\x02' + + bytes.fromhex('3039003500080000')) + +#: Echo arguments shared by both versions. The defaults are all zero and empty, +#: which cannot tell a swapped ``id`` and ``seq``, or a dropped ``data``, apart. +ICMP_ECHO = {'id': 0x1234, 'seq': 0x5678, 'data': b'ping'} + + +def _icmpv4_registry() -> 'Any': + from pcapkit.protocols.internet.icmpv4 import ICMPv4 + return ICMPv4.__type__ + + +def _icmpv4_overrides() -> 'dict[Any, dict[str, Any]]': + from pcapkit.const.icmp.type import Type as Enum_Type + return { + Enum_Type.Echo_Reply: dict(ICMP_ECHO), + Enum_Type.Echo: dict(ICMP_ECHO), + Enum_Type.Destination_Unreachable: {'data': ICMPV4_QUOTED}, + Enum_Type.Time_Exceeded: {'data': ICMPV4_QUOTED}, + # Octet 8 of the quoted header is its TTL. + Enum_Type.Parameter_Problem: {'pointer': 8, 'data': ICMPV4_QUOTED}, + } + + +def _icmpv4_build(code: 'Any', kwargs: 'dict[str, Any]') -> 'Any': + from pcapkit.protocols.internet.icmpv4 import ICMPv4 + + # ``body`` has to be a mapping for ``make`` to dispatch on ``__type__``; + # a ``bytes`` body is parsed against the schema registry instead. + return ICMPv4(type=code, code=0, body=dict(kwargs)) + + +def _icmpv4_parse(octets: 'bytes') -> 'Any': + from pcapkit.protocols.internet.icmpv4 import ICMPv4 + return ICMPv4(octets, len(octets)) + + +def _icmp_extract(parsed: 'Any') -> 'Any': + return parsed.info + + +def _icmpv4_rebuild(message: 'Any') -> 'Any': + from pcapkit.protocols.internet.icmpv4 import ICMPv4 + return ICMPv4(type=message.type, code=message.code, checksum=message.checksum, + body=message) + + +def _icmpv6_registry() -> 'Any': + from pcapkit.protocols.internet.icmpv6 import ICMPv6 + return ICMPv6.__type__ + + +def _icmpv6_overrides() -> 'dict[Any, dict[str, Any]]': + from pcapkit.const.icmpv6.type import Type as Enum_Type + return { + Enum_Type.Destination_Unreachable: {'data': ICMPV6_QUOTED}, + # The IPv6 minimum link MTU [RFC 8200 section 5]. + Enum_Type.Packet_Too_Big: {'mtu': 1280, 'data': ICMPV6_QUOTED}, + Enum_Type.Time_Exceeded: {'data': ICMPV6_QUOTED}, + # Octet 6 of the quoted header is its Next Header. + Enum_Type.Parameter_Problem: {'pointer': 6, 'data': ICMPV6_QUOTED}, + Enum_Type.Echo_Request: dict(ICMP_ECHO), + Enum_Type.Echo_Reply: dict(ICMP_ECHO), + } + + +def _icmpv6_build(code: 'Any', kwargs: 'dict[str, Any]') -> 'Any': + from pcapkit.protocols.internet.icmpv6 import ICMPv6 + + # With no enclosing IPv6 layer to supply the pseudo-header, the checksum is + # left zero, and the rebuild passes the parsed zero back. + return ICMPv6(type=code, code=0, body=dict(kwargs)) + + +def _icmpv6_parse(octets: 'bytes') -> 'Any': + from pcapkit.protocols.internet.icmpv6 import ICMPv6 + return ICMPv6(octets, len(octets)) + + +def _icmpv6_rebuild(message: 'Any') -> 'Any': + from pcapkit.protocols.internet.icmpv6 import ICMPv6 + return ICMPv6(type=message.type, code=message.code, checksum=message.checksum, + body=message) + + +#: The families this module adds. +FAMILIES = ( + # NOTE: The ICMP families write no capture; their wire form is pinned against + # RFC 792 and RFC 4443 by test_option_generator_icmp_2560_unit instead. + generator.Family('icmpv4-type', _icmpv4_registry, _icmpv4_overrides, + _icmpv4_build, _icmpv4_parse, _icmp_extract, _icmpv4_rebuild, + capture=None, envelope=None), + generator.Family('icmpv6-type', _icmpv6_registry, _icmpv6_overrides, + _icmpv6_build, _icmpv6_parse, _icmp_extract, _icmpv6_rebuild, + capture=None, envelope=None), +) diff --git a/examples/generators/option_families/ipv4.py b/examples/generators/option_families/ipv4.py new file mode 100644 index 0000000000..c1433b5249 --- /dev/null +++ b/examples/generators/option_families/ipv4.py @@ -0,0 +1,85 @@ +# -*- coding: utf-8 -*- +"""The ``ipv4-option`` family: IPv4 options. + +A family module of :file:`examples/generators/options.py`, which loads it by +path and binds ``generator`` to itself before this body runs (#2615). + +""" +# Deferring these is the design, as in options.py -- see its module docstring. +# pylint: disable=import-outside-toplevel + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any + +#: :file:`examples/generators/options.py`, which its loader binds here +#: before this body runs. +generator = globals()['generator'] # type: Any + +#: Header fields shared by every constructed IPv4 packet. +IPV4_BASE = {'protocol': 6, 'src': '192.0.2.1', 'dst': '198.51.100.1', 'payload': b''} + + +def _ipv4_registry() -> 'Any': + from pcapkit.const.ipv4.option_number import OptionNumber + from pcapkit.protocols.internet.ipv4 import IPv4 + return generator._named_registry(IPv4, OptionNumber, '_make_opt_', '__option__') + + +def _ipv4_overrides() -> 'dict[Any, dict[str, Any]]': + from pcapkit.const.ipv4.option_number import OptionNumber + from pcapkit.const.ipv4.protection_authority import ProtectionAuthority + return { + # Two authorities rather than one because two bits set in the bitmap say + # more than one does, not because one is unrepresentable. Without + # #537's fix, a single ``GENSER`` (value 0) makes ``_make_opt_sec`` size + # a zero-octet bitmap and then index into it for a bare ``IndexError``. + # So this is a coverage choice and routes around nothing. + OptionNumber.SEC: {'authorities': [ProtectionAuthority.GENSER, + ProtectionAuthority.NSA]}, + # ``counts=10``, the default, needs 43 option octets, which overflows + # the 4-bit ``ihl``. Nine is the largest an IPv4 header can carry. + OptionNumber.LSR: {'counts': 9}, + OptionNumber.RR: {'counts': 9}, + OptionNumber.SSR: {'counts': 9}, + # The default builds an option with no timestamp in it, which is + # degenerate; one is recorded. + OptionNumber.TS: {'counts': 1, 'timestamp': [1]}, + OptionNumber.E_SEC: {'info': b'\x00'}, + } + + +def _ipv4_build(code: 'Any', kwargs: 'dict[str, Any]') -> 'Any': + from pcapkit.protocols.internet.ipv4 import IPv4 + return IPv4(options=[(code, kwargs)], **IPV4_BASE) + + +def _ipv4_parse(octets: 'bytes') -> 'Any': + from pcapkit.protocols.internet.ipv4 import IPv4 + return IPv4(octets, len(octets)) + + +def _ipv4_extract(parsed: 'Any') -> 'Any': + from pcapkit.corekit.multidict import OrderedMultiDict + + # ``_make_ipv4_options`` drops EOOL and NOP from a fresh option list as + # padding, so the header a case builds from either has no options at all and ``info`` carries no + # ``options`` attribute. An empty collection is the honest reading of that, + # and it keeps the two padding codes from being reported as parse failures. + return getattr(parsed.info, 'options', OrderedMultiDict()) + + +def _ipv4_rebuild(options: 'Any') -> 'Any': + from pcapkit.protocols.internet.ipv4 import IPv4 + return IPv4(options=options, **IPV4_BASE) + + +#: The families this module adds. +FAMILIES = ( + generator.Family('ipv4-option', _ipv4_registry, _ipv4_overrides, + _ipv4_build, _ipv4_parse, _ipv4_extract, _ipv4_rebuild, + capture='options-ipv4.pcap', envelope='ethernet-ipv4'), +) diff --git a/examples/generators/option_families/ipv6.py b/examples/generators/option_families/ipv6.py new file mode 100644 index 0000000000..4e8e54bd1e --- /dev/null +++ b/examples/generators/option_families/ipv6.py @@ -0,0 +1,168 @@ +# -*- coding: utf-8 -*- +"""The ``hopopt-option``, ``ipv6-opts-option`` and ``ipv6-route-type`` families: the +IPv6 extension headers HOPOPT, IPv6-Opts and IPv6-Route. + +A family module of :file:`examples/generators/options.py`, which loads it by +path and binds ``generator`` to itself before this body runs (#2615). + +""" +# Deferring these is the design, as in options.py -- see its module docstring. +# pylint: disable=import-outside-toplevel + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any + +#: :file:`examples/generators/options.py`, which its loader binds here +#: before this body runs. +generator = globals()['generator'] # type: Any + + +def _hopopt_registry() -> 'Any': + from pcapkit.protocols.internet.hopopt import HOPOPT + return HOPOPT.__option__ + + +def _ipv6_opts_registry() -> 'Any': + from pcapkit.protocols.internet.ipv6_opts import IPv6_Opts + return IPv6_Opts.__option__ + + +def _ipv6_option_overrides() -> 'dict[Any, dict[str, Any]]': + from pcapkit.const.ipv6.option import Option as Enum_Option + + # HOPOPT and IPv6-Opts key on the same enumeration and behave identically + # on every code, so one table serves both; ``_hopopt_overrides`` adds the + # one value that matters only in a Hop-by-Hop Options header. + return { + # The nonce is sized to its bit length rounded up to whole octets, at + # least one, so the default ``nonce=0`` is a one-octet zero. A + # three-octet value exercises a multi-octet width; every width from 1 + # to 16 octets builds and round-trips. + Enum_Option.ILNP_Nonce: {'nonce': 0xFFFFFF}, + Enum_Option.Line_Identification_Option: {'id': b'line-1'}, + } + + +def _hopopt_overrides() -> 'dict[Any, dict[str, Any]]': + from pcapkit.const.ipv6.option import Option as Enum_Option + + # A Jumbo Payload option in the Hop-by-Hop Options header after an IPv6 + # header makes the packet a jumbogram, and one whose Jumbo Payload Length is + # not above 65,535 is an error (RFC 2675 section 3) the default engine + # rejects (#1687). So the option carries a legal length, and options.py's + # ``_frame`` builds the jumbogram around it. In IPv6-Opts the option is just + # an option. + return {**_ipv6_option_overrides(), Enum_Option.Jumbo_Payload: {'len': generator.JUMBO_LEN}} + + +def _hopopt_base() -> 'dict[str, Any]': + from pcapkit.const.reg.transtype import TransType + return {'next': TransType.UDP, 'payload': b''} + + +def _hopopt_build(code: 'Any', kwargs: 'dict[str, Any]') -> 'Any': + from pcapkit.protocols.internet.hopopt import HOPOPT + return HOPOPT(options=[(code, kwargs)], **_hopopt_base()) + + +def _hopopt_parse(octets: 'bytes') -> 'Any': + from pcapkit.protocols.internet.hopopt import HOPOPT + return HOPOPT(octets, len(octets), extension=True) + + +def _ipv6_extract(parsed: 'Any') -> 'Any': + from pcapkit.corekit.multidict import OrderedMultiDict + return getattr(parsed.info, 'options', OrderedMultiDict()) + + +def _hopopt_rebuild(options: 'Any') -> 'Any': + from pcapkit.protocols.internet.hopopt import HOPOPT + return HOPOPT(options=options, **_hopopt_base()) + + +def _ipv6_opts_build(code: 'Any', kwargs: 'dict[str, Any]') -> 'Any': + from pcapkit.protocols.internet.ipv6_opts import IPv6_Opts + return IPv6_Opts(options=[(code, kwargs)], **_hopopt_base()) + + +def _ipv6_opts_parse(octets: 'bytes') -> 'Any': + from pcapkit.protocols.internet.ipv6_opts import IPv6_Opts + return IPv6_Opts(octets, len(octets), extension=True) + + +def _ipv6_opts_rebuild(options: 'Any') -> 'Any': + from pcapkit.protocols.internet.ipv6_opts import IPv6_Opts + return IPv6_Opts(options=options, **_hopopt_base()) + + +def _ipv6_route_registry() -> 'Any': + from pcapkit.protocols.internet.ipv6_route import IPv6_Route + return IPv6_Route.__routing__ + + +def _ipv6_route_overrides() -> 'dict[Any, dict[str, Any]]': + from pcapkit.const.ipv6.routing import Routing as Enum_Routing + return { + Enum_Routing.Source_Route: {'ip': ['2001:db8::1', '2001:db8::2']}, + Enum_Routing.Type_2_Routing_Header: {'ip': '2001:db8::2'}, + Enum_Routing.RPL_Source_Route_Header: {'ip': ['2001:db8::1', '2001:db8::2']}, + } + + +def _ipv6_route_base() -> 'dict[str, Any]': + from pcapkit.const.reg.transtype import TransType + return {'next': TransType.UDP, 'seg_left': 0, 'payload': b''} + + +def _ipv6_route_build(code: 'Any', kwargs: 'dict[str, Any]') -> 'Any': + from pcapkit.protocols.internet.ipv6_route import IPv6_Route + return IPv6_Route(type=code, data=dict(kwargs), **_ipv6_route_base()) + + +def _ipv6_route_parse(octets: 'bytes') -> 'Any': + from pcapkit.protocols.internet.ipv6_route import IPv6_Route + return IPv6_Route(octets, len(octets), extension=True) + + +def _ipv6_route_extract(parsed: 'Any') -> 'Any': + # A routing header carries one routing type, not a collection of them, so + # the "collection" handed on is the message itself. + return parsed.info + + +def _ipv6_route_rebuild(info: 'Any') -> 'Any': + from pcapkit.protocols.internet.ipv6_route import IPv6_Route + return IPv6_Route(type=info.type, data=info, next=info.next, + seg_left=info.seg_left, payload=b'') + + +#: The families this module adds. +FAMILIES = ( + generator.Family('hopopt-option', _hopopt_registry, _hopopt_overrides, + _hopopt_build, _hopopt_parse, _ipv6_extract, _hopopt_rebuild, + capture='options-ipv6.pcap', envelope='ipv6', proto=0), + generator.Family('ipv6-opts-option', _ipv6_opts_registry, _ipv6_option_overrides, + _ipv6_opts_build, _ipv6_opts_parse, _ipv6_extract, _ipv6_opts_rebuild, + capture='options-ipv6.pcap', envelope='ipv6', proto=60), + generator.Family('ipv6-route-type', _ipv6_route_registry, _ipv6_route_overrides, + _ipv6_route_build, _ipv6_route_parse, _ipv6_route_extract, + _ipv6_route_rebuild, + capture='options-ipv6.pcap', envelope='ipv6', proto=43), +) + +#: Codes of :data:`FAMILIES` left out of the case list, with the reason; see +#: :data:`examples.generators.options.SKIP`. +#: +#: Every padding option has a case except PadN in HOPOPT and IPv6-Opts. Their +#: ``_make_*_options`` keep the padding of parsed options in place, but drop +#: any in a list of fresh options and insert their own to reach 8-octet +#: alignment. A case is built from such a list, so the Pad1 and PadN cases +#: would both emit the same option-less header; Pad1 is kept, PadN skipped. +SKIP = { + ('hopopt-option', 'PadN'): 'padding, indistinguishable from Pad1 here', + ('ipv6-opts-option', 'PadN'): 'padding, indistinguishable from Pad1 here', +} diff --git a/examples/generators/option_families/mh.py b/examples/generators/option_families/mh.py new file mode 100644 index 0000000000..e14cc772b0 --- /dev/null +++ b/examples/generators/option_families/mh.py @@ -0,0 +1,189 @@ +# -*- coding: utf-8 -*- +"""The ``mh-message``, ``mh-option`` and ``mh-extension`` families: Mobility Header +messages, options and extensions. + +A family module of :file:`examples/generators/options.py`, which loads it by +path and binds ``generator`` to itself before this body runs (#2615). + +""" +# Deferring these is the design, as in options.py -- see its module docstring. +# pylint: disable=import-outside-toplevel + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any + +#: :file:`examples/generators/options.py`, which its loader binds here +#: before this body runs. +generator = globals()['generator'] # type: Any + +#: Header fields shared by every constructed Mobility Header. ``next`` is +#: IPv6-NoNxt, so nothing downstream has to be constructed as well. +MH_BASE = {'next': 59, 'chksum': b'\x00\x00', 'payload': b''} + + +def _mh_message_registry() -> 'Any': + from pcapkit.protocols.internet.mh import MH + return MH.__message__ + + +def _mh_message_overrides() -> 'dict[Any, dict[str, Any]]': + return {} + + +def _mh_message_build(code: 'Any', kwargs: 'dict[str, Any]') -> 'Any': + from pcapkit.protocols.internet.mh import MH + + # ``data`` has to be a mapping for ``make`` to dispatch on the message + # type; a Data object takes the other branch, which is what rebuild uses. + return MH(type=code, data=dict(kwargs), **MH_BASE) + + +def _mh_parse(octets: 'bytes') -> 'Any': + from pcapkit.protocols.internet.mh import MH + return MH(octets, len(octets), extension=True) + + +def _mh_message_extract(parsed: 'Any') -> 'Any': + return parsed.info + + +def _mh_message_rebuild(message: 'Any') -> 'Any': + from pcapkit.protocols.internet.mh import MH + return MH(type=message.type, data=message, **MH_BASE) + + +def _mh_option_registry() -> 'Any': + from pcapkit.protocols.internet.mh import MH + return MH.__option__ + + +def _mh_option_overrides() -> 'dict[Any, dict[str, Any]]': + import ipaddress + + from pcapkit.const.mh.ani_suboption import ANISuboption as Enum_ANISuboption + from pcapkit.const.mh.lma_mag_suboption import \ + LMAControlledMAGSuboption as Enum_LMAControlledMAGSuboption + from pcapkit.const.mh.option import Option as Enum_Option + return { + # ``{}`` gives a valid zero-length PadN [RFC 6275 section 6.2.3]; a + # non-zero length exercises the padding octets as well. + Enum_Option.PadN: {'length': 4}, + # The authenticator has to be a multiple of 8 octets. + Enum_Option.Authorization_Data: {'data': b'\xa5' * 8}, + Enum_Option.Mobility_Header_Link_Layer_Address_option: { + 'address': b'\x00\x11\x22\x33\x44\x55'}, + # ``(len + 6) % 4`` has to be 0. + Enum_Option.AUTH_OPTION_TYPE: {'data': b'\xa5\xa5'}, + # Without this ``_make_opt_mesg_id`` reads the clock, and the capture + # would differ on every regeneration. + Enum_Option.MESG_ID_OPTION_TYPE: {'interval': generator.FIXED_TIME}, + Enum_Option.Signature: {'signature': b'\xaa' * 4}, + Enum_Option.Permanent_Home_Keygen_Token: {'token': b'\xbb' * 8}, + Enum_Option.Experimental_Mobility_Option: {'data': b'\xcc' * 4}, + Enum_Option.Binding_Authorization_Data_for_FMIPv6: {'data': b'\xb5'}, + # The entries below are not workarounds for defects: each option has a + # minimum content its RFC requires, and the no-argument default is not a + # well-formed instance of it. The constructors refuse it, correctly, so + # the arguments here are what make the code reachable at all. + # + # ``Length`` of 0 "is not allowed" and the identifier is 1-255 octets + # [RFC 5149 section 3]. ``'ims'`` is that RFC's own example. + Enum_Option.Service_Selection_Mobility_Option: {'identifier': 'ims'}, + # "Both the 'K' and 'N' flags cannot be set or unset simultaneously" + # [RFC 6463 section 4.2], so exactly one address is present and the + # option is 18 octets long for IPv6 or 6 for IPv4. With neither given + # the option's own length is undetermined. + Enum_Option.Redirect_Mobility_Option: { + 'ipv6': ipaddress.IPv6Address('2001:db8::1')}, + # The option "MUST contain at least one ANI sub-option" + # [RFC 6757 section 3]; it is a pure container and carries nothing else. + Enum_Option.Access_Network_Identifier: { + 'suboptions': [(Enum_ANISuboption.Network_Identifier, + {'net_name': b'wifi', + 'ap_name': b'\x00\x11\x22\x33\x44\x55'})]}, + # Likewise at least one LCMP sub-option [RFC 8127 section 3]. + Enum_Option.LMA_Controlled_MAG_Parameters: { + 'suboptions': [(Enum_LMAControlledMAGSuboption.Heartbeat_Control, + {'interval': 60, 'retransmission_delay': 3, + 'max_retransmissions': 5})]}, + } + + +def _mh_carrier() -> 'Any': + from pcapkit.const.mh.packet import Packet as Enum_Packet + + # Binding Refresh Request is the smallest message that carries an + # arbitrary option list: two reserved octets and then the options. + return Enum_Packet.Binding_Refresh_Request + + +def _mh_option_build(code: 'Any', kwargs: 'dict[str, Any]') -> 'Any': + from pcapkit.protocols.internet.mh import MH + return MH(type=_mh_carrier(), data={'options': [(code, dict(kwargs))]}, **MH_BASE) + + +def _mh_option_extract(parsed: 'Any') -> 'Any': + from pcapkit.corekit.multidict import OrderedMultiDict + return getattr(parsed.info, 'options', OrderedMultiDict()) + + +def _mh_option_rebuild(options: 'Any') -> 'Any': + from pcapkit.protocols.internet.mh import MH + return MH(type=_mh_carrier(), data={'options': options}, **MH_BASE) + + +def _mh_extension_registry() -> 'Any': + from pcapkit.protocols.internet.mh import MH + return MH.__extension__ + + +def _mh_extension_overrides() -> 'dict[Any, dict[str, Any]]': + from pcapkit.const.mh.cga_extension import CGAExtension as Enum_CGAExtension + return {Enum_CGAExtension.Multi_Prefix: {'prefixes': [0x20010db800000001]}} + + +def _mh_extension_build(code: 'Any', kwargs: 'dict[str, Any]') -> 'Any': + from pcapkit.const.mh.cga_type import CGAType as Enum_CGAType + from pcapkit.const.mh.option import Option as Enum_Option + from pcapkit.protocols.data.internet.mh import CGAParameter as Data_CGAParameter + from pcapkit.protocols.internet.mh import MH + + # A CGA extension has exactly one carrier: the ``extensions`` list of a CGA + # parameter, inside a CGA Parameters option. It has to be a *Data* model + # rather than a schema, because ``_make_opt_cga_param`` keeps only the + # length of the extensions it builds for a schema and throws the schemas + # themselves away. + # The data model annotates ``modifier`` as a CGAType and ``extensions`` as an + # already-built OrderedMultiDict, but the constructor this feeds takes the + # tuple form and resolves both -- so the arguments go through an untyped + # mapping rather than being spelled as keywords a checker would reject. + fields = { + 'modifier': Enum_CGAType.Tag_086F_CA5E_10B2_00C9_9C8C_E001_6427_7C08, + 'prefix': 0x20010db800000000, + 'collision_count': 0, + 'public_key': b'\x30\x02\xaa\xbb', + 'extensions': [(code, dict(kwargs))], + } # type: dict[str, Any] + parameter = Data_CGAParameter(**fields) + return MH(type=_mh_carrier(), + data={'options': [(Enum_Option.CGA_Parameters, + {'parameters': [parameter]})]}, + **MH_BASE) + + +#: The families this module adds. +FAMILIES = ( + generator.Family('mh-message', _mh_message_registry, _mh_message_overrides, + _mh_message_build, _mh_parse, _mh_message_extract, _mh_message_rebuild, + capture='options-internet.pcap', envelope='ipv6', proto=135), + generator.Family('mh-option', _mh_option_registry, _mh_option_overrides, + _mh_option_build, _mh_parse, _mh_option_extract, _mh_option_rebuild, + capture='options-internet.pcap', envelope='ipv6', proto=135), + generator.Family('mh-extension', _mh_extension_registry, _mh_extension_overrides, + _mh_extension_build, _mh_parse, _mh_option_extract, _mh_option_rebuild, + capture='options-internet.pcap', envelope='ipv6', proto=135), +) diff --git a/examples/generators/option_families/pcapng.py b/examples/generators/option_families/pcapng.py new file mode 100644 index 0000000000..a12956c1b0 --- /dev/null +++ b/examples/generators/option_families/pcapng.py @@ -0,0 +1,421 @@ +# -*- coding: utf-8 -*- +"""The ``pcapng-block``, ``pcapng-option``, ``pcapng-record`` and ``pcapng-secrets`` +families: PCAP-NG blocks, block options, name records and decryption secrets. + +These families are exercised by +:func:`~examples.generators.options.roundtrip` but deliberately write no +capture, and the reason is not that a PCAP-NG capture would be uninteresting -- +it is that the construction API's defaults do not produce a *reproducible* +one. ``_make_block_shb`` takes the section's byte order from +:data:`sys.byteorder` unless given ``byteorder``, and the TLS and WireGuard +key-log writers stamp the running pcapkit version into the block body. A +fixture built this way would differ between a little-endian and a big-endian +host, and the two key-log types would differ between one release and the next. + +The PCAP-NG option space is not left uncovered by that decision: +:file:`examples/generators/pcapng.py` already builds ``profile.pcapng``, +``test.pcapng`` and ``many_interfaces.pcapng`` over it, hand-rolling every +octet with :func:`struct.pack` for exactly this reason. What is added here is +the half that generator cannot reach, since it never calls the construction +API at all: whether ``_make_block_*`` and ``_make_option_*`` can produce what +``_read_block_*`` and ``_read_option_*`` consume. + +A family module of :file:`examples/generators/options.py`, which loads it by +path and binds ``generator`` to itself before this body runs (#2615). + +""" +# Deferring these is the design, as in options.py -- see its module docstring. +# pylint: disable=import-outside-toplevel + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any + +#: :file:`examples/generators/options.py`, which its loader binds here +#: before this body runs. +generator = globals()['generator'] # type: Any + +#: Fixed PCAP-NG timestamp, the same instant :file:`pcapng.py` uses. Not +#: optional: leaving ``timestamp`` unset reaches ``self._info`` before +#: ``__post_init__`` has assigned it, so the "now" default is unreachable as +#: well as non-deterministic. +PCAPNG_TIME = 1500000000 + +#: A minimal Ethernet frame for the packet-carrying blocks: destination and +#: source address, an unassigned EtherType, and two octets of body. +#: +#: It has to be non-empty. A ``PayloadField`` whose computed length is +#: legitimately zero reads the whole remainder of the block instead of nothing, +#: which swallows the option area -- so an empty packet would silently take the +#: whole ``epb_*`` and ``pack_*`` option space with it. Sixteen octets rather +#: than the bare 14-octet header, so the Ethernet dissector does not under-run. +#: 6 octets of destination, 6 of source, the EtherType, then the body. +PCAPNG_PACKET = bytes.fromhex('02000000000102000000000288b50000') + +#: A systemd journal export entry whose length is a multiple of four, so that +#: ``_make_block_systemd`` has no padding to add to it. +PCAPNG_JOURNAL = (b'__REALTIME_TIMESTAMP=1500000000000000\n_TRANSPORT=journal\n' + b'PRIORITY=6\nMESSAGE=probe.....\n\n') + + +def _pcapng_context() -> 'tuple[Any, Any]': + """A one-section PCAP-NG context with one Ethernet interface in it. + + Every block other than a Section Header Block is read and written relative + to a section, and the packet-carrying blocks resolve their snap length + through the section's interface list, so there has to be one. + + Returns: + The :class:`~pcapkit.foundation.engines.pcapng.Context` and the + interface-description block registered in it. + + """ + from pcapkit.const.pcapng.block_type import BlockType as Enum_BlockType + from pcapkit.const.reg.linktype import LinkType as Enum_LinkType + from pcapkit.foundation.engines.pcapng import Context + from pcapkit.protocols.misc.pcapng import PCAPNG + + section = PCAPNG(num=0, sct=1, ctx=None, + type=Enum_BlockType.Section_Header_Block, block={}) + # ``Protocol.info`` is annotated as the generic data model for the protocol, + # so a checker cannot see that a Section Header Block's is the one ``Context`` + # wants. The engine does exactly this, in ``PCAPNG.run`` of + # pcapkit/foundation/engines/pcapng.py. + section_info = section.info # type: Any + context = Context(section_info) + interface = PCAPNG(num=1, sct=1, ctx=context, + type=Enum_BlockType.Interface_Description_Block, + block={'linktype': Enum_LinkType.ETHERNET, 'snaplen': 0x40000}) + interface_info = interface.info # type: Any + context.interfaces.append(interface_info) + return context, interface_info + + +def _pcapng_block(code: 'Any', block: 'dict[str, Any] | Any') -> 'Any': + """Construct one PCAP-NG block of ``code``. + + A fresh context per call, deliberately. ``PCAPNG.__post_init__`` packs and + then re-reads *on the same instance*, and the two halves share the + per-instance option counter, so state left by one case would change the + next one's result. + + Args: + code: Block type. + block: Block body, as a mapping of constructor arguments or as the + parsed data model. + + Returns: + The constructed block. + + """ + from pcapkit.const.pcapng.block_type import BlockType as Enum_BlockType + from pcapkit.protocols.misc.pcapng import PCAPNG + + # A Section Header Block is what *starts* a section, so it is the one block + # the engine reads with no context at all. + context = None if code == Enum_BlockType.Section_Header_Block else _pcapng_context()[0] + return PCAPNG(num=2, sct=1, ctx=context, type=code, block=block) + + +#: Block type of a Section Header Block, as it appears on the wire. The value +#: is byte-order independent by design -- that is what lets a reader work out a +#: section's endianness -- so it can be recognised before the byte order is +#: known, which is exactly what :func:`_pcapng_parse` needs. +PCAPNG_SHB_MAGIC = bytes.fromhex('0a0d0d0a') + + +def _pcapng_parse(octets: 'bytes') -> 'Any': + """Parse one PCAP-NG block back from its own octets. + + The block type is read out of the octets rather than passed in, so that + this keeps the one-argument shape every other family's ``parse`` has. It is + the first four octets of any block, and a Section Header Block -- the one + block read with no section context, because it is what *starts* a + section -- is recognisable by :data:`PCAPNG_SHB_MAGIC`. + + Args: + octets: The constructed block. + + Returns: + The parsed block. + + """ + from pcapkit.protocols.misc.pcapng import PCAPNG + + context = None if octets[:4] == PCAPNG_SHB_MAGIC else _pcapng_context()[0] + return PCAPNG(octets, len(octets), num=2, sct=1, ctx=context) + + +def _pcapng_block_registry() -> 'Any': + from pcapkit.protocols.misc.pcapng import PCAPNG + return PCAPNG.__block__ + + +def _pcapng_block_overrides() -> 'dict[Any, dict[str, Any]]': + from pcapkit.const.pcapng.block_type import BlockType as Enum_BlockType + from pcapkit.const.pcapng.record_type import RecordType as Enum_RecordType + from pcapkit.const.pcapng.secrets_type import SecretsType as Enum_SecretsType + from pcapkit.const.reg.linktype import LinkType as Enum_LinkType + return { + Enum_BlockType.Interface_Description_Block: { + 'linktype': Enum_LinkType.ETHERNET, 'snaplen': 0x40000}, + Enum_BlockType.Enhanced_Packet_Block: { + 'timestamp': PCAPNG_TIME, 'packet_data': PCAPNG_PACKET}, + Enum_BlockType.Simple_Packet_Block: {'packet_data': PCAPNG_PACKET}, + # Without the terminating record the records field consumes the whole + # block and the option area disappears. + Enum_BlockType.Name_Resolution_Block: { + 'records': [(Enum_RecordType.nrb_record_end, {})]}, + Enum_BlockType.Interface_Statistics_Block: {'timestamp': PCAPNG_TIME}, + Enum_BlockType.systemd_Journal_Export_Block: {'entries': PCAPNG_JOURNAL}, + # ZigBee rather than the default TLS key log: the default empty + # ``secrets_data`` does not read back as a key log. + Enum_BlockType.Decryption_Secrets_Block: { + 'secrets_type': Enum_SecretsType.ZigBee_NWK_Key, + 'secrets_data': {'nwk_key': bytes(range(16)), 'pan_id': 0x1234}}, + Enum_BlockType.Custom_Block_that_rewriters_can_copy_into_new_files: { + 'pen': 32473, 'data': b'\x01\x02\x03\x04'}, + Enum_BlockType.Custom_Block_that_rewriters_should_not_copy_into_new_files: { + 'pen': 32473, 'data': b'\x01\x02\x03\x04'}, + Enum_BlockType.Packet_Block: { + 'timestamp': PCAPNG_TIME, 'packet_data': PCAPNG_PACKET}, + } + + +def _pcapng_block_build(code: 'Any', kwargs: 'dict[str, Any]') -> 'Any': + return _pcapng_block(code, dict(kwargs)) + + +def _pcapng_block_extract(parsed: 'Any') -> 'Any': + return parsed.info + + +def _pcapng_block_rebuild(info: 'Any') -> 'Any': + return _pcapng_block(info.type, info) + + +#: Which block hosts each block-option namespace, and that host's own +#: arguments. The namespace is the part of the ``_option_key`` tuple's first +#: element before the first underscore. +PCAPNG_OPTION_HOSTS = { + 'opt': 'Section_Header_Block', + 'if': 'Interface_Description_Block', + 'epb': 'Enhanced_Packet_Block', + 'ns': 'Name_Resolution_Block', + 'isb': 'Interface_Statistics_Block', + 'pack': 'Packet_Block', +} + + +def _pcapng_option_registry() -> 'Any': + from pcapkit.protocols.misc.pcapng import PCAPNG + return PCAPNG.__option__ + + +def _pcapng_option_overrides() -> 'dict[Any, dict[str, Any]]': + from pcapkit.const.pcapng.option_type import OptionType as Enum_OptionType + from pcapkit.protocols.misc.pcapng import _option_key + return { + _option_key(Enum_OptionType.opt_comment): {'comment': 'probe'}, + _option_key(Enum_OptionType.opt_custom_2988): {'pen': 32473, 'data': b'\x01\x02\x03\x04'}, + _option_key(Enum_OptionType.opt_custom_2989): {'pen': 32473, 'data': b'\x01\x02\x03\x04'}, + _option_key(Enum_OptionType.opt_custom_19372): {'pen': 32473, 'data': b'\x01\x02\x03\x04'}, + _option_key(Enum_OptionType.opt_custom_19373): {'pen': 32473, 'data': b'\x01\x02\x03\x04'}, + _option_key(Enum_OptionType.if_name): {'name': 'eth0'}, + _option_key(Enum_OptionType.if_description): {'description': 'probe interface'}, + _option_key(Enum_OptionType.if_IPv4addr): {'interface': '10.0.0.1/255.255.255.0'}, + _option_key(Enum_OptionType.if_IPv6addr): { + 'interface': '2001:db8:85a3:8d3:1319:8a2e:370:7344/64'}, + _option_key(Enum_OptionType.if_MACaddr): {'interface': '02:00:00:00:00:01'}, + _option_key(Enum_OptionType.if_EUIaddr): {'interface': '02:00:00:FF:FE:00:00:01'}, + _option_key(Enum_OptionType.if_speed): {'speed': 1000000000}, + _option_key(Enum_OptionType.if_tsresol): {'resolution': 1000000}, + _option_key(Enum_OptionType.if_tzone): {'tzone': 0}, + _option_key(Enum_OptionType.if_filter): {'expression': b'udp port 67'}, + # ``if_os`` and ``if_hardware`` default to the live platform strings, so + # leaving them unset would make the case machine-dependent. + _option_key(Enum_OptionType.if_os): {'os': 'probe OS'}, + _option_key(Enum_OptionType.if_hardware): {'hardware': 'probe adapter'}, + _option_key(Enum_OptionType.if_fcslen): {'fcs_length': 4}, + _option_key(Enum_OptionType.if_tsoffset): {'offset': 0}, + _option_key(Enum_OptionType.if_txspeed): {'speed': 1000000000}, + _option_key(Enum_OptionType.if_rxspeed): {'speed': 1000000000}, + _option_key(Enum_OptionType.epb_hash): {'hash': b'\x01\x02\x03\x04'}, + _option_key(Enum_OptionType.epb_verdict): {'value': b'\x01'}, + _option_key(Enum_OptionType.ns_dnsname): {'name': 'resolver.example'}, + _option_key(Enum_OptionType.ns_dnsIP4addr): {'ip': '10.0.0.53'}, + # The default is the IPv4 literal ``'8.8.8.8'``, which the v6 field + # rejects outright. + _option_key(Enum_OptionType.ns_dnsIP6addr): {'ip': '2001:db8::35'}, + _option_key(Enum_OptionType.isb_starttime): {'timestamp': PCAPNG_TIME}, + _option_key(Enum_OptionType.isb_endtime): {'timestamp': PCAPNG_TIME}, + _option_key(Enum_OptionType.isb_ifrecv): {'packets': 4}, + _option_key(Enum_OptionType.isb_ifdrop): {'packets': 1}, + _option_key(Enum_OptionType.isb_filteraccept): {'packets': 4}, + _option_key(Enum_OptionType.isb_osdrop): {'packets': 0}, + _option_key(Enum_OptionType.isb_usrdeliv): {'packets': 3}, + _option_key(Enum_OptionType.pack_hash): {'hash': b'\x01\x02\x03\x04'}, + } + + +def _pcapng_option_enum(key: 'Any') -> 'Any': + """The enumeration member behind an ``_option_key`` tuple. + + ``PCAPNG.__option__`` is keyed by ``(name, value)`` tuples because the same + numeric option code means different things in different block types, but + the construction API takes the enumeration member. + + Args: + key: The ``(str, int)`` registry key. + + Returns: + The matching :class:`~pcapkit.const.pcapng.option_type.OptionType`. + + Raises: + KeyError: If no member matches, which would mean the registry and the + enumeration have diverged. + + """ + from pcapkit.const.pcapng.option_type import OptionType as Enum_OptionType + from pcapkit.protocols.misc.pcapng import _option_key + + # The enumeration is an :mod:`aenum` one, whose metaclass does not advertise + # ``__iter__`` to a type checker even though it iterates perfectly well. + members = Enum_OptionType # type: Any + for member in members: + if _option_key(member) == key: + return member + raise KeyError(f'no OptionType matches registry key {key!r}') + + +def _pcapng_option_host(key: 'Any') -> 'tuple[Any, dict[str, Any]]': + """The block type that carries option ``key``, and its own arguments.""" + from pcapkit.const.pcapng.block_type import BlockType as Enum_BlockType + + namespace = str(key[0]).split('_', 1)[0] + name = PCAPNG_OPTION_HOSTS[namespace] + code = getattr(Enum_BlockType, name) + return code, dict(_pcapng_block_overrides().get(code, {})) + + +def _pcapng_option_build(key: 'Any', kwargs: 'dict[str, Any]') -> 'Any': + code, block = _pcapng_option_host(key) + block['options'] = [(_pcapng_option_enum(key), dict(kwargs))] + return _pcapng_block(code, block) + + +def _pcapng_option_rebuild(info: 'Any') -> 'Any': + """Rebuild an option's host block, taking only the options from the parse. + + The host's own fields come from :func:`_pcapng_block_overrides` rather than + from the parsed block, and that is the point. Reconstructing the whole block + from ``info`` would drop the captured payload of a packet-carrying block -- + ``_make_block_epb`` and its siblings never restore ``packet_data``, and the + data model has nowhere to keep it -- so every ``epb_*`` and ``pack_*`` + option would be reported as a mismatch on account of a defect that has + nothing to do with the option. Re-supplying the host fields isolates the + option, which is what this family is measuring. + + Args: + info: The parsed block. + + Returns: + The reconstructed block. + + """ + block = dict(_pcapng_block_overrides().get(info.type, {})) + block['options'] = info.options + return _pcapng_block(info.type, block) + + +def _pcapng_record_registry() -> 'Any': + from pcapkit.protocols.misc.pcapng import PCAPNG + return PCAPNG.__record__ + + +def _pcapng_record_overrides() -> 'dict[Any, dict[str, Any]]': + from pcapkit.const.pcapng.record_type import RecordType as Enum_RecordType + return { + Enum_RecordType.nrb_record_ipv4: {'ip': '10.0.0.1', 'names': ['gateway.example']}, + # The default is the IPv4 literal ``'127.0.0.1'``, which the v6 field + # rejects outright. + Enum_RecordType.nrb_record_ipv6: {'ip': '2001:db8::1', 'names': ['v6.example']}, + } + + +def _pcapng_record_build(code: 'Any', kwargs: 'dict[str, Any]') -> 'Any': + from pcapkit.const.pcapng.block_type import BlockType as Enum_BlockType + from pcapkit.const.pcapng.record_type import RecordType as Enum_RecordType + + records = [(code, dict(kwargs))] + if code != Enum_RecordType.nrb_record_end: + # A record list has to be terminated, or the records field eats the + # rest of the block. + records.append((Enum_RecordType.nrb_record_end, {})) + return _pcapng_block(Enum_BlockType.Name_Resolution_Block, {'records': records}) + + +def _pcapng_record_extract(parsed: 'Any') -> 'Any': + return parsed.info.records + + +def _pcapng_record_rebuild(records: 'Any') -> 'Any': + from pcapkit.const.pcapng.block_type import BlockType as Enum_BlockType + return _pcapng_block(Enum_BlockType.Name_Resolution_Block, {'records': records}) + + +def _pcapng_secrets_registry() -> 'Any': + from pcapkit.protocols.misc.pcapng import PCAPNG + return PCAPNG.__secrets__ + + +def _pcapng_secrets_overrides() -> 'dict[Any, dict[str, Any]]': + from pcapkit.const.pcapng.secrets_type import SecretsType as Enum_SecretsType + from pcapkit.const.pcapng.tls_key_label import TLSKeyLabel as Enum_TLSKeyLabel + from pcapkit.corekit.multidict import OrderedMultiDict + from pcapkit.protocols.misc.pcapng import WireGuardKeyLabel + return { + Enum_SecretsType.TLS_Key_Log: { + 'entries': {Enum_TLSKeyLabel.CLIENT_RANDOM: OrderedMultiDict( + [(bytes(range(0x20, 0x40)), bytes(range(0x40, 0x70)))])}}, + Enum_SecretsType.WireGuard_Key_Log: { + 'entries': OrderedMultiDict( + [(WireGuardKeyLabel.LOCAL_STATIC_PRIVATE_KEY, bytes(range(32)))])}, + Enum_SecretsType.ZigBee_NWK_Key: {'nwk_key': bytes(range(16)), 'pan_id': 0x1234}, + Enum_SecretsType.ZigBee_APS_Key: { + 'aps_key': bytes(range(16)), 'pan_id': 0x1234, 'short_address': 0xABCD}, + } + + +def _pcapng_secrets_build(code: 'Any', kwargs: 'dict[str, Any]') -> 'Any': + from pcapkit.const.pcapng.block_type import BlockType as Enum_BlockType + return _pcapng_block(Enum_BlockType.Decryption_Secrets_Block, + {'secrets_type': code, 'secrets_data': dict(kwargs)}) + + +#: The families this module adds. +FAMILIES = ( + # The PCAP-NG families write no capture -- see the module docstring for + # why the construction API's defaults do not make a reproducible one, and + # what already covers that space instead. + generator.Family('pcapng-block', _pcapng_block_registry, _pcapng_block_overrides, + _pcapng_block_build, _pcapng_parse, _pcapng_block_extract, + _pcapng_block_rebuild, capture=None, envelope=None), + # Both of these reconstruct from the parsed *block* rather than from the + # option or secrets collection alone, because neither collection records + # which block type carried it -- but the option family re-supplies the + # host's own fields on the way, for the reason ``_pcapng_option_rebuild`` + # gives. + generator.Family('pcapng-option', _pcapng_option_registry, _pcapng_option_overrides, + _pcapng_option_build, _pcapng_parse, _pcapng_block_extract, + _pcapng_option_rebuild, capture=None, envelope=None), + generator.Family('pcapng-record', _pcapng_record_registry, _pcapng_record_overrides, + _pcapng_record_build, _pcapng_parse, _pcapng_record_extract, + _pcapng_record_rebuild, capture=None, envelope=None), + generator.Family('pcapng-secrets', _pcapng_secrets_registry, _pcapng_secrets_overrides, + _pcapng_secrets_build, _pcapng_parse, _pcapng_block_extract, + _pcapng_block_rebuild, capture=None, envelope=None), +) diff --git a/examples/generators/option_families/sctp.py b/examples/generators/option_families/sctp.py new file mode 100644 index 0000000000..1cf2a8f0ad --- /dev/null +++ b/examples/generators/option_families/sctp.py @@ -0,0 +1,165 @@ +# -*- coding: utf-8 -*- +"""The ``sctp-chunk``, ``sctp-parameter`` and ``sctp-cause`` families: SCTP chunks, +parameters and error causes. + +A family module of :file:`examples/generators/options.py`, which loads it by +path and binds ``generator`` to itself before this body runs (#2615). + +""" +# Deferring these is the design, as in options.py -- see its module docstring. +# pylint: disable=import-outside-toplevel + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any + +#: :file:`examples/generators/options.py`, which its loader binds here +#: before this body runs. +generator = globals()['generator'] # type: Any + +#: Header fields shared by every constructed SCTP packet. ``chksum`` is pinned +#: rather than left to the CRC32c path, so a mismatch points at the chunk under +#: test and not at a recomputed checksum. +SCTP_BASE = { + 'srcport': 50000, 'dstport': 80, 'vtag': 0x11223344, + 'chksum': b'\x00\x00\x00\x00', +} + + +def _sctp_chunk_registry() -> 'Any': + from pcapkit.protocols.transport.sctp import SCTP + return SCTP.__chunk__ + + +def _sctp_chunk_overrides() -> 'dict[Any, dict[str, Any]]': + from pcapkit.const.sctp.cause_code import CauseCode as Enum_CauseCode + from pcapkit.const.sctp.chunk import Chunk as Enum_Chunk + from pcapkit.const.sctp.parameter import Parameter as Enum_Parameter + + # The three-octet payloads are deliberate: they make each item's length + # 3 (mod 4), so every case runs the PaddingField path instead of landing + # already aligned. + return { + # ``data=b''`` is rejected outright -- RFC 9260 s3.3.1 wants at least + # one octet of user data -- so this override is required, not cosmetic. + Enum_Chunk.Payload_Data: {'data': b'\x01\x02\x03'}, + Enum_Chunk.Initiation: { + 'parameters': [(Enum_Parameter.State_Cookie, {'cookie': b'\x01\x02\x03'})]}, + Enum_Chunk.Initiation_Acknowledgement: { + 'parameters': [(Enum_Parameter.State_Cookie, {'cookie': b'\x01\x02\x03'})]}, + Enum_Chunk.Selective_Acknowledgement: {'gap_blocks': [(1, 2)], 'dup_tsn': [3]}, + # RFC 9260 s3.3.5 and s3.3.6 mandate exactly one Heartbeat Info + # parameter, so the default -- none at all -- is structurally invalid. + Enum_Chunk.Heartbeat_Request: { + 'parameters': [(Enum_Parameter.Heartbeat_Info, {'info': b'\x01\x02\x03'})]}, + Enum_Chunk.Heartbeat_Acknowledgement: { + 'parameters': [(Enum_Parameter.Heartbeat_Info, {'info': b'\x01\x02\x03'})]}, + Enum_Chunk.Abort: { + 'error': [(Enum_CauseCode.User_Initiated_Abort, {'info': b'\x01\x02\x03'})]}, + # ``_make_chunk_error`` documents "one or more error causes"; the + # default gives zero. + Enum_Chunk.Operation_Error: { + 'error': [(Enum_CauseCode.Protocol_Violation, {'info': b'\x01\x02\x03'})]}, + Enum_Chunk.State_Cookie: {'cookie': b'\x01\x02\x03'}, + } + + +def _sctp_chunk_build(code: 'Any', kwargs: 'dict[str, Any]') -> 'Any': + from pcapkit.protocols.transport.sctp import SCTP + return SCTP(chunks=[(code, kwargs)], **SCTP_BASE) + + +def _sctp_parse(octets: 'bytes') -> 'Any': + from pcapkit.protocols.transport.sctp import SCTP + return SCTP(octets, len(octets)) + + +def _sctp_extract(parsed: 'Any') -> 'Any': + return parsed.info.chunks + + +def _sctp_rebuild(chunks: 'Any') -> 'Any': + from pcapkit.protocols.transport.sctp import SCTP + return SCTP(chunks=chunks, **SCTP_BASE) + + +def _sctp_param_registry() -> 'Any': + from pcapkit.protocols.transport.sctp import SCTP + return SCTP.__parameter__ + + +def _sctp_param_overrides() -> 'dict[Any, dict[str, Any]]': + from pcapkit.const.sctp.parameter import Parameter as Enum_Parameter + return { + Enum_Parameter.Heartbeat_Info: {'info': b'\x01\x02\x03'}, + Enum_Parameter.IPv4_Address: {'address': '192.0.2.1'}, + Enum_Parameter.IPv6_Address: {'address': '2001:db8::1'}, + Enum_Parameter.State_Cookie: {'cookie': b'\x01\x02\x03'}, + # A complete unrecognised TLV -- type 0xffff, length 5, one value + # octet -- rather than the empty default, which is not a TLV at all. + Enum_Parameter.Unrecognized_Parameter: {'value': b'\xff\xff\x00\x05\x01'}, + Enum_Parameter.Host_Name_Address: {'name': b'localhost\x00'}, + Enum_Parameter.Supported_Address_Types: { + 'types': [Enum_Parameter.IPv4_Address, Enum_Parameter.IPv6_Address]}, + } + + +def _sctp_param_build(code: 'Any', kwargs: 'dict[str, Any]') -> 'Any': + from pcapkit.const.sctp.chunk import Chunk as Enum_Chunk + from pcapkit.protocols.transport.sctp import SCTP + + # A parameter is not a top-level item: it travels inside a chunk, and + # HEARTBEAT is the smallest chunk that carries an arbitrary one. + return SCTP(chunks=[(Enum_Chunk.Heartbeat_Request, + {'parameters': [(code, kwargs)]})], **SCTP_BASE) + + +def _sctp_cause_registry() -> 'Any': + from pcapkit.protocols.transport.sctp import SCTP + return SCTP.__cause__ + + +def _sctp_cause_overrides() -> 'dict[Any, dict[str, Any]]': + from pcapkit.const.sctp.cause_code import CauseCode as Enum_CauseCode + from pcapkit.const.sctp.parameter import Parameter as Enum_Parameter + return { + Enum_CauseCode.Missing_Mandatory_Parameter: { + 'types': [Enum_Parameter.State_Cookie]}, + # The causes below carry an embedded TLV, and each one's default + # is a zero-length value, i.e. not a TLV. + Enum_CauseCode.Unresolvable_Address: { + 'value': b'\x00\x0b\x00\x0elocalhost\x00'}, + Enum_CauseCode.Unrecognized_Chunk_Type: {'value': b'\xff\x00\x00\x05\x01'}, + Enum_CauseCode.Unrecognized_Parameters: {'value': b'\xff\xff\x00\x05\x01'}, + Enum_CauseCode.Restart_of_an_Association_with_New_Addresses: { + 'value': b'\x00\x05\x00\x08\xc0\x00\x02\x01'}, + Enum_CauseCode.User_Initiated_Abort: {'info': b'\x01\x02\x03'}, + Enum_CauseCode.Protocol_Violation: {'info': b'\x01\x02\x03'}, + } + + +def _sctp_cause_build(code: 'Any', kwargs: 'dict[str, Any]') -> 'Any': + from pcapkit.const.sctp.chunk import Chunk as Enum_Chunk + from pcapkit.protocols.transport.sctp import SCTP + + # An error cause travels inside an ERROR (or ABORT) chunk, under the + # keyword ``error`` rather than ``causes``. + return SCTP(chunks=[(Enum_Chunk.Operation_Error, + {'error': [(code, kwargs)]})], **SCTP_BASE) + + +#: The families this module adds. +FAMILIES = ( + generator.Family('sctp-chunk', _sctp_chunk_registry, _sctp_chunk_overrides, + _sctp_chunk_build, _sctp_parse, _sctp_extract, _sctp_rebuild, + capture='options-transport.pcap', envelope='ipv4', proto=132), + generator.Family('sctp-parameter', _sctp_param_registry, _sctp_param_overrides, + _sctp_param_build, _sctp_parse, _sctp_extract, _sctp_rebuild, + capture='options-transport.pcap', envelope='ipv4', proto=132), + generator.Family('sctp-cause', _sctp_cause_registry, _sctp_cause_overrides, + _sctp_cause_build, _sctp_parse, _sctp_extract, _sctp_rebuild, + capture='options-transport.pcap', envelope='ipv4', proto=132), +) diff --git a/examples/generators/option_families/tcp.py b/examples/generators/option_families/tcp.py new file mode 100644 index 0000000000..886a405009 --- /dev/null +++ b/examples/generators/option_families/tcp.py @@ -0,0 +1,138 @@ +# -*- coding: utf-8 -*- +"""The ``tcp-option`` and ``tcp-mptcp`` families: TCP options, and Multipath TCP subtypes. + +A family module of :file:`examples/generators/options.py`, which loads it by +path and binds ``generator`` to itself before this body runs (#2615). + +""" +# Deferring these is the design, as in options.py -- see its module docstring. +# pylint: disable=import-outside-toplevel + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any + +#: :file:`examples/generators/options.py`, which its loader binds here +#: before this body runs. +generator = globals()['generator'] # type: Any + +#: Header fields shared by every constructed TCP segment. Only ``options`` +#: varies between cases, so a difference in the octets is a difference in the +#: option and nothing else. +#: +#: Every key here has to be spelled the way :meth:`TCP.make +#: ` declares it: an undeclared +#: keyword raises :exc:`~pcapkit.utilities.exceptions.UnsupportedCall` (#617). +#: Without that check, ``make``'s ``**kwargs`` drops it silently, so a key +#: spelled wrong in this table is ignored (GitHub issue #602): +#: +#: * ``'seq'`` for ``seq_no``. The mapping reads as sequence number 1 while +#: every generated frame carries 0, so the captures are not the packets this +#: table describes. +#: * ``'urgent_pointer'`` for ``urgent``. Harmless in effect, since the value +#: asked for and the default are both 0. +#: * ``'ack_flag'`` for ``ack``, where ``ack`` is the *acknowledgement flag* +#: rather than the acknowledgement number, which is ``ack_no``: ``'ack': 0`` +#: sets the flag (to a falsy 0) and ``'ack_flag': False`` sets nothing at +#: all. +#: +#: The mapping is a SYN-only segment with sequence number 1, every other flag +#: clear, and the acknowledgement number, checksum and urgent pointer at zero. +TCP_BASE = { + 'srcport': 50000, 'dstport': 80, 'seq_no': 1, 'ack_no': 0, + 'ns': False, 'cwr': False, 'ece': False, 'urg': False, 'ack': False, + 'psh': False, 'rst': False, 'syn': True, 'fin': False, + 'window': 8192, 'checksum': b'\x00\x00', 'urgent': 0, + 'payload': b'', +} + + +def _tcp_registry() -> 'Any': + from pcapkit.protocols.transport.tcp import TCP + return TCP.__option__ + + +def _tcp_overrides() -> 'dict[Any, dict[str, Any]]': + from pcapkit.const.tcp.option import Option as Enum_Option + return { + # Each of these defaults to an empty variable-length field, which + # is the one shape that never runs the field's packing loop. + Enum_Option.SACK: {'sack': [(1, 2), (3, 4)]}, + Enum_Option.TCP_Alternate_Checksum_Data: {'data': b'\x01\x02'}, + Enum_Option.TCP_Authentication_Option: {'mac': b'\x01\x02\x03\x04'}, + Enum_Option.TCP_Fast_Open_Cookie: {'cookie': b'\x01\x02\x03\x04\x05\x06\x07\x08'}, + } + + +def _tcp_build(code: 'Any', kwargs: 'dict[str, Any]') -> 'Any': + from pcapkit.protocols.transport.tcp import TCP + return TCP(options=[(code, kwargs)], **TCP_BASE) + + +def _tcp_parse(octets: 'bytes') -> 'Any': + from pcapkit.protocols.transport.tcp import TCP + return TCP(octets, len(octets)) + + +def _tcp_extract(parsed: 'Any') -> 'Any': + return parsed.info.options + + +def _tcp_rebuild(options: 'Any') -> 'Any': + from pcapkit.protocols.transport.tcp import TCP + return TCP(options=options, **TCP_BASE) + + +def _mptcp_registry() -> 'Any': + from pcapkit.protocols.transport.tcp import TCP + return TCP.__mp_option__ + + +def _mptcp_overrides() -> 'dict[Any, dict[str, Any]]': + from pcapkit.const.tcp.mp_tcp_option import MPTCPOption as Enum_MPTCPOption + return { + Enum_MPTCPOption.MP_CAPABLE: {'skey': 0x0102030405060708}, + Enum_MPTCPOption.DSS: { + 'ack': 1, 'dsn': 2, 'ssn': 3, 'dl_len': 4, 'checksum': b'\x00\x00'}, + Enum_MPTCPOption.ADD_ADDR: {'addr_id': 1, 'addr': '192.0.2.1'}, + Enum_MPTCPOption.REMOVE_ADDR: {'addr_id': [1]}, + Enum_MPTCPOption.MP_PRIO: {'addr_id': 1}, + Enum_MPTCPOption.MP_FAIL: {'dsn': 7}, + Enum_MPTCPOption.MP_FASTCLOSE: {'key': 9}, + } + + +def _mptcp_build(code: 'Any', kwargs: 'dict[str, Any]') -> 'Any': + from pcapkit.const.tcp.option import Option as Enum_Option + from pcapkit.protocols.transport.tcp import TCP + + # The subtype travels as an argument of the enclosing Multipath TCP option + # rather than as a registry key of its own, so it is spliced in here rather + # than repeated in every entry of the table above. + args = dict(kwargs) + args['subtype'] = code + return TCP(options=[(Enum_Option.Multipath_TCP, args)], **TCP_BASE) + + +#: The families this module adds. +FAMILIES = ( + generator.Family('tcp-option', _tcp_registry, _tcp_overrides, + _tcp_build, _tcp_parse, _tcp_extract, _tcp_rebuild, + capture='options-tcp.pcap', envelope='ipv4', proto=6), + generator.Family('tcp-mptcp', _mptcp_registry, _mptcp_overrides, + _mptcp_build, _tcp_parse, _tcp_extract, _tcp_rebuild, + capture='options-tcp.pcap', envelope='ipv4', proto=6), +) + +#: Codes of :data:`FAMILIES` left out of the case list, with the reason; see +#: :data:`examples.generators.options.SKIP`. ``_make_tcp_options`` emits +#: ``EOOL`` and ``NOP`` as given, so the TCP padding cases are distinct +#: headers and both round-trip. +SKIP = { + # Multipath TCP is the envelope for the whole tcp-mptcp family, so a case + # here would silently duplicate whichever subtype it defaulted to. + ('tcp-option', 'Multipath_TCP'): 'envelope for the tcp-mptcp family', +} diff --git a/examples/generators/option_families/tls.py b/examples/generators/option_families/tls.py new file mode 100644 index 0000000000..aeabe1dfb1 --- /dev/null +++ b/examples/generators/option_families/tls.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- +"""The ``tls-extension`` family: TLS handshake extensions. + +Each case is one extension in a ``ClientHello``, built from keyword arguments +by the extension's ``_make_ext_*``. No capture is written: the ``tcp`` +envelope is addressed to port 80, which dispatches to HTTP rather than TLS. + +A family module of :file:`examples/generators/options.py`, which loads it by +path and binds ``generator`` to itself before this body runs (#2615). + +""" +# Deferring these is the design, as in options.py -- see its module docstring. +# pylint: disable=import-outside-toplevel + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any + +#: :file:`examples/generators/options.py`, which its loader binds here +#: before this body runs. +generator = globals()['generator'] # type: Any + + +def _tls_extension_registry() -> 'Any': + from pcapkit.protocols.application.tls import TLS + return TLS.__extension__ + + +def _tls_extension_overrides() -> 'dict[Any, dict[str, Any]]': + from pcapkit.const.tls.extension_type import ExtensionType as Enum_ExtensionType + from pcapkit.protocols.data.application.tls import KeyShareEntry, ServerName + return { + # Each of these defaults to an empty list. + Enum_ExtensionType.server_name: { + 'server_name_list': (ServerName(name_type=0, name=b'example.com'),)}, + Enum_ExtensionType.supported_groups: {'named_group_list': (0x001D, 0x0017)}, + Enum_ExtensionType.signature_algorithms: {'supported_signature_algorithms': (0x0403, 0x0804)}, + Enum_ExtensionType.application_layer_protocol_negotiation: { + 'protocol_name_list': (b'h2', b'http/1.1')}, + Enum_ExtensionType.supported_versions: {'versions': (0x0304, 0x0303)}, + Enum_ExtensionType.key_share: { + 'client_shares': (KeyShareEntry(group=0x001D, key_exchange=bytes(range(32))),)}, + } + + +def _tls_client_hello(extensions: 'Any') -> 'Any': + """A TLS record carrying one ``ClientHello`` with ``extensions``.""" + from pcapkit.const.tls.content_type import ContentType as Enum_ContentType + from pcapkit.const.tls.handshake_type import HandshakeType as Enum_HandshakeType + from pcapkit.protocols.application.tls import TLS + from pcapkit.protocols.data.application.tls import ClientHello, Handshake + + hello = ClientHello(version=0x0303, random=bytes(32), session_id=b'', cookie=None, + cipher_suites=(0x1301,), compression_methods=(0,), extensions=extensions) + return TLS(content_type=Enum_ContentType.handshake, version=0x0303, + content=(Handshake(msg_type=Enum_HandshakeType.client_hello, length=0, body=hello),)) + + +def _tls_extension_build(code: 'Any', kwargs: 'dict[str, Any]') -> 'Any': + return _tls_client_hello(((code, dict(kwargs)),)) + + +def _tls_parse(octets: 'bytes') -> 'Any': + from pcapkit.protocols.application.tls import TLS + return TLS(octets, len(octets)) + + +def _tls_extension_extract(parsed: 'Any') -> 'Any': + return parsed.info.content[0].body.extensions + + +#: The families this module adds. +FAMILIES = ( + # No capture -- see the comment at the top of this module. + generator.Family('tls-extension', _tls_extension_registry, _tls_extension_overrides, + _tls_extension_build, _tls_parse, _tls_extension_extract, + _tls_client_hello, capture=None, envelope=None), +) diff --git a/examples/generators/options.py b/examples/generators/options.py index b3848270fd..1a05992f27 100644 --- a/examples/generators/options.py +++ b/examples/generators/options.py @@ -43,9 +43,11 @@ :func:`cases` walks the dispatch registries -- ``TCP.__option__``, ``SCTP.__chunk__``, ``MH.__message__`` and the rest -- so a code registered tomorrow appears here tomorrow, and the unit test fails until somebody gives it -a case. What the tables below hold is only the *arguments*, which cannot be -derived: each option means something different, so each needs its own value. -Most need nothing at all, since nearly every ``_make_*`` in the tree is fully +a case. The families live one protocol per module under +:file:`examples/generators/option_families/`, so that an added family is an +added file (#2615), and what their tables hold is only the *arguments*, which +cannot be derived: each option means something different, so each needs its +own value. Most need nothing at all, since nearly every ``_make_*`` in the tree is fully defaulted; an entry exists where the default is degenerate (a zero-length variable field), machine-dependent, or outright rejected. @@ -118,6 +120,7 @@ import collections import datetime +import importlib.util import math import os import pathlib @@ -127,6 +130,7 @@ from typing import TYPE_CHECKING, NamedTuple if TYPE_CHECKING: + from types import ModuleType from typing import Any, Callable, Iterator, Optional __all__ = ['generate', 'cases', 'roundtrip', 'outcomes', 'Case', 'Outcome', 'FAMILIES'] @@ -410,1463 +414,67 @@ def _named_registry(owner: 'type', enum: 'Any', prefix: 'str', ############################################################################### -# TCP -- options, and Multipath TCP subtypes -############################################################################### - -#: Header fields shared by every constructed TCP segment. Only ``options`` -#: varies between cases, so a difference in the octets is a difference in the -#: option and nothing else. -#: -#: Every key here has to be spelled the way :meth:`TCP.make -#: ` declares it: an undeclared -#: keyword raises :exc:`~pcapkit.utilities.exceptions.UnsupportedCall` (#617). -#: Without that check, ``make``'s ``**kwargs`` drops it silently, so a key -#: spelled wrong in this table is ignored (GitHub issue #602): -#: -#: * ``'seq'`` for ``seq_no``. The mapping reads as sequence number 1 while -#: every generated frame carries 0, so the captures are not the packets this -#: table describes. -#: * ``'urgent_pointer'`` for ``urgent``. Harmless in effect, since the value -#: asked for and the default are both 0. -#: * ``'ack_flag'`` for ``ack``, where ``ack`` is the *acknowledgement flag* -#: rather than the acknowledgement number, which is ``ack_no``: ``'ack': 0`` -#: sets the flag (to a falsy 0) and ``'ack_flag': False`` sets nothing at -#: all. -#: -#: The mapping is a SYN-only segment with sequence number 1, every other flag -#: clear, and the acknowledgement number, checksum and urgent pointer at zero. -TCP_BASE = { - 'srcport': 50000, 'dstport': 80, 'seq_no': 1, 'ack_no': 0, - 'ns': False, 'cwr': False, 'ece': False, 'urg': False, 'ack': False, - 'psh': False, 'rst': False, 'syn': True, 'fin': False, - 'window': 8192, 'checksum': b'\x00\x00', 'urgent': 0, - 'payload': b'', -} - - -def _tcp_registry() -> 'Any': - from pcapkit.protocols.transport.tcp import TCP - return TCP.__option__ - - -def _tcp_overrides() -> 'dict[Any, dict[str, Any]]': - from pcapkit.const.tcp.option import Option as Enum_Option - return { - # Each of these defaults to an empty variable-length field, which - # is the one shape that never runs the field's packing loop. - Enum_Option.SACK: {'sack': [(1, 2), (3, 4)]}, - Enum_Option.TCP_Alternate_Checksum_Data: {'data': b'\x01\x02'}, - Enum_Option.TCP_Authentication_Option: {'mac': b'\x01\x02\x03\x04'}, - Enum_Option.TCP_Fast_Open_Cookie: {'cookie': b'\x01\x02\x03\x04\x05\x06\x07\x08'}, - } - - -def _tcp_build(code: 'Any', kwargs: 'dict[str, Any]') -> 'Any': - from pcapkit.protocols.transport.tcp import TCP - return TCP(options=[(code, kwargs)], **TCP_BASE) - - -def _tcp_parse(octets: 'bytes') -> 'Any': - from pcapkit.protocols.transport.tcp import TCP - return TCP(octets, len(octets)) - - -def _tcp_extract(parsed: 'Any') -> 'Any': - return parsed.info.options - - -def _tcp_rebuild(options: 'Any') -> 'Any': - from pcapkit.protocols.transport.tcp import TCP - return TCP(options=options, **TCP_BASE) - - -def _mptcp_registry() -> 'Any': - from pcapkit.protocols.transport.tcp import TCP - return TCP.__mp_option__ - - -def _mptcp_overrides() -> 'dict[Any, dict[str, Any]]': - from pcapkit.const.tcp.mp_tcp_option import MPTCPOption as Enum_MPTCPOption - return { - Enum_MPTCPOption.MP_CAPABLE: {'skey': 0x0102030405060708}, - Enum_MPTCPOption.DSS: { - 'ack': 1, 'dsn': 2, 'ssn': 3, 'dl_len': 4, 'checksum': b'\x00\x00'}, - Enum_MPTCPOption.ADD_ADDR: {'addr_id': 1, 'addr': '192.0.2.1'}, - Enum_MPTCPOption.REMOVE_ADDR: {'addr_id': [1]}, - Enum_MPTCPOption.MP_PRIO: {'addr_id': 1}, - Enum_MPTCPOption.MP_FAIL: {'dsn': 7}, - Enum_MPTCPOption.MP_FASTCLOSE: {'key': 9}, - } - - -def _mptcp_build(code: 'Any', kwargs: 'dict[str, Any]') -> 'Any': - from pcapkit.const.tcp.option import Option as Enum_Option - from pcapkit.protocols.transport.tcp import TCP - - # The subtype travels as an argument of the enclosing Multipath TCP option - # rather than as a registry key of its own, so it is spliced in here rather - # than repeated in every entry of the table above. - args = dict(kwargs) - args['subtype'] = code - return TCP(options=[(Enum_Option.Multipath_TCP, args)], **TCP_BASE) - - -############################################################################### -# IPv4 -- options -############################################################################### - -#: Header fields shared by every constructed IPv4 packet. -IPV4_BASE = {'protocol': 6, 'src': '192.0.2.1', 'dst': '198.51.100.1', 'payload': b''} - - -def _ipv4_registry() -> 'Any': - from pcapkit.const.ipv4.option_number import OptionNumber - from pcapkit.protocols.internet.ipv4 import IPv4 - return _named_registry(IPv4, OptionNumber, '_make_opt_', '__option__') - - -def _ipv4_overrides() -> 'dict[Any, dict[str, Any]]': - from pcapkit.const.ipv4.option_number import OptionNumber - from pcapkit.const.ipv4.protection_authority import ProtectionAuthority - return { - # Two authorities rather than one because two bits set in the bitmap say - # more than one does, not because one is unrepresentable. Without - # #537's fix, a single ``GENSER`` (value 0) makes ``_make_opt_sec`` size - # a zero-octet bitmap and then index into it for a bare ``IndexError``. - # So this is a coverage choice and routes around nothing. - OptionNumber.SEC: {'authorities': [ProtectionAuthority.GENSER, - ProtectionAuthority.NSA]}, - # ``counts=10``, the default, needs 43 option octets, which overflows - # the 4-bit ``ihl``. Nine is the largest an IPv4 header can carry. - OptionNumber.LSR: {'counts': 9}, - OptionNumber.RR: {'counts': 9}, - OptionNumber.SSR: {'counts': 9}, - # The default builds an option with no timestamp in it, which is - # degenerate; one is recorded. - OptionNumber.TS: {'counts': 1, 'timestamp': [1]}, - OptionNumber.E_SEC: {'info': b'\x00'}, - } - - -def _ipv4_build(code: 'Any', kwargs: 'dict[str, Any]') -> 'Any': - from pcapkit.protocols.internet.ipv4 import IPv4 - return IPv4(options=[(code, kwargs)], **IPV4_BASE) - - -def _ipv4_parse(octets: 'bytes') -> 'Any': - from pcapkit.protocols.internet.ipv4 import IPv4 - return IPv4(octets, len(octets)) - - -def _ipv4_extract(parsed: 'Any') -> 'Any': - from pcapkit.corekit.multidict import OrderedMultiDict - - # ``_make_ipv4_options`` drops EOOL and NOP from a fresh option list as - # padding, so the header a case builds from either has no options at all and ``info`` carries no - # ``options`` attribute. An empty collection is the honest reading of that, - # and it keeps the two padding codes from being reported as parse failures. - return getattr(parsed.info, 'options', OrderedMultiDict()) - - -def _ipv4_rebuild(options: 'Any') -> 'Any': - from pcapkit.protocols.internet.ipv4 import IPv4 - return IPv4(options=options, **IPV4_BASE) - - -############################################################################### -# IPv6 extension headers -- HOPOPT, IPv6-Opts, IPv6-Route -############################################################################### - - -def _hopopt_registry() -> 'Any': - from pcapkit.protocols.internet.hopopt import HOPOPT - return HOPOPT.__option__ - - -def _ipv6_opts_registry() -> 'Any': - from pcapkit.protocols.internet.ipv6_opts import IPv6_Opts - return IPv6_Opts.__option__ - - -def _ipv6_option_overrides() -> 'dict[Any, dict[str, Any]]': - from pcapkit.const.ipv6.option import Option as Enum_Option - - # HOPOPT and IPv6-Opts key on the same enumeration and behave identically - # on every code, so one table serves both; ``_hopopt_overrides`` adds the - # one value that matters only in a Hop-by-Hop Options header. - return { - # The nonce is sized to its bit length rounded up to whole octets, at - # least one, so the default ``nonce=0`` is a one-octet zero. A - # three-octet value exercises a multi-octet width; every width from 1 - # to 16 octets builds and round-trips. - Enum_Option.ILNP_Nonce: {'nonce': 0xFFFFFF}, - Enum_Option.Line_Identification_Option: {'id': b'line-1'}, - } - - -def _hopopt_overrides() -> 'dict[Any, dict[str, Any]]': - from pcapkit.const.ipv6.option import Option as Enum_Option - - # A Jumbo Payload option in the Hop-by-Hop Options header after an IPv6 - # header makes the packet a jumbogram, and one whose Jumbo Payload Length is - # not above 65,535 is an error (RFC 2675 section 3) the default engine - # rejects (#1687). So the option carries a legal length, and ``_frame`` - # builds the jumbogram around it. In IPv6-Opts the option is just an option. - return {**_ipv6_option_overrides(), Enum_Option.Jumbo_Payload: {'len': JUMBO_LEN}} - - -def _hopopt_base() -> 'dict[str, Any]': - from pcapkit.const.reg.transtype import TransType - return {'next': TransType.UDP, 'payload': b''} - - -def _hopopt_build(code: 'Any', kwargs: 'dict[str, Any]') -> 'Any': - from pcapkit.protocols.internet.hopopt import HOPOPT - return HOPOPT(options=[(code, kwargs)], **_hopopt_base()) - - -def _hopopt_parse(octets: 'bytes') -> 'Any': - from pcapkit.protocols.internet.hopopt import HOPOPT - return HOPOPT(octets, len(octets), extension=True) - - -def _ipv6_extract(parsed: 'Any') -> 'Any': - from pcapkit.corekit.multidict import OrderedMultiDict - return getattr(parsed.info, 'options', OrderedMultiDict()) - - -def _hopopt_rebuild(options: 'Any') -> 'Any': - from pcapkit.protocols.internet.hopopt import HOPOPT - return HOPOPT(options=options, **_hopopt_base()) - - -def _ipv6_opts_build(code: 'Any', kwargs: 'dict[str, Any]') -> 'Any': - from pcapkit.protocols.internet.ipv6_opts import IPv6_Opts - return IPv6_Opts(options=[(code, kwargs)], **_hopopt_base()) - - -def _ipv6_opts_parse(octets: 'bytes') -> 'Any': - from pcapkit.protocols.internet.ipv6_opts import IPv6_Opts - return IPv6_Opts(octets, len(octets), extension=True) - - -def _ipv6_opts_rebuild(options: 'Any') -> 'Any': - from pcapkit.protocols.internet.ipv6_opts import IPv6_Opts - return IPv6_Opts(options=options, **_hopopt_base()) - - -def _ipv6_route_registry() -> 'Any': - from pcapkit.protocols.internet.ipv6_route import IPv6_Route - return IPv6_Route.__routing__ - - -def _ipv6_route_overrides() -> 'dict[Any, dict[str, Any]]': - from pcapkit.const.ipv6.routing import Routing as Enum_Routing - return { - Enum_Routing.Source_Route: {'ip': ['2001:db8::1', '2001:db8::2']}, - Enum_Routing.Type_2_Routing_Header: {'ip': '2001:db8::2'}, - Enum_Routing.RPL_Source_Route_Header: {'ip': ['2001:db8::1', '2001:db8::2']}, - } - - -def _ipv6_route_base() -> 'dict[str, Any]': - from pcapkit.const.reg.transtype import TransType - return {'next': TransType.UDP, 'seg_left': 0, 'payload': b''} - - -def _ipv6_route_build(code: 'Any', kwargs: 'dict[str, Any]') -> 'Any': - from pcapkit.protocols.internet.ipv6_route import IPv6_Route - return IPv6_Route(type=code, data=dict(kwargs), **_ipv6_route_base()) - - -def _ipv6_route_parse(octets: 'bytes') -> 'Any': - from pcapkit.protocols.internet.ipv6_route import IPv6_Route - return IPv6_Route(octets, len(octets), extension=True) - - -def _ipv6_route_extract(parsed: 'Any') -> 'Any': - # A routing header carries one routing type, not a collection of them, so - # the "collection" handed on is the message itself. - return parsed.info - - -def _ipv6_route_rebuild(info: 'Any') -> 'Any': - from pcapkit.protocols.internet.ipv6_route import IPv6_Route - return IPv6_Route(type=info.type, data=info, next=info.next, - seg_left=info.seg_left, payload=b'') - - -############################################################################### -# SCTP -- chunks, parameters and error causes -############################################################################### - -#: Header fields shared by every constructed SCTP packet. ``chksum`` is pinned -#: rather than left to the CRC32c path, so a mismatch points at the chunk under -#: test and not at a recomputed checksum. -SCTP_BASE = { - 'srcport': 50000, 'dstport': 80, 'vtag': 0x11223344, - 'chksum': b'\x00\x00\x00\x00', -} - - -def _sctp_chunk_registry() -> 'Any': - from pcapkit.protocols.transport.sctp import SCTP - return SCTP.__chunk__ - - -def _sctp_chunk_overrides() -> 'dict[Any, dict[str, Any]]': - from pcapkit.const.sctp.cause_code import CauseCode as Enum_CauseCode - from pcapkit.const.sctp.chunk import Chunk as Enum_Chunk - from pcapkit.const.sctp.parameter import Parameter as Enum_Parameter - - # The three-octet payloads are deliberate: they make each item's length - # 3 (mod 4), so every case runs the PaddingField path instead of landing - # already aligned. - return { - # ``data=b''`` is rejected outright -- RFC 9260 s3.3.1 wants at least - # one octet of user data -- so this override is required, not cosmetic. - Enum_Chunk.Payload_Data: {'data': b'\x01\x02\x03'}, - Enum_Chunk.Initiation: { - 'parameters': [(Enum_Parameter.State_Cookie, {'cookie': b'\x01\x02\x03'})]}, - Enum_Chunk.Initiation_Acknowledgement: { - 'parameters': [(Enum_Parameter.State_Cookie, {'cookie': b'\x01\x02\x03'})]}, - Enum_Chunk.Selective_Acknowledgement: {'gap_blocks': [(1, 2)], 'dup_tsn': [3]}, - # RFC 9260 s3.3.5 and s3.3.6 mandate exactly one Heartbeat Info - # parameter, so the default -- none at all -- is structurally invalid. - Enum_Chunk.Heartbeat_Request: { - 'parameters': [(Enum_Parameter.Heartbeat_Info, {'info': b'\x01\x02\x03'})]}, - Enum_Chunk.Heartbeat_Acknowledgement: { - 'parameters': [(Enum_Parameter.Heartbeat_Info, {'info': b'\x01\x02\x03'})]}, - Enum_Chunk.Abort: { - 'error': [(Enum_CauseCode.User_Initiated_Abort, {'info': b'\x01\x02\x03'})]}, - # ``_make_chunk_error`` documents "one or more error causes"; the - # default gives zero. - Enum_Chunk.Operation_Error: { - 'error': [(Enum_CauseCode.Protocol_Violation, {'info': b'\x01\x02\x03'})]}, - Enum_Chunk.State_Cookie: {'cookie': b'\x01\x02\x03'}, - } - - -def _sctp_chunk_build(code: 'Any', kwargs: 'dict[str, Any]') -> 'Any': - from pcapkit.protocols.transport.sctp import SCTP - return SCTP(chunks=[(code, kwargs)], **SCTP_BASE) - - -def _sctp_parse(octets: 'bytes') -> 'Any': - from pcapkit.protocols.transport.sctp import SCTP - return SCTP(octets, len(octets)) - - -def _sctp_extract(parsed: 'Any') -> 'Any': - return parsed.info.chunks - - -def _sctp_rebuild(chunks: 'Any') -> 'Any': - from pcapkit.protocols.transport.sctp import SCTP - return SCTP(chunks=chunks, **SCTP_BASE) - - -def _sctp_param_registry() -> 'Any': - from pcapkit.protocols.transport.sctp import SCTP - return SCTP.__parameter__ - - -def _sctp_param_overrides() -> 'dict[Any, dict[str, Any]]': - from pcapkit.const.sctp.parameter import Parameter as Enum_Parameter - return { - Enum_Parameter.Heartbeat_Info: {'info': b'\x01\x02\x03'}, - Enum_Parameter.IPv4_Address: {'address': '192.0.2.1'}, - Enum_Parameter.IPv6_Address: {'address': '2001:db8::1'}, - Enum_Parameter.State_Cookie: {'cookie': b'\x01\x02\x03'}, - # A complete unrecognised TLV -- type 0xffff, length 5, one value - # octet -- rather than the empty default, which is not a TLV at all. - Enum_Parameter.Unrecognized_Parameter: {'value': b'\xff\xff\x00\x05\x01'}, - Enum_Parameter.Host_Name_Address: {'name': b'localhost\x00'}, - Enum_Parameter.Supported_Address_Types: { - 'types': [Enum_Parameter.IPv4_Address, Enum_Parameter.IPv6_Address]}, - } - - -def _sctp_param_build(code: 'Any', kwargs: 'dict[str, Any]') -> 'Any': - from pcapkit.const.sctp.chunk import Chunk as Enum_Chunk - from pcapkit.protocols.transport.sctp import SCTP - - # A parameter is not a top-level item: it travels inside a chunk, and - # HEARTBEAT is the smallest chunk that carries an arbitrary one. - return SCTP(chunks=[(Enum_Chunk.Heartbeat_Request, - {'parameters': [(code, kwargs)]})], **SCTP_BASE) - - -def _sctp_cause_registry() -> 'Any': - from pcapkit.protocols.transport.sctp import SCTP - return SCTP.__cause__ - - -def _sctp_cause_overrides() -> 'dict[Any, dict[str, Any]]': - from pcapkit.const.sctp.cause_code import CauseCode as Enum_CauseCode - from pcapkit.const.sctp.parameter import Parameter as Enum_Parameter - return { - Enum_CauseCode.Missing_Mandatory_Parameter: { - 'types': [Enum_Parameter.State_Cookie]}, - # The causes below carry an embedded TLV, and each one's default - # is a zero-length value, i.e. not a TLV. - Enum_CauseCode.Unresolvable_Address: { - 'value': b'\x00\x0b\x00\x0elocalhost\x00'}, - Enum_CauseCode.Unrecognized_Chunk_Type: {'value': b'\xff\x00\x00\x05\x01'}, - Enum_CauseCode.Unrecognized_Parameters: {'value': b'\xff\xff\x00\x05\x01'}, - Enum_CauseCode.Restart_of_an_Association_with_New_Addresses: { - 'value': b'\x00\x05\x00\x08\xc0\x00\x02\x01'}, - Enum_CauseCode.User_Initiated_Abort: {'info': b'\x01\x02\x03'}, - Enum_CauseCode.Protocol_Violation: {'info': b'\x01\x02\x03'}, - } - - -def _sctp_cause_build(code: 'Any', kwargs: 'dict[str, Any]') -> 'Any': - from pcapkit.const.sctp.chunk import Chunk as Enum_Chunk - from pcapkit.protocols.transport.sctp import SCTP - - # An error cause travels inside an ERROR (or ABORT) chunk, under the - # keyword ``error`` rather than ``causes``. - return SCTP(chunks=[(Enum_Chunk.Operation_Error, - {'error': [(code, kwargs)]})], **SCTP_BASE) - - -############################################################################### -# Mobility Header -- messages, options and extensions -############################################################################### - -#: Header fields shared by every constructed Mobility Header. ``next`` is -#: IPv6-NoNxt, so nothing downstream has to be constructed as well. -MH_BASE = {'next': 59, 'chksum': b'\x00\x00', 'payload': b''} - - -def _mh_message_registry() -> 'Any': - from pcapkit.protocols.internet.mh import MH - return MH.__message__ - - -def _mh_message_overrides() -> 'dict[Any, dict[str, Any]]': - return {} - - -def _mh_message_build(code: 'Any', kwargs: 'dict[str, Any]') -> 'Any': - from pcapkit.protocols.internet.mh import MH - - # ``data`` has to be a mapping for ``make`` to dispatch on the message - # type; a Data object takes the other branch, which is what rebuild uses. - return MH(type=code, data=dict(kwargs), **MH_BASE) - - -def _mh_parse(octets: 'bytes') -> 'Any': - from pcapkit.protocols.internet.mh import MH - return MH(octets, len(octets), extension=True) - - -def _mh_message_extract(parsed: 'Any') -> 'Any': - return parsed.info - - -def _mh_message_rebuild(message: 'Any') -> 'Any': - from pcapkit.protocols.internet.mh import MH - return MH(type=message.type, data=message, **MH_BASE) - - -def _mh_option_registry() -> 'Any': - from pcapkit.protocols.internet.mh import MH - return MH.__option__ - - -def _mh_option_overrides() -> 'dict[Any, dict[str, Any]]': - import ipaddress - - from pcapkit.const.mh.ani_suboption import ANISuboption as Enum_ANISuboption - from pcapkit.const.mh.lma_mag_suboption import \ - LMAControlledMAGSuboption as Enum_LMAControlledMAGSuboption - from pcapkit.const.mh.option import Option as Enum_Option - return { - # ``{}`` gives a valid zero-length PadN [RFC 6275 section 6.2.3]; a - # non-zero length exercises the padding octets as well. - Enum_Option.PadN: {'length': 4}, - # The authenticator has to be a multiple of 8 octets. - Enum_Option.Authorization_Data: {'data': b'\xa5' * 8}, - Enum_Option.Mobility_Header_Link_Layer_Address_option: { - 'address': b'\x00\x11\x22\x33\x44\x55'}, - # ``(len + 6) % 4`` has to be 0. - Enum_Option.AUTH_OPTION_TYPE: {'data': b'\xa5\xa5'}, - # Without this ``_make_opt_mesg_id`` reads the clock, and the capture - # would differ on every regeneration. - Enum_Option.MESG_ID_OPTION_TYPE: {'interval': FIXED_TIME}, - Enum_Option.Signature: {'signature': b'\xaa' * 4}, - Enum_Option.Permanent_Home_Keygen_Token: {'token': b'\xbb' * 8}, - Enum_Option.Experimental_Mobility_Option: {'data': b'\xcc' * 4}, - Enum_Option.Binding_Authorization_Data_for_FMIPv6: {'data': b'\xb5'}, - # The entries below are not workarounds for defects: each option has a - # minimum content its RFC requires, and the no-argument default is not a - # well-formed instance of it. The constructors refuse it, correctly, so - # the arguments here are what make the code reachable at all. - # - # ``Length`` of 0 "is not allowed" and the identifier is 1-255 octets - # [RFC 5149 section 3]. ``'ims'`` is that RFC's own example. - Enum_Option.Service_Selection_Mobility_Option: {'identifier': 'ims'}, - # "Both the 'K' and 'N' flags cannot be set or unset simultaneously" - # [RFC 6463 section 4.2], so exactly one address is present and the - # option is 18 octets long for IPv6 or 6 for IPv4. With neither given - # the option's own length is undetermined. - Enum_Option.Redirect_Mobility_Option: { - 'ipv6': ipaddress.IPv6Address('2001:db8::1')}, - # The option "MUST contain at least one ANI sub-option" - # [RFC 6757 section 3]; it is a pure container and carries nothing else. - Enum_Option.Access_Network_Identifier: { - 'suboptions': [(Enum_ANISuboption.Network_Identifier, - {'net_name': b'wifi', - 'ap_name': b'\x00\x11\x22\x33\x44\x55'})]}, - # Likewise at least one LCMP sub-option [RFC 8127 section 3]. - Enum_Option.LMA_Controlled_MAG_Parameters: { - 'suboptions': [(Enum_LMAControlledMAGSuboption.Heartbeat_Control, - {'interval': 60, 'retransmission_delay': 3, - 'max_retransmissions': 5})]}, - } - - -def _mh_carrier() -> 'Any': - from pcapkit.const.mh.packet import Packet as Enum_Packet - - # Binding Refresh Request is the smallest message that carries an - # arbitrary option list: two reserved octets and then the options. - return Enum_Packet.Binding_Refresh_Request - - -def _mh_option_build(code: 'Any', kwargs: 'dict[str, Any]') -> 'Any': - from pcapkit.protocols.internet.mh import MH - return MH(type=_mh_carrier(), data={'options': [(code, dict(kwargs))]}, **MH_BASE) - - -def _mh_option_extract(parsed: 'Any') -> 'Any': - from pcapkit.corekit.multidict import OrderedMultiDict - return getattr(parsed.info, 'options', OrderedMultiDict()) - - -def _mh_option_rebuild(options: 'Any') -> 'Any': - from pcapkit.protocols.internet.mh import MH - return MH(type=_mh_carrier(), data={'options': options}, **MH_BASE) - - -def _mh_extension_registry() -> 'Any': - from pcapkit.protocols.internet.mh import MH - return MH.__extension__ - - -def _mh_extension_overrides() -> 'dict[Any, dict[str, Any]]': - from pcapkit.const.mh.cga_extension import CGAExtension as Enum_CGAExtension - return {Enum_CGAExtension.Multi_Prefix: {'prefixes': [0x20010db800000001]}} - - -def _mh_extension_build(code: 'Any', kwargs: 'dict[str, Any]') -> 'Any': - from pcapkit.const.mh.cga_type import CGAType as Enum_CGAType - from pcapkit.const.mh.option import Option as Enum_Option - from pcapkit.protocols.data.internet.mh import CGAParameter as Data_CGAParameter - from pcapkit.protocols.internet.mh import MH - - # A CGA extension has exactly one carrier: the ``extensions`` list of a CGA - # parameter, inside a CGA Parameters option. It has to be a *Data* model - # rather than a schema, because ``_make_opt_cga_param`` keeps only the - # length of the extensions it builds for a schema and throws the schemas - # themselves away. - # The data model annotates ``modifier`` as a CGAType and ``extensions`` as an - # already-built OrderedMultiDict, but the constructor this feeds takes the - # tuple form and resolves both -- so the arguments go through an untyped - # mapping rather than being spelled as keywords a checker would reject. - fields = { - 'modifier': Enum_CGAType.Tag_086F_CA5E_10B2_00C9_9C8C_E001_6427_7C08, - 'prefix': 0x20010db800000000, - 'collision_count': 0, - 'public_key': b'\x30\x02\xaa\xbb', - 'extensions': [(code, dict(kwargs))], - } # type: dict[str, Any] - parameter = Data_CGAParameter(**fields) - return MH(type=_mh_carrier(), - data={'options': [(Enum_Option.CGA_Parameters, - {'parameters': [parameter]})]}, - **MH_BASE) - - -############################################################################### -# HIP -- parameters -############################################################################### - -#: Header fields shared by every constructed HIP packet. -HIP_BASE = { - 'next': 6, 'packet': 1, 'version': 2, 'checksum': b'\x00\x00', - 'controls_anonymous': False, 'shit': 0, 'rhit': 0, 'payload': b'', -} - -#: Codes tied to one HIP version, and the version their constructor demands: -#: ``R1_Counter`` (128) and ``HIP_TRANSFORM`` (577) are :rfc:`5201` only. -HIP_VERSION = {129: 2, 128: 1, 577: 1} - -#: How many copies of the parameter under test go in one packet. -#: -#: One (#689), because a pair hides the length defects a single parameter -#: exposes. ``HIP.make`` computes the header's ``len`` field as -#: ``total_length // 8 + 4``, which is lossless only when the parameter octets -#: are a multiple of eight, and each parameter is padded to :rfc:`7401` -#: Section 5.2.1's ``Total Length = 11 + Length - (Length + 3) % 8`` so that it -#: is one on its own (#651). A record that is ``4 (mod 8)`` -- from padding that -#: aligns only the contents and ignores the four-octet type-and-length header -#: (#651), or from a packed length that disagrees with the declared ``Length`` -#: (#672, #679) -- makes the floor division drop four octets, and -#: ``_read_hip_param``, which compares the recovered length exactly, rejects the -#: packet. Two such records sum to a multiple of eight, so a pair makes the -#: arithmetic come out exact and the defect invisible. -#: -#: ``HIP_TRANSFORM`` is HIPv1-only -- ``_read_param_hip_transform`` raises for -#: any other version -- and is built at version 1 through :data:`HIP_VERSION`. -#: ``R1_Counter`` (128) parses through ``R1CounterParameter``, which the schema -#: registry holds for both counter codes (#690). Every HIP code round-trips at -#: one copy, so :data:`tests.protocols.test_option_roundtrip_unit.EXPECTED_FAILURES` -#: has no HIP entries. -#: -#: The single-parameter case is asserted directly, and positively, by -#: ``test_a_hip_packet_carrying_one_parameter_round_trips`` in -#: :mod:`tests.protocols.test_option_roundtrip_unit`, which is also where the -#: two-copy shape is exercised, since this constant does not produce it. -HIP_COPIES = 1 - - -def _hip_registry() -> 'Any': - from pcapkit.const.hip.parameter import Parameter - from pcapkit.protocols.internet.hip import HIP - return _named_registry(HIP, Parameter, '_make_param_', '__parameter__') - - -def _hip_overrides() -> 'dict[Any, dict[str, Any]]': - from pcapkit.const.hip.ecdsa_curve import ECDSACurve - from pcapkit.const.hip.parameter import Parameter - return { - # ``lifetime=0`` reaches ``math.log2(0)``. - Parameter.PUZZLE: {'lifetime': 1}, - Parameter.SOLUTION: {'lifetime': 1}, - # ``hi_curve=None`` falls through to an explicit raise. - Parameter.HOST_ID: {'hi_curve': ECDSACurve.NIST_P_256}, - # These two are not here because the default fails -- ``counter=0`` - # constructs and parses perfectly well. They are here because a - # zero-valued field cannot discriminate a width defect (#672) from a - # correct width. An RFC-only walk over ``options-internet.pcap``, which - # takes its stride solely from :rfc:`7401` Section 5.2.1's ``Total - # Length = 11 + Length - (Length + 3) % 8``, mis-strides over a counter - # packed at the wrong width; where every octet it mis-reads is zero, its - # zero-padding check passes and it reports no violation. With a non-zero - # counter the fixture cannot conceal a mis-stride here, whether or not - # the width is right. - Parameter.R1_COUNTER: {'counter': 0xaabbccdd}, - Parameter.R1_Counter: {'counter': 0xaabbccdd}, - } - - -def _hip_base(code: 'Any') -> 'dict[str, Any]': - base = dict(HIP_BASE) - base['version'] = HIP_VERSION.get(int(code), 2) - return base - - -def _hip_build(code: 'Any', kwargs: 'dict[str, Any]') -> 'Any': - from pcapkit.protocols.internet.hip import HIP - - # ``extension=True`` is not decoration. ``HIP.alias`` reads ``self._info``, - # which ``__post_init__`` only assigns after ``read`` returns, and the sole - # path that returns before touching it is the ``if extension`` early exit. - # Without it every HIP code fails with ``AttributeError: 'HIP' - # object has no attribute '_info'``. - return HIP(parameters=[(code, kwargs)] * HIP_COPIES, - extension=True, **_hip_base(code)) - - -def _hip_parse(octets: 'bytes') -> 'Any': - from pcapkit.protocols.internet.hip import HIP - return HIP(octets, len(octets), extension=True) - - -def _hip_extract(parsed: 'Any') -> 'Any': - from pcapkit.corekit.multidict import OrderedMultiDict - return getattr(parsed.info, 'parameters', OrderedMultiDict()) - - -def _hip_rebuild(parameters: 'Any') -> 'Any': - from pcapkit.protocols.internet.hip import HIP - - # ``parameters`` is the parsed collection, so the version has to come off - # one of its members rather than from the code being tested. - first = next(iter(parameters), None) - base = dict(HIP_BASE) - if first is not None: - base['version'] = HIP_VERSION.get(int(first), 2) - return HIP(parameters=parameters, extension=True, **base) - - -############################################################################### -# DCCP -- options -############################################################################### - -#: Header fields shared by every constructed DCCP packet: a DCCP-Ack (type 3), -#: whose Acknowledgement Number the Ack Vector and Data Dropped options read -#: against (RFC 4340 sections 11.4 and 11.7). ``checksum`` is pinned, so a -#: mismatch points at the option under test and not at a recomputed checksum. -DCCP_BASE = { - 'srcport': 50000, 'dstport': 5001, 'type': 3, 'x': True, 'seq': 1, 'ack': 1, - 'checksum': b'\x00\x00', 'payload': b'', -} - - -def _dccp_registry() -> 'Any': - from pcapkit.protocols.transport.dccp import DCCP - return DCCP.__option__ - - -def _dccp_overrides() -> 'dict[Any, dict[str, Any]]': - from pcapkit.const.dccp.feature import Feature as Enum_Feature - from pcapkit.const.dccp.option import Option as Enum_Option - - # Each default is an empty or zero field; these give every variable part - # at least one octet, and the Elapsed Time its four-octet form. - vector = [(0, 3), (1, 0), (3, 63)] - return { - Enum_Option.Change_L: {'feature': Enum_Feature.Congestion_Control_ID, 'value': b'\x02\x03'}, - Enum_Option.Confirm_L: {'feature': Enum_Feature.Congestion_Control_ID, 'value': b'\x02\x02\x03'}, - Enum_Option.Change_R: {'feature': Enum_Feature.Send_Ack_Vector, 'value': b'\x01'}, - Enum_Option.Confirm_R: {'feature': Enum_Feature.Send_Ack_Vector, 'value': b'\x01'}, - Enum_Option.Init_Cookie: {'cookie': b'\x01\x02\x03'}, - Enum_Option.NDP_Count: {'count': 0x0102}, - Enum_Option.Ack_Vector_Nonce_0: {'vector': vector}, - Enum_Option.Ack_Vector_Nonce_1: {'vector': vector}, - Enum_Option.Data_Dropped: {'vector': [(None, 5), (3, 2)]}, - Enum_Option.Timestamp: {'timestamp': 0x11223344}, - Enum_Option.Timestamp_Echo: {'timestamp': 0x11223344, 'elapsed': 0x1234}, - Enum_Option.Elapsed_Time: {'elapsed': 0x123456}, - Enum_Option.Data_Checksum: {'checksum': b'\x01\x02\x03\x04'}, - } - - -def _dccp_build(code: 'Any', kwargs: 'dict[str, Any]') -> 'Any': - from pcapkit.protocols.transport.dccp import DCCP - return DCCP(options=[(code, kwargs)], **DCCP_BASE) - - -def _dccp_parse(octets: 'bytes') -> 'Any': - from pcapkit.protocols.transport.dccp import DCCP - return DCCP(octets, len(octets)) - - -def _dccp_extract(parsed: 'Any') -> 'Any': - return parsed.info.options - - -def _dccp_rebuild(options: 'Any') -> 'Any': - from pcapkit.protocols.transport.dccp import DCCP - return DCCP(options=options, **DCCP_BASE) - - -############################################################################### -# ICMP and ICMPv6 -- message types -############################################################################### - -#: The invoking packet an ICMP error message quotes: an IPv4 header and the -#: first eight octets of its UDP datagram [:rfc:`792`]. -ICMPV4_QUOTED = (bytes.fromhex('4500001c00010000401100000a0000010a000002') - + bytes.fromhex('3039003500080000')) - -#: The invoking packet an ICMPv6 error message quotes: an IPv6 header and its -#: UDP header [:rfc:`4443#section-3`]. -ICMPV6_QUOTED = (bytes.fromhex('6000000000081140') + bytes(15) + b'\x01' + bytes(15) + b'\x02' - + bytes.fromhex('3039003500080000')) - -#: Echo arguments shared by both versions. The defaults are all zero and empty, -#: which cannot tell a swapped ``id`` and ``seq``, or a dropped ``data``, apart. -ICMP_ECHO = {'id': 0x1234, 'seq': 0x5678, 'data': b'ping'} - - -def _icmpv4_registry() -> 'Any': - from pcapkit.protocols.internet.icmpv4 import ICMPv4 - return ICMPv4.__type__ - - -def _icmpv4_overrides() -> 'dict[Any, dict[str, Any]]': - from pcapkit.const.icmp.type import Type as Enum_Type - return { - Enum_Type.Echo_Reply: dict(ICMP_ECHO), - Enum_Type.Echo: dict(ICMP_ECHO), - Enum_Type.Destination_Unreachable: {'data': ICMPV4_QUOTED}, - Enum_Type.Time_Exceeded: {'data': ICMPV4_QUOTED}, - # Octet 8 of the quoted header is its TTL. - Enum_Type.Parameter_Problem: {'pointer': 8, 'data': ICMPV4_QUOTED}, - } - - -def _icmpv4_build(code: 'Any', kwargs: 'dict[str, Any]') -> 'Any': - from pcapkit.protocols.internet.icmpv4 import ICMPv4 - - # ``body`` has to be a mapping for ``make`` to dispatch on ``__type__``; - # a ``bytes`` body is parsed against the schema registry instead. - return ICMPv4(type=code, code=0, body=dict(kwargs)) - - -def _icmpv4_parse(octets: 'bytes') -> 'Any': - from pcapkit.protocols.internet.icmpv4 import ICMPv4 - return ICMPv4(octets, len(octets)) - - -def _icmp_extract(parsed: 'Any') -> 'Any': - return parsed.info - - -def _icmpv4_rebuild(message: 'Any') -> 'Any': - from pcapkit.protocols.internet.icmpv4 import ICMPv4 - return ICMPv4(type=message.type, code=message.code, checksum=message.checksum, - body=message) - - -def _icmpv6_registry() -> 'Any': - from pcapkit.protocols.internet.icmpv6 import ICMPv6 - return ICMPv6.__type__ - - -def _icmpv6_overrides() -> 'dict[Any, dict[str, Any]]': - from pcapkit.const.icmpv6.type import Type as Enum_Type - return { - Enum_Type.Destination_Unreachable: {'data': ICMPV6_QUOTED}, - # The IPv6 minimum link MTU [RFC 8200 section 5]. - Enum_Type.Packet_Too_Big: {'mtu': 1280, 'data': ICMPV6_QUOTED}, - Enum_Type.Time_Exceeded: {'data': ICMPV6_QUOTED}, - # Octet 6 of the quoted header is its Next Header. - Enum_Type.Parameter_Problem: {'pointer': 6, 'data': ICMPV6_QUOTED}, - Enum_Type.Echo_Request: dict(ICMP_ECHO), - Enum_Type.Echo_Reply: dict(ICMP_ECHO), - } - - -def _icmpv6_build(code: 'Any', kwargs: 'dict[str, Any]') -> 'Any': - from pcapkit.protocols.internet.icmpv6 import ICMPv6 - - # With no enclosing IPv6 layer to supply the pseudo-header, the checksum is - # left zero, and the rebuild passes the parsed zero back. - return ICMPv6(type=code, code=0, body=dict(kwargs)) - - -def _icmpv6_parse(octets: 'bytes') -> 'Any': - from pcapkit.protocols.internet.icmpv6 import ICMPv6 - return ICMPv6(octets, len(octets)) - - -def _icmpv6_rebuild(message: 'Any') -> 'Any': - from pcapkit.protocols.internet.icmpv6 import ICMPv6 - return ICMPv6(type=message.type, code=message.code, checksum=message.checksum, - body=message) - - -############################################################################### -# HTTP/2 -- frames -############################################################################### - - -def _httpv2_registry() -> 'Any': - from pcapkit.protocols.application.httpv2 import HTTPv2 - return HTTPv2.__frame__ - - -def _httpv2_overrides() -> 'dict[Any, dict[str, Any]]': - from pcapkit.const.http.frame import Frame as Enum_Frame - from pcapkit.const.http.setting import Setting as Enum_Setting - return { - # ``settings=None``, the default, matches none of the accepted forms. - Enum_Frame.SETTINGS: {'settings': [(Enum_Setting.HEADER_TABLE_SIZE, 4096)]}, - Enum_Frame.GOAWAY: {'debug_data': b'\xde\xad'}, - # A server-initiated stream is even and non-zero (RFC 9113 Section 5.1.1). - Enum_Frame.PUSH_PROMISE: {'promised_sid': 2}, - } - - -#: HTTP/2 frame types that apply to the connection as a whole and so are sent -#: on stream 0 (RFC 9113 Sections 6.5, 6.7 and 6.8). Every other frame type is -#: sent on :data:`HTTPV2_STREAM`. -HTTPV2_CONNECTION_FRAMES = (4, 6, 7) # SETTINGS, PING, GOAWAY - -#: Stream identifier of every stream-level HTTP/2 frame: the first -#: client-initiated stream (RFC 9113 Section 5.1.1). ``DATA``, ``HEADERS``, -#: ``PRIORITY``, ``RST_STREAM``, ``PUSH_PROMISE`` and ``CONTINUATION`` are a -#: connection error on stream 0, and ``WINDOW_UPDATE`` may use either. -HTTPV2_STREAM = 1 - - -def _httpv2_build(code: 'Any', kwargs: 'dict[str, Any]') -> 'Any': - from pcapkit.protocols.application.httpv2 import HTTPv2 - - sid = 0 if int(code) in HTTPV2_CONNECTION_FRAMES else HTTPV2_STREAM - return HTTPv2(type=code, sid=sid, frame=dict(kwargs)) - - -def _httpv2_parse(octets: 'bytes') -> 'Any': - from pcapkit.protocols.application.httpv2 import HTTPv2 - return HTTPv2(octets, len(octets)) - - -def _httpv2_extract(parsed: 'Any') -> 'Any': - return parsed.info - - -def _httpv2_rebuild(frame: 'Any') -> 'Any': - from pcapkit.protocols.application.httpv2 import HTTPv2 - - # ``HTTP.make`` ORs ``flags`` into the bits the ``_make_http_*`` constructor - # derives from ``frame``, and that constructor knows only the flags its - # frame type defines. So the raw flags octet is passed on, as - # ``HTTP._make_data`` does, or a bit the type leaves undefined is dropped. - # ``reserved`` is passed on for the same reason: ``make`` defaults it to 0. - return HTTPv2(type=frame.type, sid=frame.sid, reserved=frame.reserved, - flags=frame.flags.__value__ if frame.flags is not None else 0, - frame=frame) - - -############################################################################### -# TLS -- handshake extensions -############################################################################### -# -# Each case is one extension in a ``ClientHello``, built from keyword arguments -# by the extension's ``_make_ext_*``. No capture is written: the ``tcp`` -# envelope is addressed to port 80, which dispatches to HTTP rather than TLS. - - -def _tls_extension_registry() -> 'Any': - from pcapkit.protocols.application.tls import TLS - return TLS.__extension__ - - -def _tls_extension_overrides() -> 'dict[Any, dict[str, Any]]': - from pcapkit.const.tls.extension_type import ExtensionType as Enum_ExtensionType - from pcapkit.protocols.data.application.tls import KeyShareEntry, ServerName - return { - # Each of these defaults to an empty list. - Enum_ExtensionType.server_name: { - 'server_name_list': (ServerName(name_type=0, name=b'example.com'),)}, - Enum_ExtensionType.supported_groups: {'named_group_list': (0x001D, 0x0017)}, - Enum_ExtensionType.signature_algorithms: {'supported_signature_algorithms': (0x0403, 0x0804)}, - Enum_ExtensionType.application_layer_protocol_negotiation: { - 'protocol_name_list': (b'h2', b'http/1.1')}, - Enum_ExtensionType.supported_versions: {'versions': (0x0304, 0x0303)}, - Enum_ExtensionType.key_share: { - 'client_shares': (KeyShareEntry(group=0x001D, key_exchange=bytes(range(32))),)}, - } - - -def _tls_client_hello(extensions: 'Any') -> 'Any': - """A TLS record carrying one ``ClientHello`` with ``extensions``.""" - from pcapkit.const.tls.content_type import ContentType as Enum_ContentType - from pcapkit.const.tls.handshake_type import HandshakeType as Enum_HandshakeType - from pcapkit.protocols.application.tls import TLS - from pcapkit.protocols.data.application.tls import ClientHello, Handshake - - hello = ClientHello(version=0x0303, random=bytes(32), session_id=b'', cookie=None, - cipher_suites=(0x1301,), compression_methods=(0,), extensions=extensions) - return TLS(content_type=Enum_ContentType.handshake, version=0x0303, - content=(Handshake(msg_type=Enum_HandshakeType.client_hello, length=0, body=hello),)) - - -def _tls_extension_build(code: 'Any', kwargs: 'dict[str, Any]') -> 'Any': - return _tls_client_hello(((code, dict(kwargs)),)) - - -def _tls_parse(octets: 'bytes') -> 'Any': - from pcapkit.protocols.application.tls import TLS - return TLS(octets, len(octets)) - - -def _tls_extension_extract(parsed: 'Any') -> 'Any': - return parsed.info.content[0].body.extensions - - -############################################################################### -# PCAP-NG -- blocks, block options, name records and decryption secrets +# The families, and the cycle ############################################################################### -# -# These families are exercised by :func:`roundtrip` but deliberately write -# no capture, and the reason is not that a PCAP-NG capture would be -# uninteresting -- it is that the construction API's defaults do not produce a -# *reproducible* one. ``_make_block_shb`` takes the section's byte order from -# :data:`sys.byteorder` unless given ``byteorder``, and the TLS and WireGuard -# key-log writers stamp the running pcapkit version into the block body. A -# fixture built this way would differ between a little-endian and a big-endian -# host, and the two key-log types would differ between one release and the -# next. -# -# The PCAP-NG option space is not left uncovered by that decision: -# :file:`examples/generators/pcapng.py` already builds ``profile.pcapng``, -# ``test.pcapng`` and ``many_interfaces.pcapng`` over it, hand-rolling every -# octet with :func:`struct.pack` for exactly this reason. What is added here is -# the half that generator cannot reach, since it never calls the construction -# API at all: whether ``_make_block_*`` and ``_make_option_*`` can produce what -# ``_read_block_*`` and ``_read_option_*`` consume. - -#: Fixed PCAP-NG timestamp, the same instant :file:`pcapng.py` uses. Not -#: optional: leaving ``timestamp`` unset reaches ``self._info`` before -#: ``__post_init__`` has assigned it, so the "now" default is unreachable as -#: well as non-deterministic. -PCAPNG_TIME = 1500000000 - -#: A minimal Ethernet frame for the packet-carrying blocks: destination and -#: source address, an unassigned EtherType, and two octets of body. -#: -#: It has to be non-empty. A ``PayloadField`` whose computed length is -#: legitimately zero reads the whole remainder of the block instead of nothing, -#: which swallows the option area -- so an empty packet would silently take the -#: whole ``epb_*`` and ``pack_*`` option space with it. Sixteen octets rather -#: than the bare 14-octet header, so the Ethernet dissector does not under-run. -#: 6 octets of destination, 6 of source, the EtherType, then the body. -PCAPNG_PACKET = bytes.fromhex('02000000000102000000000288b50000') - -#: A systemd journal export entry whose length is a multiple of four, so that -#: ``_make_block_systemd`` has no padding to add to it. -PCAPNG_JOURNAL = (b'__REALTIME_TIMESTAMP=1500000000000000\n_TRANSPORT=journal\n' - b'PRIORITY=6\nMESSAGE=probe.....\n\n') - - -def _pcapng_context() -> 'tuple[Any, Any]': - """A one-section PCAP-NG context with one Ethernet interface in it. - - Every block other than a Section Header Block is read and written relative - to a section, and the packet-carrying blocks resolve their snap length - through the section's interface list, so there has to be one. - - Returns: - The :class:`~pcapkit.foundation.engines.pcapng.Context` and the - interface-description block registered in it. - - """ - from pcapkit.const.pcapng.block_type import BlockType as Enum_BlockType - from pcapkit.const.reg.linktype import LinkType as Enum_LinkType - from pcapkit.foundation.engines.pcapng import Context - from pcapkit.protocols.misc.pcapng import PCAPNG - - section = PCAPNG(num=0, sct=1, ctx=None, - type=Enum_BlockType.Section_Header_Block, block={}) - # ``Protocol.info`` is annotated as the generic data model for the protocol, - # so a checker cannot see that a Section Header Block's is the one ``Context`` - # wants. The engine does exactly this, in ``PCAPNG.run`` of - # pcapkit/foundation/engines/pcapng.py. - section_info = section.info # type: Any - context = Context(section_info) - interface = PCAPNG(num=1, sct=1, ctx=context, - type=Enum_BlockType.Interface_Description_Block, - block={'linktype': Enum_LinkType.ETHERNET, 'snaplen': 0x40000}) - interface_info = interface.info # type: Any - context.interfaces.append(interface_info) - return context, interface_info - - -def _pcapng_block(code: 'Any', block: 'dict[str, Any] | Any') -> 'Any': - """Construct one PCAP-NG block of ``code``. - - A fresh context per call, deliberately. ``PCAPNG.__post_init__`` packs and - then re-reads *on the same instance*, and the two halves share the - per-instance option counter, so state left by one case would change the - next one's result. - - Args: - code: Block type. - block: Block body, as a mapping of constructor arguments or as the - parsed data model. - - Returns: - The constructed block. - - """ - from pcapkit.const.pcapng.block_type import BlockType as Enum_BlockType - from pcapkit.protocols.misc.pcapng import PCAPNG - - # A Section Header Block is what *starts* a section, so it is the one block - # the engine reads with no context at all. - context = None if code == Enum_BlockType.Section_Header_Block else _pcapng_context()[0] - return PCAPNG(num=2, sct=1, ctx=context, type=code, block=block) - -#: Block type of a Section Header Block, as it appears on the wire. The value -#: is byte-order independent by design -- that is what lets a reader work out a -#: section's endianness -- so it can be recognised before the byte order is -#: known, which is exactly what :func:`_pcapng_parse` needs. -PCAPNG_SHB_MAGIC = bytes.fromhex('0a0d0d0a') +#: The directory holding the family modules: one file per protocol, holding +#: that protocol's families and their argument tables, so that a family added +#: in one change and a family added in another touch different files (#2615). +FAMILY_DIR = ROOT / 'examples' / 'generators' / 'option_families' -def _pcapng_parse(octets: 'bytes') -> 'Any': - """Parse one PCAP-NG block back from its own octets. +class _Generator: + """This module as its family modules read it: its globals, looked up live.""" - The block type is read out of the octets rather than passed in, so that - this keeps the one-argument shape every other family's ``parse`` has. It is - the first four octets of any block, and a Section Header Block -- the one - block read with no section context, because it is what *starts* a - section -- is recognisable by :data:`PCAPNG_SHB_MAGIC`. - - Args: - octets: The constructed block. + def __getattr__(self, name: 'str') -> 'Any': + try: + return globals()[name] + except KeyError: + raise AttributeError(f'{__name__!r} has no attribute {name!r}') from None - Returns: - The parsed block. - """ - from pcapkit.protocols.misc.pcapng import PCAPNG - - context = None if octets[:4] == PCAPNG_SHB_MAGIC else _pcapng_context()[0] - return PCAPNG(octets, len(octets), num=2, sct=1, ctx=context) - - -def _pcapng_block_registry() -> 'Any': - from pcapkit.protocols.misc.pcapng import PCAPNG - return PCAPNG.__block__ - - -def _pcapng_block_overrides() -> 'dict[Any, dict[str, Any]]': - from pcapkit.const.pcapng.block_type import BlockType as Enum_BlockType - from pcapkit.const.pcapng.record_type import RecordType as Enum_RecordType - from pcapkit.const.pcapng.secrets_type import SecretsType as Enum_SecretsType - from pcapkit.const.reg.linktype import LinkType as Enum_LinkType - return { - Enum_BlockType.Interface_Description_Block: { - 'linktype': Enum_LinkType.ETHERNET, 'snaplen': 0x40000}, - Enum_BlockType.Enhanced_Packet_Block: { - 'timestamp': PCAPNG_TIME, 'packet_data': PCAPNG_PACKET}, - Enum_BlockType.Simple_Packet_Block: {'packet_data': PCAPNG_PACKET}, - # Without the terminating record the records field consumes the whole - # block and the option area disappears. - Enum_BlockType.Name_Resolution_Block: { - 'records': [(Enum_RecordType.nrb_record_end, {})]}, - Enum_BlockType.Interface_Statistics_Block: {'timestamp': PCAPNG_TIME}, - Enum_BlockType.systemd_Journal_Export_Block: {'entries': PCAPNG_JOURNAL}, - # ZigBee rather than the default TLS key log: the default empty - # ``secrets_data`` does not read back as a key log. - Enum_BlockType.Decryption_Secrets_Block: { - 'secrets_type': Enum_SecretsType.ZigBee_NWK_Key, - 'secrets_data': {'nwk_key': bytes(range(16)), 'pan_id': 0x1234}}, - Enum_BlockType.Custom_Block_that_rewriters_can_copy_into_new_files: { - 'pen': 32473, 'data': b'\x01\x02\x03\x04'}, - Enum_BlockType.Custom_Block_that_rewriters_should_not_copy_into_new_files: { - 'pen': 32473, 'data': b'\x01\x02\x03\x04'}, - Enum_BlockType.Packet_Block: { - 'timestamp': PCAPNG_TIME, 'packet_data': PCAPNG_PACKET}, - } - - -def _pcapng_block_build(code: 'Any', kwargs: 'dict[str, Any]') -> 'Any': - return _pcapng_block(code, dict(kwargs)) - - -def _pcapng_block_extract(parsed: 'Any') -> 'Any': - return parsed.info - - -def _pcapng_block_rebuild(info: 'Any') -> 'Any': - return _pcapng_block(info.type, info) - - -#: Which block hosts each block-option namespace, and that host's own -#: arguments. The namespace is the part of the ``_option_key`` tuple's first -#: element before the first underscore. -PCAPNG_OPTION_HOSTS = { - 'opt': 'Section_Header_Block', - 'if': 'Interface_Description_Block', - 'epb': 'Enhanced_Packet_Block', - 'ns': 'Name_Resolution_Block', - 'isb': 'Interface_Statistics_Block', - 'pack': 'Packet_Block', -} - - -def _pcapng_option_registry() -> 'Any': - from pcapkit.protocols.misc.pcapng import PCAPNG - return PCAPNG.__option__ - - -def _pcapng_option_overrides() -> 'dict[Any, dict[str, Any]]': - from pcapkit.const.pcapng.option_type import OptionType as Enum_OptionType - from pcapkit.protocols.misc.pcapng import _option_key - return { - _option_key(Enum_OptionType.opt_comment): {'comment': 'probe'}, - _option_key(Enum_OptionType.opt_custom_2988): {'pen': 32473, 'data': b'\x01\x02\x03\x04'}, - _option_key(Enum_OptionType.opt_custom_2989): {'pen': 32473, 'data': b'\x01\x02\x03\x04'}, - _option_key(Enum_OptionType.opt_custom_19372): {'pen': 32473, 'data': b'\x01\x02\x03\x04'}, - _option_key(Enum_OptionType.opt_custom_19373): {'pen': 32473, 'data': b'\x01\x02\x03\x04'}, - _option_key(Enum_OptionType.if_name): {'name': 'eth0'}, - _option_key(Enum_OptionType.if_description): {'description': 'probe interface'}, - _option_key(Enum_OptionType.if_IPv4addr): {'interface': '10.0.0.1/255.255.255.0'}, - _option_key(Enum_OptionType.if_IPv6addr): { - 'interface': '2001:db8:85a3:8d3:1319:8a2e:370:7344/64'}, - _option_key(Enum_OptionType.if_MACaddr): {'interface': '02:00:00:00:00:01'}, - _option_key(Enum_OptionType.if_EUIaddr): {'interface': '02:00:00:FF:FE:00:00:01'}, - _option_key(Enum_OptionType.if_speed): {'speed': 1000000000}, - _option_key(Enum_OptionType.if_tsresol): {'resolution': 1000000}, - _option_key(Enum_OptionType.if_tzone): {'tzone': 0}, - _option_key(Enum_OptionType.if_filter): {'expression': b'udp port 67'}, - # ``if_os`` and ``if_hardware`` default to the live platform strings, so - # leaving them unset would make the case machine-dependent. - _option_key(Enum_OptionType.if_os): {'os': 'probe OS'}, - _option_key(Enum_OptionType.if_hardware): {'hardware': 'probe adapter'}, - _option_key(Enum_OptionType.if_fcslen): {'fcs_length': 4}, - _option_key(Enum_OptionType.if_tsoffset): {'offset': 0}, - _option_key(Enum_OptionType.if_txspeed): {'speed': 1000000000}, - _option_key(Enum_OptionType.if_rxspeed): {'speed': 1000000000}, - _option_key(Enum_OptionType.epb_hash): {'hash': b'\x01\x02\x03\x04'}, - _option_key(Enum_OptionType.epb_verdict): {'value': b'\x01'}, - _option_key(Enum_OptionType.ns_dnsname): {'name': 'resolver.example'}, - _option_key(Enum_OptionType.ns_dnsIP4addr): {'ip': '10.0.0.53'}, - # The default is the IPv4 literal ``'8.8.8.8'``, which the v6 field - # rejects outright. - _option_key(Enum_OptionType.ns_dnsIP6addr): {'ip': '2001:db8::35'}, - _option_key(Enum_OptionType.isb_starttime): {'timestamp': PCAPNG_TIME}, - _option_key(Enum_OptionType.isb_endtime): {'timestamp': PCAPNG_TIME}, - _option_key(Enum_OptionType.isb_ifrecv): {'packets': 4}, - _option_key(Enum_OptionType.isb_ifdrop): {'packets': 1}, - _option_key(Enum_OptionType.isb_filteraccept): {'packets': 4}, - _option_key(Enum_OptionType.isb_osdrop): {'packets': 0}, - _option_key(Enum_OptionType.isb_usrdeliv): {'packets': 3}, - _option_key(Enum_OptionType.pack_hash): {'hash': b'\x01\x02\x03\x04'}, - } - - -def _pcapng_option_enum(key: 'Any') -> 'Any': - """The enumeration member behind an ``_option_key`` tuple. - - ``PCAPNG.__option__`` is keyed by ``(name, value)`` tuples because the same - numeric option code means different things in different block types, but - the construction API takes the enumeration member. +def _family_modules() -> 'dict[str, ModuleType]': + """Load every module of :data:`FAMILY_DIR`, in file name order, keyed by stem. - Args: - key: The ``(str, int)`` registry key. + Each is loaded by path, as this module is, with ``generator`` bound before + its body runs, so that it builds its :class:`Family` entries and reaches + the shared helpers here as ``generator.``. Each declares + ``FAMILIES``, the families it adds, and may declare ``SKIP``, its entries + of :data:`SKIP`. Returns: - The matching :class:`~pcapkit.const.pcapng.option_type.OptionType`. + The loaded modules. Raises: - KeyError: If no member matches, which would mean the registry and the - enumeration have diverged. - - """ - from pcapkit.const.pcapng.option_type import OptionType as Enum_OptionType - from pcapkit.protocols.misc.pcapng import _option_key - - # The enumeration is an :mod:`aenum` one, whose metaclass does not advertise - # ``__iter__`` to a type checker even though it iterates perfectly well. - members = Enum_OptionType # type: Any - for member in members: - if _option_key(member) == key: - return member - raise KeyError(f'no OptionType matches registry key {key!r}') - - -def _pcapng_option_host(key: 'Any') -> 'tuple[Any, dict[str, Any]]': - """The block type that carries option ``key``, and its own arguments.""" - from pcapkit.const.pcapng.block_type import BlockType as Enum_BlockType - - namespace = str(key[0]).split('_', 1)[0] - name = PCAPNG_OPTION_HOSTS[namespace] - code = getattr(Enum_BlockType, name) - return code, dict(_pcapng_block_overrides().get(code, {})) - - -def _pcapng_option_build(key: 'Any', kwargs: 'dict[str, Any]') -> 'Any': - code, block = _pcapng_option_host(key) - block['options'] = [(_pcapng_option_enum(key), dict(kwargs))] - return _pcapng_block(code, block) - - -def _pcapng_option_rebuild(info: 'Any') -> 'Any': - """Rebuild an option's host block, taking only the options from the parse. - - The host's own fields come from :func:`_pcapng_block_overrides` rather than - from the parsed block, and that is the point. Reconstructing the whole block - from ``info`` would drop the captured payload of a packet-carrying block -- - ``_make_block_epb`` and its siblings never restore ``packet_data``, and the - data model has nowhere to keep it -- so every ``epb_*`` and ``pack_*`` - option would be reported as a mismatch on account of a defect that has - nothing to do with the option. Re-supplying the host fields isolates the - option, which is what this family is measuring. - - Args: - info: The parsed block. - - Returns: - The reconstructed block. + RuntimeError: If a module cannot be loaded. """ - block = dict(_pcapng_block_overrides().get(info.type, {})) - block['options'] = info.options - return _pcapng_block(info.type, block) - - -def _pcapng_record_registry() -> 'Any': - from pcapkit.protocols.misc.pcapng import PCAPNG - return PCAPNG.__record__ - - -def _pcapng_record_overrides() -> 'dict[Any, dict[str, Any]]': - from pcapkit.const.pcapng.record_type import RecordType as Enum_RecordType - return { - Enum_RecordType.nrb_record_ipv4: {'ip': '10.0.0.1', 'names': ['gateway.example']}, - # The default is the IPv4 literal ``'127.0.0.1'``, which the v6 field - # rejects outright. - Enum_RecordType.nrb_record_ipv6: {'ip': '2001:db8::1', 'names': ['v6.example']}, - } + modules = {} # type: dict[str, ModuleType] + for path in sorted(FAMILY_DIR.glob('*.py')): + spec = importlib.util.spec_from_file_location(f'{__name__}_{path.stem}', path) + if spec is None or spec.loader is None: # pragma: no cover + raise RuntimeError(f'cannot load the option family module {path}') + module = importlib.util.module_from_spec(spec) + setattr(module, 'generator', _Generator()) + spec.loader.exec_module(module) + modules[path.stem] = module + return modules -def _pcapng_record_build(code: 'Any', kwargs: 'dict[str, Any]') -> 'Any': - from pcapkit.const.pcapng.block_type import BlockType as Enum_BlockType - from pcapkit.const.pcapng.record_type import RecordType as Enum_RecordType +#: Each family module, keyed by its file stem. +FAMILY_MODULES = _family_modules() - records = [(code, dict(kwargs))] - if code != Enum_RecordType.nrb_record_end: - # A record list has to be terminated, or the records field eats the - # rest of the block. - records.append((Enum_RecordType.nrb_record_end, {})) - return _pcapng_block(Enum_BlockType.Name_Resolution_Block, {'records': records}) - - -def _pcapng_record_extract(parsed: 'Any') -> 'Any': - return parsed.info.records - - -def _pcapng_record_rebuild(records: 'Any') -> 'Any': - from pcapkit.const.pcapng.block_type import BlockType as Enum_BlockType - return _pcapng_block(Enum_BlockType.Name_Resolution_Block, {'records': records}) - - -def _pcapng_secrets_registry() -> 'Any': - from pcapkit.protocols.misc.pcapng import PCAPNG - return PCAPNG.__secrets__ - - -def _pcapng_secrets_overrides() -> 'dict[Any, dict[str, Any]]': - from pcapkit.const.pcapng.secrets_type import SecretsType as Enum_SecretsType - from pcapkit.const.pcapng.tls_key_label import TLSKeyLabel as Enum_TLSKeyLabel - from pcapkit.corekit.multidict import OrderedMultiDict - from pcapkit.protocols.misc.pcapng import WireGuardKeyLabel - return { - Enum_SecretsType.TLS_Key_Log: { - 'entries': {Enum_TLSKeyLabel.CLIENT_RANDOM: OrderedMultiDict( - [(bytes(range(0x20, 0x40)), bytes(range(0x40, 0x70)))])}}, - Enum_SecretsType.WireGuard_Key_Log: { - 'entries': OrderedMultiDict( - [(WireGuardKeyLabel.LOCAL_STATIC_PRIVATE_KEY, bytes(range(32)))])}, - Enum_SecretsType.ZigBee_NWK_Key: {'nwk_key': bytes(range(16)), 'pan_id': 0x1234}, - Enum_SecretsType.ZigBee_APS_Key: { - 'aps_key': bytes(range(16)), 'pan_id': 0x1234, 'short_address': 0xABCD}, - } - - -def _pcapng_secrets_build(code: 'Any', kwargs: 'dict[str, Any]') -> 'Any': - from pcapkit.const.pcapng.block_type import BlockType as Enum_BlockType - return _pcapng_block(Enum_BlockType.Decryption_Secrets_Block, - {'secrets_type': code, 'secrets_data': dict(kwargs)}) - - -############################################################################### -# The families, and the cycle -############################################################################### - -#: Every family this module exercises, in the order their frames are emitted. -FAMILIES = ( - Family('tcp-option', _tcp_registry, _tcp_overrides, - _tcp_build, _tcp_parse, _tcp_extract, _tcp_rebuild, - capture='options-tcp.pcap', envelope='ipv4', proto=6), - Family('tcp-mptcp', _mptcp_registry, _mptcp_overrides, - _mptcp_build, _tcp_parse, _tcp_extract, _tcp_rebuild, - capture='options-tcp.pcap', envelope='ipv4', proto=6), - Family('ipv4-option', _ipv4_registry, _ipv4_overrides, - _ipv4_build, _ipv4_parse, _ipv4_extract, _ipv4_rebuild, - capture='options-ipv4.pcap', envelope='ethernet-ipv4'), - Family('hopopt-option', _hopopt_registry, _hopopt_overrides, - _hopopt_build, _hopopt_parse, _ipv6_extract, _hopopt_rebuild, - capture='options-ipv6.pcap', envelope='ipv6', proto=0), - Family('ipv6-opts-option', _ipv6_opts_registry, _ipv6_option_overrides, - _ipv6_opts_build, _ipv6_opts_parse, _ipv6_extract, _ipv6_opts_rebuild, - capture='options-ipv6.pcap', envelope='ipv6', proto=60), - Family('ipv6-route-type', _ipv6_route_registry, _ipv6_route_overrides, - _ipv6_route_build, _ipv6_route_parse, _ipv6_route_extract, - _ipv6_route_rebuild, - capture='options-ipv6.pcap', envelope='ipv6', proto=43), - Family('sctp-chunk', _sctp_chunk_registry, _sctp_chunk_overrides, - _sctp_chunk_build, _sctp_parse, _sctp_extract, _sctp_rebuild, - capture='options-transport.pcap', envelope='ipv4', proto=132), - Family('sctp-parameter', _sctp_param_registry, _sctp_param_overrides, - _sctp_param_build, _sctp_parse, _sctp_extract, _sctp_rebuild, - capture='options-transport.pcap', envelope='ipv4', proto=132), - Family('sctp-cause', _sctp_cause_registry, _sctp_cause_overrides, - _sctp_cause_build, _sctp_parse, _sctp_extract, _sctp_rebuild, - capture='options-transport.pcap', envelope='ipv4', proto=132), - Family('httpv2-frame', _httpv2_registry, _httpv2_overrides, - _httpv2_build, _httpv2_parse, _httpv2_extract, _httpv2_rebuild, - capture='options-transport.pcap', envelope='tcp'), - Family('dccp-option', _dccp_registry, _dccp_overrides, - _dccp_build, _dccp_parse, _dccp_extract, _dccp_rebuild, - capture='options-transport.pcap', envelope='ipv4', proto=33), - Family('mh-message', _mh_message_registry, _mh_message_overrides, - _mh_message_build, _mh_parse, _mh_message_extract, _mh_message_rebuild, - capture='options-internet.pcap', envelope='ipv6', proto=135), - Family('mh-option', _mh_option_registry, _mh_option_overrides, - _mh_option_build, _mh_parse, _mh_option_extract, _mh_option_rebuild, - capture='options-internet.pcap', envelope='ipv6', proto=135), - Family('mh-extension', _mh_extension_registry, _mh_extension_overrides, - _mh_extension_build, _mh_parse, _mh_option_extract, _mh_option_rebuild, - capture='options-internet.pcap', envelope='ipv6', proto=135), - Family('hip-parameter', _hip_registry, _hip_overrides, - _hip_build, _hip_parse, _hip_extract, _hip_rebuild, - capture='options-internet.pcap', envelope='ipv6', proto=139), - # NOTE: The ICMP families write no capture; their wire form is pinned against - # RFC 792 and RFC 4443 by test_option_generator_icmp_2560_unit instead. - Family('icmpv4-type', _icmpv4_registry, _icmpv4_overrides, - _icmpv4_build, _icmpv4_parse, _icmp_extract, _icmpv4_rebuild, - capture=None, envelope=None), - Family('icmpv6-type', _icmpv6_registry, _icmpv6_overrides, - _icmpv6_build, _icmpv6_parse, _icmp_extract, _icmpv6_rebuild, - capture=None, envelope=None), - # No capture -- see the section comment above ``_tls_extension_registry``. - Family('tls-extension', _tls_extension_registry, _tls_extension_overrides, - _tls_extension_build, _tls_parse, _tls_extension_extract, - _tls_client_hello, capture=None, envelope=None), - # The PCAP-NG families write no capture -- see the section comment - # above ``PCAPNG_TIME`` for why the construction API's defaults do not - # make a reproducible one, and what already covers that space instead. - Family('pcapng-block', _pcapng_block_registry, _pcapng_block_overrides, - _pcapng_block_build, _pcapng_parse, _pcapng_block_extract, - _pcapng_block_rebuild, capture=None, envelope=None), - # Both of these reconstruct from the parsed *block* rather than from the - # option or secrets collection alone, because neither collection records - # which block type carried it -- but the option family re-supplies the - # host's own fields on the way, for the reason ``_pcapng_option_rebuild`` - # gives. - Family('pcapng-option', _pcapng_option_registry, _pcapng_option_overrides, - _pcapng_option_build, _pcapng_parse, _pcapng_block_extract, - _pcapng_option_rebuild, capture=None, envelope=None), - Family('pcapng-record', _pcapng_record_registry, _pcapng_record_overrides, - _pcapng_record_build, _pcapng_parse, _pcapng_record_extract, - _pcapng_record_rebuild, capture=None, envelope=None), - Family('pcapng-secrets', _pcapng_secrets_registry, _pcapng_secrets_overrides, - _pcapng_secrets_build, _pcapng_parse, _pcapng_block_extract, - _pcapng_block_rebuild, capture=None, envelope=None), -) +#: Every family this module exercises, in the order their frames are emitted: +#: those of each family module, in file name order. +FAMILIES = tuple(family for module in FAMILY_MODULES.values() for family in module.FAMILIES) #: The families keyed by label, for a caller that wants just one. FAMILY_MAP = {family.label: family for family in FAMILIES} -#: Codes deliberately left out of the case list, with the reason. -#: -#: Every padding option has a case except PadN in HOPOPT and IPv6-Opts. Their -#: ``_make_*_options`` keep the padding of parsed options in place, but drop -#: any in a list of fresh options and insert their own to reach 8-octet -#: alignment. A case is built from such a list, so the Pad1 and PadN cases -#: would both emit the same option-less header; Pad1 is kept, PadN skipped. -#: ``_make_tcp_options`` emits ``EOOL`` and ``NOP`` as given, so the TCP -#: padding cases are distinct headers and both round-trip. -SKIP = { - # Multipath TCP is the envelope for the whole tcp-mptcp family, so a case - # here would silently duplicate whichever subtype it defaulted to. - ('tcp-option', 'Multipath_TCP'): 'envelope for the tcp-mptcp family', - ('hopopt-option', 'PadN'): 'padding, indistinguishable from Pad1 here', - ('ipv6-opts-option', 'PadN'): 'padding, indistinguishable from Pad1 here', -} +#: Codes deliberately left out of the case list, keyed by ``(family label, +#: code name)``, with the reason: the ``SKIP`` of each family module, merged. +SKIP = {key: reason for module in FAMILY_MODULES.values() + for key, reason in getattr(module, 'SKIP', {}).items()} def cases(families: 'Optional[tuple[Family, ...]]' = None) -> 'list[Case]': @@ -2134,8 +742,9 @@ def _jumbogram_datagram(octets: 'bytes', jumbo: 'int') -> 'bytes': With a Payload Length of 0 it is the Jumbo Payload Length that says where the packet ends, so the octets after the Hop-by-Hop Options header have to fill it, or the frame is a jumbogram cut short. The header names UDP as its - next header, as :func:`_hopopt_base` has every HOPOPT case do, so they are a - datagram of zeros to the Discard port [:rfc:`863`], with the checksum + next header, as ``_hopopt_base`` in + :file:`examples/generators/option_families/ipv6.py` has every HOPOPT case + do, so they are a datagram of zeros to the Discard port [:rfc:`863`], with the checksum :rfc:`8200#section-8.1` has a receiver verify. Its length fits UDP's Length field, which :rfc:`2675#section-4` therefore leaves as is. The checksum is the one's complement sum of :rfc:`1071`, computed here rather than by diff --git a/tests/const/census/__init__.py b/tests/const/census/__init__.py new file mode 100644 index 0000000000..f521cd531b --- /dev/null +++ b/tests/const/census/__init__.py @@ -0,0 +1,109 @@ +# -*- coding: utf-8 -*- +"""The registries :mod:`pcapkit.const` defines, and the kind of each. C.f. #2615. + +The const sweeps -- :mod:`tests.const.test_const_enum_builtin_parity`, +:mod:`tests.const.test_const_enum_get`, :mod:`tests.const.test_const_enum_lookup` +and :mod:`tests.const.test_const_registry_protocol` -- each discover the +registries by walking the tree, and each holds what it found to this census by +name, through :func:`compare`. So a registry added, dropped or changed in kind +fails every sweep that covers it, and the failure names the registry, where a +pinned count fails on the same changes without naming anything. + +The census is one module per package of :mod:`pcapkit.const`, named after the +package and found by :func:`pkgutil.iter_modules`. Each holds a ``REGISTRIES`` +mapping of ``module:QualName`` to a kind, sorted, one registry per line. A +registry added to a package is one line in that package's module, and an added +package is one added module, so changes to unrelated packages touch disjoint +files rather than the same pins in every sweep. +:mod:`tests.const.test_const_census_2615_unit` holds the modules to that layout. + +The kinds partition the registries as the sweeps split them: ``'flag'`` is an +:class:`aenum.Flag`, ``'str'`` a :class:`str` enumeration that is no flag, and +``'int'`` an :class:`int` one that is no flag. + +Nothing here imports :mod:`pcapkit`, so loading the census leaves nothing behind +for a sweep that purges and re-imports it. + +""" +from __future__ import annotations + +import collections +import importlib +import pkgutil +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from types import ModuleType + from typing import Iterable + +__all__ = ['KINDS', 'MANIFESTS', 'REGISTRIES', 'AGREES', 'ADVICE', + 'names', 'modules', 'paths', 'compare'] + +#: The kinds a registry is recorded as. +KINDS = ('int', 'flag', 'str') + + +def _manifests() -> 'dict[str, ModuleType]': + """Every census module, keyed by the package of :mod:`pcapkit.const` it covers.""" + found = sorted(info.name for info in pkgutil.iter_modules(__path__)) + return {name: importlib.import_module(f'{__name__}.{name}') for name in found} + + +#: Each census module, keyed by the package of :mod:`pcapkit.const` it covers. +MANIFESTS = _manifests() + +#: Every registry in the census, as ``module:QualName``, and its kind. +REGISTRIES = {name: kind for manifest in MANIFESTS.values() + for name, kind in manifest.REGISTRIES.items()} + +#: What :func:`compare` returns when a sweep and the census agree. +AGREES = {'found twice': [], 'not in the census': [], 'not found': []} # type: dict[str, list[str]] + +#: What a sweep that disagrees with the census asks for. +ADVICE = ('the registries under pcapkit.const differ from tests/const/census: record an ' + "added, removed or re-kinded registry in its package's census module, then check " + "that the sweep's exception sets still hold for it") + + +def names(*kinds: 'str') -> 'frozenset[str]': + """Every registry of ``kinds``, as the dotted ``module.QualName`` the sweeps key on. + + Args: + kinds: Any of :data:`KINDS`. + + Returns: + The registries' dotted names. + + """ + return frozenset(name.replace(':', '.') for name, kind in REGISTRIES.items() if kind in kinds) + + +def modules() -> 'frozenset[str]': + """Every module the census places a registry in.""" + return frozenset(name.split(':', 1)[0] for name in REGISTRIES) + + +def paths() -> 'list[str]': + """:func:`modules`, as sorted repository-relative file paths.""" + return sorted(module.replace('.', '/') + '.py' for module in modules()) + + +def compare(found: 'Iterable[str]', expected: 'Iterable[str]') -> 'dict[str, list[str]]': + """How what a sweep found differs from what the census expects. + + Args: + found: What the sweep discovered, one entry per discovery. + expected: What the census holds for that sweep. + + Returns: + A mapping equal to :data:`AGREES` exactly when ``found`` holds each of + ``expected`` once and nothing else; otherwise its lists name the + entries found more than once, found but not expected, and expected but + not found. + + """ + seen = collections.Counter(found) + wanted = set(expected) + return {'found twice': sorted(name for name, count in seen.items() if count > 1), + 'not in the census': sorted(set(seen) - wanted), + 'not found': sorted(wanted - set(seen))} diff --git a/tests/const/census/arp.py b/tests/const/census/arp.py new file mode 100644 index 0000000000..6d93b928e1 --- /dev/null +++ b/tests/const/census/arp.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- +"""The registries of :mod:`pcapkit.const.arp`, for :mod:`tests.const.census`.""" + +#: Each registry, as ``module:QualName``, and its kind. +REGISTRIES = { + 'pcapkit.const.arp.hardware:Hardware': 'int', + 'pcapkit.const.arp.operation:Operation': 'int', +} diff --git a/tests/const/census/dccp.py b/tests/const/census/dccp.py new file mode 100644 index 0000000000..4e8ff3dc9f --- /dev/null +++ b/tests/const/census/dccp.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +"""The registries of :mod:`pcapkit.const.dccp`, for :mod:`tests.const.census`.""" + +#: Each registry, as ``module:QualName``, and its kind. +REGISTRIES = { + 'pcapkit.const.dccp.ack_vector_state:AckVectorState': 'int', + 'pcapkit.const.dccp.drop_code:DropCode': 'int', + 'pcapkit.const.dccp.feature:Feature': 'int', + 'pcapkit.const.dccp.option:Option': 'int', + 'pcapkit.const.dccp.packet_type:PacketType': 'int', + 'pcapkit.const.dccp.reset_code:ResetCode': 'int', +} diff --git a/tests/const/census/esp.py b/tests/const/census/esp.py new file mode 100644 index 0000000000..2b84c93efe --- /dev/null +++ b/tests/const/census/esp.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- +"""The registries of :mod:`pcapkit.const.esp`, for :mod:`tests.const.census`.""" + +#: Each registry, as ``module:QualName``, and its kind. +REGISTRIES = { + 'pcapkit.const.esp.cipher:Cipher': 'int', + 'pcapkit.const.esp.integrity:Integrity': 'int', +} diff --git a/tests/const/census/ftp.py b/tests/const/census/ftp.py new file mode 100644 index 0000000000..f65003a378 --- /dev/null +++ b/tests/const/census/ftp.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- +"""The registries of :mod:`pcapkit.const.ftp`, for :mod:`tests.const.census`.""" + +#: Each registry, as ``module:QualName``, and its kind. +REGISTRIES = { + 'pcapkit.const.ftp.command:Command': 'str', + 'pcapkit.const.ftp.command:CommandType': 'flag', + 'pcapkit.const.ftp.command:ConformanceRequirement': 'int', + 'pcapkit.const.ftp.command:FEATCode': 'str', + 'pcapkit.const.ftp.return_code:GroupingInformation': 'int', + 'pcapkit.const.ftp.return_code:ResponseKind': 'int', + 'pcapkit.const.ftp.return_code:ReturnCode': 'int', +} diff --git a/tests/const/census/hip.py b/tests/const/census/hip.py new file mode 100644 index 0000000000..10deaabe31 --- /dev/null +++ b/tests/const/census/hip.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +"""The registries of :mod:`pcapkit.const.hip`, for :mod:`tests.const.census`.""" + +#: Each registry, as ``module:QualName``, and its kind. +REGISTRIES = { + 'pcapkit.const.hip.certificate:Certificate': 'int', + 'pcapkit.const.hip.cipher:Cipher': 'int', + 'pcapkit.const.hip.di:DITypes': 'int', + 'pcapkit.const.hip.ecdsa_curve:ECDSACurve': 'int', + 'pcapkit.const.hip.ecdsa_low_curve:ECDSALowCurve': 'int', + 'pcapkit.const.hip.eddsa_curve:EdDSACurve': 'int', + 'pcapkit.const.hip.esp_transform_suite:ESPTransformSuite': 'int', + 'pcapkit.const.hip.group:Group': 'int', + 'pcapkit.const.hip.hi_algorithm:HIAlgorithm': 'int', + 'pcapkit.const.hip.hit_suite:HITSuite': 'int', + 'pcapkit.const.hip.nat_traversal:NATTraversal': 'int', + 'pcapkit.const.hip.notify_message:NotifyMessage': 'int', + 'pcapkit.const.hip.packet:Packet': 'int', + 'pcapkit.const.hip.parameter:Parameter': 'int', + 'pcapkit.const.hip.registration:Registration': 'int', + 'pcapkit.const.hip.registration_failure:RegistrationFailure': 'int', + 'pcapkit.const.hip.suite:Suite': 'int', + 'pcapkit.const.hip.transport:Transport': 'int', +} diff --git a/tests/const/census/http.py b/tests/const/census/http.py new file mode 100644 index 0000000000..a82f99b385 --- /dev/null +++ b/tests/const/census/http.py @@ -0,0 +1,11 @@ +# -*- coding: utf-8 -*- +"""The registries of :mod:`pcapkit.const.http`, for :mod:`tests.const.census`.""" + +#: Each registry, as ``module:QualName``, and its kind. +REGISTRIES = { + 'pcapkit.const.http.error_code:ErrorCode': 'int', + 'pcapkit.const.http.frame:Frame': 'int', + 'pcapkit.const.http.method:Method': 'str', + 'pcapkit.const.http.setting:Setting': 'int', + 'pcapkit.const.http.status_code:StatusCode': 'int', +} diff --git a/tests/const/census/icmp.py b/tests/const/census/icmp.py new file mode 100644 index 0000000000..400ecff759 --- /dev/null +++ b/tests/const/census/icmp.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +"""The registries of :mod:`pcapkit.const.icmp`, for :mod:`tests.const.census`.""" + +#: Each registry, as ``module:QualName``, and its kind. +REGISTRIES = { + 'pcapkit.const.icmp.destination_unreachable_code:DestinationUnreachableCode': 'int', + 'pcapkit.const.icmp.parameter_problem_code:ParameterProblemCode': 'int', + 'pcapkit.const.icmp.time_exceeded_code:TimeExceededCode': 'int', + 'pcapkit.const.icmp.type:Type': 'int', +} diff --git a/tests/const/census/icmpv6.py b/tests/const/census/icmpv6.py new file mode 100644 index 0000000000..175f4ce48b --- /dev/null +++ b/tests/const/census/icmpv6.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +"""The registries of :mod:`pcapkit.const.icmpv6`, for :mod:`tests.const.census`.""" + +#: Each registry, as ``module:QualName``, and its kind. +REGISTRIES = { + 'pcapkit.const.icmpv6.destination_unreachable_code:DestinationUnreachableCode': 'int', + 'pcapkit.const.icmpv6.parameter_problem_code:ParameterProblemCode': 'int', + 'pcapkit.const.icmpv6.time_exceeded_code:TimeExceededCode': 'int', + 'pcapkit.const.icmpv6.type:Type': 'int', +} diff --git a/tests/const/census/ieee802_11.py b/tests/const/census/ieee802_11.py new file mode 100644 index 0000000000..ab988b0316 --- /dev/null +++ b/tests/const/census/ieee802_11.py @@ -0,0 +1,11 @@ +# -*- coding: utf-8 -*- +"""The registries of :mod:`pcapkit.const.ieee802_11`, for :mod:`tests.const.census`.""" + +#: Each registry, as ``module:QualName``, and its kind. +REGISTRIES = { + 'pcapkit.const.ieee802_11.control_subtype:ControlSubtype': 'int', + 'pcapkit.const.ieee802_11.data_subtype:DataSubtype': 'int', + 'pcapkit.const.ieee802_11.extension_subtype:ExtensionSubtype': 'int', + 'pcapkit.const.ieee802_11.management_subtype:ManagementSubtype': 'int', + 'pcapkit.const.ieee802_11.type:Type': 'int', +} diff --git a/tests/const/census/igmp.py b/tests/const/census/igmp.py new file mode 100644 index 0000000000..3446055232 --- /dev/null +++ b/tests/const/census/igmp.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- +"""The registries of :mod:`pcapkit.const.igmp`, for :mod:`tests.const.census`.""" + +#: Each registry, as ``module:QualName``, and its kind. +REGISTRIES = { + 'pcapkit.const.igmp.record_type:RecordType': 'int', + 'pcapkit.const.igmp.type:Type': 'int', +} diff --git a/tests/const/census/ipv4.py b/tests/const/census/ipv4.py new file mode 100644 index 0000000000..26d8d7f92c --- /dev/null +++ b/tests/const/census/ipv4.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +"""The registries of :mod:`pcapkit.const.ipv4`, for :mod:`tests.const.census`.""" + +#: Each registry, as ``module:QualName``, and its kind. +REGISTRIES = { + 'pcapkit.const.ipv4.classification_level:ClassificationLevel': 'int', + 'pcapkit.const.ipv4.option_class:OptionClass': 'int', + 'pcapkit.const.ipv4.option_number:OptionNumber': 'int', + 'pcapkit.const.ipv4.protection_authority:ProtectionAuthority': 'int', + 'pcapkit.const.ipv4.qs_function:QSFunction': 'int', + 'pcapkit.const.ipv4.router_alert:RouterAlert': 'int', + 'pcapkit.const.ipv4.tos_del:ToSDelay': 'int', + 'pcapkit.const.ipv4.tos_ecn:ToSECN': 'int', + 'pcapkit.const.ipv4.tos_pre:ToSPrecedence': 'int', + 'pcapkit.const.ipv4.tos_rel:ToSReliability': 'int', + 'pcapkit.const.ipv4.tos_thr:ToSThroughput': 'int', + 'pcapkit.const.ipv4.ts_flag:TSFlag': 'int', +} diff --git a/tests/const/census/ipv6.py b/tests/const/census/ipv6.py new file mode 100644 index 0000000000..5728820fbd --- /dev/null +++ b/tests/const/census/ipv6.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +"""The registries of :mod:`pcapkit.const.ipv6`, for :mod:`tests.const.census`.""" + +#: Each registry, as ``module:QualName``, and its kind. +REGISTRIES = { + 'pcapkit.const.ipv6.extension_header:ExtensionHeader': 'int', + 'pcapkit.const.ipv6.option:Option': 'int', + 'pcapkit.const.ipv6.option_action:OptionAction': 'int', + 'pcapkit.const.ipv6.qs_function:QSFunction': 'int', + 'pcapkit.const.ipv6.router_alert:RouterAlert': 'int', + 'pcapkit.const.ipv6.routing:Routing': 'int', + 'pcapkit.const.ipv6.seed_id:SeedID': 'int', + 'pcapkit.const.ipv6.smf_dpd_mode:SMFDPDMode': 'int', + 'pcapkit.const.ipv6.tagger_id:TaggerID': 'int', +} diff --git a/tests/const/census/ipx.py b/tests/const/census/ipx.py new file mode 100644 index 0000000000..76e17bba75 --- /dev/null +++ b/tests/const/census/ipx.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- +"""The registries of :mod:`pcapkit.const.ipx`, for :mod:`tests.const.census`.""" + +#: Each registry, as ``module:QualName``, and its kind. +REGISTRIES = { + 'pcapkit.const.ipx.packet:Packet': 'int', + 'pcapkit.const.ipx.socket:Socket': 'int', +} diff --git a/tests/const/census/l2tp.py b/tests/const/census/l2tp.py new file mode 100644 index 0000000000..d20cf76dac --- /dev/null +++ b/tests/const/census/l2tp.py @@ -0,0 +1,7 @@ +# -*- coding: utf-8 -*- +"""The registries of :mod:`pcapkit.const.l2tp`, for :mod:`tests.const.census`.""" + +#: Each registry, as ``module:QualName``, and its kind. +REGISTRIES = { + 'pcapkit.const.l2tp.type:Type': 'int', +} diff --git a/tests/const/census/linux.py b/tests/const/census/linux.py new file mode 100644 index 0000000000..6fd3d62774 --- /dev/null +++ b/tests/const/census/linux.py @@ -0,0 +1,7 @@ +# -*- coding: utf-8 -*- +"""The registries of :mod:`pcapkit.const.linux`, for :mod:`tests.const.census`.""" + +#: Each registry, as ``module:QualName``, and its kind. +REGISTRIES = { + 'pcapkit.const.linux.arphrd:ARPHRD': 'int', +} diff --git a/tests/const/census/mh.py b/tests/const/census/mh.py new file mode 100644 index 0000000000..7e4546decc --- /dev/null +++ b/tests/const/census/mh.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +"""The registries of :mod:`pcapkit.const.mh`, for :mod:`tests.const.census`.""" + +#: Each registry, as ``module:QualName``, and its kind. +REGISTRIES = { + 'pcapkit.const.mh.access_type:AccessType': 'int', + 'pcapkit.const.mh.ack_status_code:ACKStatusCode': 'int', + 'pcapkit.const.mh.ani_suboption:ANISuboption': 'int', + 'pcapkit.const.mh.auth_subtype:AuthSubtype': 'int', + 'pcapkit.const.mh.binding_ack_flag:BindingACKFlag': 'flag', + 'pcapkit.const.mh.binding_error:BindingError': 'int', + 'pcapkit.const.mh.binding_revocation:BindingRevocation': 'int', + 'pcapkit.const.mh.binding_update_flag:BindingUpdateFlag': 'flag', + 'pcapkit.const.mh.cga_extension:CGAExtension': 'int', + 'pcapkit.const.mh.cga_sec:CGASec': 'int', + 'pcapkit.const.mh.cga_type:CGAType': 'int', + 'pcapkit.const.mh.dhcp_support_mode:DHCPSupportMode': 'int', + 'pcapkit.const.mh.dns_status_code:DNSStatusCode': 'int', + 'pcapkit.const.mh.dsmip6_tls_packet:DSMIP6TLSPacket': 'int', + 'pcapkit.const.mh.dsmipv6_home_address:DSMIPv6HomeAddress': 'int', + 'pcapkit.const.mh.enumerating_algorithm:EnumeratingAlgorithm': 'int', + 'pcapkit.const.mh.fb_ack_status:FlowBindingACKStatus': 'int', + 'pcapkit.const.mh.fb_action:FlowBindingAction': 'int', + 'pcapkit.const.mh.fb_indication_trigger:FlowBindingIndicationTrigger': 'int', + 'pcapkit.const.mh.fb_type:FlowBindingType': 'int', + 'pcapkit.const.mh.flow_id_status:FlowIDStatus': 'int', + 'pcapkit.const.mh.flow_id_suboption:FlowIDSuboption': 'int', + 'pcapkit.const.mh.handoff_type:HandoffType': 'int', + 'pcapkit.const.mh.handover_ack_flag:HandoverACKFlag': 'flag', + 'pcapkit.const.mh.handover_ack_status:HandoverACKStatus': 'int', + 'pcapkit.const.mh.handover_initiate_flag:HandoverInitiateFlag': 'flag', + 'pcapkit.const.mh.handover_initiate_status:HandoverInitiateStatus': 'int', + 'pcapkit.const.mh.home_address_reply:HomeAddressReply': 'int', + 'pcapkit.const.mh.lla_code:LLACode': 'int', + 'pcapkit.const.mh.lma_mag_suboption:LMAControlledMAGSuboption': 'int', + 'pcapkit.const.mh.mn_group_id:MNGroupID': 'int', + 'pcapkit.const.mh.mn_id_subtype:MNIDSubtype': 'int', + 'pcapkit.const.mh.operator_id:OperatorID': 'int', + 'pcapkit.const.mh.option:Option': 'int', + 'pcapkit.const.mh.packet:Packet': 'int', + 'pcapkit.const.mh.qos_attribute:QoSAttribute': 'int', + 'pcapkit.const.mh.revocation_status_code:RevocationStatusCode': 'int', + 'pcapkit.const.mh.revocation_trigger:RevocationTrigger': 'int', + 'pcapkit.const.mh.status_code:StatusCode': 'int', + 'pcapkit.const.mh.traffic_selector:TrafficSelector': 'int', + 'pcapkit.const.mh.upa_status:UpdateNotificationACKStatus': 'int', + 'pcapkit.const.mh.upn_reason:UpdateNotificationReason': 'int', +} diff --git a/tests/const/census/ngap.py b/tests/const/census/ngap.py new file mode 100644 index 0000000000..59cf6e8fe3 --- /dev/null +++ b/tests/const/census/ngap.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- +"""The registries of :mod:`pcapkit.const.ngap`, for :mod:`tests.const.census`.""" + +#: Each registry, as ``module:QualName``, and its kind. +REGISTRIES = { + 'pcapkit.const.ngap.procedure_code:ProcedureCode': 'int', + 'pcapkit.const.ngap.protocol_ie:ProtocolIE': 'int', +} diff --git a/tests/const/census/ospf.py b/tests/const/census/ospf.py new file mode 100644 index 0000000000..3f45c1e58f --- /dev/null +++ b/tests/const/census/ospf.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- +"""The registries of :mod:`pcapkit.const.ospf`, for :mod:`tests.const.census`.""" + +#: Each registry, as ``module:QualName``, and its kind. +REGISTRIES = { + 'pcapkit.const.ospf.authentication:Authentication': 'int', + 'pcapkit.const.ospf.packet:Packet': 'int', +} diff --git a/tests/const/census/pcapng.py b/tests/const/census/pcapng.py new file mode 100644 index 0000000000..a1b189f1ce --- /dev/null +++ b/tests/const/census/pcapng.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +"""The registries of :mod:`pcapkit.const.pcapng`, for :mod:`tests.const.census`.""" + +#: Each registry, as ``module:QualName``, and its kind. +REGISTRIES = { + 'pcapkit.const.pcapng.block_type:BlockType': 'int', + 'pcapkit.const.pcapng.filter_type:FilterType': 'int', + 'pcapkit.const.pcapng.hash_algorithm:HashAlgorithm': 'int', + 'pcapkit.const.pcapng.option_type:OptionType': 'str', + 'pcapkit.const.pcapng.record_type:RecordType': 'int', + 'pcapkit.const.pcapng.secrets_type:SecretsType': 'int', + 'pcapkit.const.pcapng.tls_key_label:TLSKeyLabel': 'str', + 'pcapkit.const.pcapng.verdict_type:VerdictType': 'int', +} diff --git a/tests/const/census/reg.py b/tests/const/census/reg.py new file mode 100644 index 0000000000..926c0d7ec9 --- /dev/null +++ b/tests/const/census/reg.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +"""The registries of :mod:`pcapkit.const.reg`, for :mod:`tests.const.census`.""" + +#: Each registry, as ``module:QualName``, and its kind. +REGISTRIES = { + 'pcapkit.const.reg.apptype.apptype:AppType': 'str', + 'pcapkit.const.reg.apptype.apptype:TransportProtocol': 'int', + 'pcapkit.const.reg.apptype.dccp:DCCP': 'str', + 'pcapkit.const.reg.apptype.sctp:SCTP': 'str', + 'pcapkit.const.reg.apptype.tcp:TCP': 'str', + 'pcapkit.const.reg.apptype.udp:UDP': 'str', + 'pcapkit.const.reg.ethertype:EtherType': 'int', + 'pcapkit.const.reg.linktype:LinkType': 'int', + 'pcapkit.const.reg.transtype:TransType': 'int', +} diff --git a/tests/const/census/sctp.py b/tests/const/census/sctp.py new file mode 100644 index 0000000000..85a19798d8 --- /dev/null +++ b/tests/const/census/sctp.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +"""The registries of :mod:`pcapkit.const.sctp`, for :mod:`tests.const.census`.""" + +#: Each registry, as ``module:QualName``, and its kind. +REGISTRIES = { + 'pcapkit.const.sctp.cause_code:CauseCode': 'int', + 'pcapkit.const.sctp.chunk:Chunk': 'int', + 'pcapkit.const.sctp.parameter:Parameter': 'int', + 'pcapkit.const.sctp.payload_protocol_identifier:PayloadProtocolIdentifier': 'int', +} diff --git a/tests/const/census/sslv2.py b/tests/const/census/sslv2.py new file mode 100644 index 0000000000..be2cfd469b --- /dev/null +++ b/tests/const/census/sslv2.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- +"""The registries of :mod:`pcapkit.const.sslv2`, for :mod:`tests.const.census`.""" + +#: Each registry, as ``module:QualName``, and its kind. +REGISTRIES = { + 'pcapkit.const.sslv2.handshake_type:HandshakeType': 'int', + 'pcapkit.const.sslv2.version:Version': 'int', +} diff --git a/tests/const/census/tcp.py b/tests/const/census/tcp.py new file mode 100644 index 0000000000..a9f1f08d19 --- /dev/null +++ b/tests/const/census/tcp.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +"""The registries of :mod:`pcapkit.const.tcp`, for :mod:`tests.const.census`.""" + +#: Each registry, as ``module:QualName``, and its kind. +REGISTRIES = { + 'pcapkit.const.tcp.checksum:Checksum': 'int', + 'pcapkit.const.tcp.flags:Flags': 'flag', + 'pcapkit.const.tcp.mp_tcp_option:MPTCPOption': 'int', + 'pcapkit.const.tcp.option:Option': 'int', +} diff --git a/tests/const/census/tls.py b/tests/const/census/tls.py new file mode 100644 index 0000000000..c4aed7343d --- /dev/null +++ b/tests/const/census/tls.py @@ -0,0 +1,9 @@ +# -*- coding: utf-8 -*- +"""The registries of :mod:`pcapkit.const.tls`, for :mod:`tests.const.census`.""" + +#: Each registry, as ``module:QualName``, and its kind. +REGISTRIES = { + 'pcapkit.const.tls.content_type:ContentType': 'int', + 'pcapkit.const.tls.extension_type:ExtensionType': 'int', + 'pcapkit.const.tls.handshake_type:HandshakeType': 'int', +} diff --git a/tests/const/census/vlan.py b/tests/const/census/vlan.py new file mode 100644 index 0000000000..17d58dff6c --- /dev/null +++ b/tests/const/census/vlan.py @@ -0,0 +1,7 @@ +# -*- coding: utf-8 -*- +"""The registries of :mod:`pcapkit.const.vlan`, for :mod:`tests.const.census`.""" + +#: Each registry, as ``module:QualName``, and its kind. +REGISTRIES = { + 'pcapkit.const.vlan.priority_level:PriorityLevel': 'int', +} diff --git a/tests/const/test_const_census_2615_unit.py b/tests/const/test_const_census_2615_unit.py new file mode 100644 index 0000000000..6a70511389 --- /dev/null +++ b/tests/const/test_const_census_2615_unit.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- +"""The layout of the const census in :mod:`tests.const.census`. C.f. #2615. + +The census keeps one module per package of :mod:`pcapkit.const` so that a +registry added to one package and a registry added to another land in +different files. That only holds while each module covers its own package +alone, and while each keeps its entries sorted, one per line, so that two +additions to one package land on different lines too. This module pins both, +and that every entry reads ``module:QualName`` with a known kind. + +Whether the census matches the tree is the sweeps' question, not this +module's: each const sweep compares what it discovers against the census. + +This module is unit-tier and reads no capture. + +""" +import pathlib +import unittest + +from tests.const import census + +#: Test tier (#1958): checks the census files, a question about the tree. +__tier__ = 'LATEST' + +#: Repository root. +ROOT = pathlib.Path(__file__).resolve().parents[2] + + +class CensusLayoutTests(unittest.TestCase): + """Each census module covers one package, sorted, one registry per line.""" + + def test_each_module_is_named_after_a_const_package(self) -> None: + for package in census.MANIFESTS: + with self.subTest(package=package): + self.assertTrue((ROOT / 'pcapkit' / 'const' / package / '__init__.py').is_file(), + f'tests/const/census/{package}.py names no package of pcapkit.const') + + def test_each_module_covers_its_own_package_alone(self) -> None: + for package, manifest in census.MANIFESTS.items(): + for name in manifest.REGISTRIES: + with self.subTest(package=package, registry=name): + module, colon, qualname = name.partition(':') + self.assertEqual(colon, ':', f'{name} is not module:QualName') + self.assertTrue(module.startswith(f'pcapkit.const.{package}.'), + f'{name} belongs in the census module of its own package') + self.assertRegex(qualname, r'^[A-Za-z_]\w*$') + + def test_each_module_is_sorted(self) -> None: + for package, manifest in census.MANIFESTS.items(): + with self.subTest(package=package): + entries = list(manifest.REGISTRIES) + self.assertEqual(entries, sorted(entries), + f'tests/const/census/{package}.py is not sorted') + + def test_each_entry_is_on_a_line_of_its_own(self) -> None: + for package, manifest in census.MANIFESTS.items(): + lines = pathlib.Path(manifest.__file__).read_text(encoding='utf-8').splitlines() + for name in manifest.REGISTRIES: + with self.subTest(package=package, registry=name): + self.assertEqual(sum(1 for line in lines if f"'{name}'" in line), 1) + line, = (line for line in lines if f"'{name}'" in line) + self.assertEqual(line.strip(), f"'{name}': '{manifest.REGISTRIES[name]}',") + + def test_every_kind_is_known(self) -> None: + for name, kind in census.REGISTRIES.items(): + with self.subTest(registry=name): + self.assertIn(kind, census.KINDS) + + def test_compare_names_what_differs(self) -> None: + self.assertEqual(census.compare(['a', 'b'], ['b', 'a']), census.AGREES) + self.assertEqual(census.compare(['a', 'a', 'c'], ['a', 'b']), + {'found twice': ['a'], 'not in the census': ['c'], 'not found': ['b']}) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/const/test_const_enum_builtin_parity.py b/tests/const/test_const_enum_builtin_parity.py index f7912e0534..ce6ed4e29c 100644 --- a/tests/const/test_const_enum_builtin_parity.py +++ b/tests/const/test_const_enum_builtin_parity.py @@ -62,6 +62,7 @@ from tests._support import (ISOLATED_PREFIXES, purge_modules, reimport_once_per_class, restore_modules, snapshot_modules) +from tests.const import census if TYPE_CHECKING: from typing import Optional @@ -351,39 +352,43 @@ def setUpClass(cls) -> None: # pre-purge modules so the polluted ones are unreachable. cls.addClassCleanup(restore_modules, snapshot, ISOLATED_PREFIXES) - def test_the_sweep_size_is_pinned(self) -> None: - """A registry added or removed needs a fresh look, not a silent pass.""" + def test_the_sweep_matches_the_census(self) -> None: + """A registry added, removed or re-kinded needs a fresh look, not a silent pass. + + Each registry is held to its line in :mod:`tests.const.census` (#2615), + so a failure names it. The decomposition is asserted, not just the + whole, so this sweep stays in step with the narrower ones it overlaps: + the non-flag IntEnum and the IntFlag sweeps of + :mod:`tests.const.test_const_enum_lookup`, and their union in + :mod:`tests.const.test_const_enum_get`. ``TransportProtocol`` has no + ``IntFlag`` base (GitHub issue #808), so it is recorded as ``'int'``. + + The application layer registry, which GitHub issue #732 split into a + package -- the memberless :class:`~pcapkit.const.reg.apptype.apptype.AppType` + base plus one registry per transport protocol -- accounts for several of + the ``'str'`` registries. It is discovered exactly like a member-bearing + registry, since this sweep is structural and never looks at members. + Another is GitHub issue #886's + :class:`~pcapkit.const.pcapng.tls_key_label.TLSKeyLabel` -- it defines no + ``_missing_`` of its own, so it is not among the ``value.upper()`` + outliers #647 fixed above, and it is outside the narrower + ``(IntEnum, IntFlag)``-only sweep of :mod:`tests.const.test_const_enum_get`. + GitHub issue #880's ``ngap.ProcedureCode`` and ``ngap.ProtocolIE`` are + on EnumRegistry, and so ``'int'``. + + """ flags = [obj for obj in self.enums if issubclass(obj, aenum.Flag)] strs = [obj for obj in self.enums if issubclass(obj, str) and not issubclass(obj, aenum.Flag)] ints = [obj for obj in self.enums if issubclass(obj, int) and not issubclass(obj, aenum.Flag)] - # The decomposition is asserted, not just the total, so this sweep stays - # in step with the three narrower ones it overlaps: the non-flag IntEnum - # and the IntFlag sweeps of tests.const.test_const_enum_lookup, and their - # sum in tests.const.test_const_enum_get. ``TransportProtocol`` has no - # ``IntFlag`` base (GitHub issue #808), so it is in the int count. - # - # The application layer registry, which GitHub issue #732 split into a - # package -- the memberless pcapkit.const.reg.apptype.apptype.AppType base - # plus one registry per transport protocol -- accounts for several of the - # string registries. It is discovered exactly like a member-bearing - # registry, since this sweep is structural and never looks at members. - # Another is GitHub issue #886's - # pcapkit.const.pcapng.tls_key_label.TLSKeyLabel -- it defines no - # ``_missing_`` of its own, so it is not among the ``value.upper()`` - # outliers #647 fixed above, and it is outside - # tests.const.test_const_enum_get's narrower (IntEnum, IntFlag)-only - # sweep. The int count includes GitHub issue #880's ngap.ProcedureCode - # and ngap.ProtocolIE, which are on EnumRegistry, as are the ICMP, - # ICMPv6, IGMP and DCCP registry families, and the IEEE 802.11 type - # and subtype registries of GitHub issue #2575. - self.assertEqual(len(ints), 141) - self.assertEqual(len(flags), 6) - self.assertEqual(len(strs), 10) - self.assertEqual(len(self.enums), 157) - self.assertEqual(len({obj.__module__ for obj in self.enums}), 151) + self.assertEqual(census.compare(map(_qualname, self.enums), census.names(*census.KINDS)), + census.AGREES, census.ADVICE) + for kind, found in (('int', ints), ('flag', flags), ('str', strs)): + with self.subTest(kind=kind): + self.assertEqual(census.compare(map(_qualname, found), census.names(kind)), + census.AGREES, census.ADVICE) def test_every_registry_rejects_a_negative_value(self) -> None: """The registry-wide form of GitHub issue #647. diff --git a/tests/const/test_const_enum_get.py b/tests/const/test_const_enum_get.py index 00a25f5634..d40e52003d 100644 --- a/tests/const/test_const_enum_get.py +++ b/tests/const/test_const_enum_get.py @@ -36,7 +36,8 @@ for a separate reason and covered by its own test below): the main sweep, plus :data:`EXPECTED_TO_RESOLVE_ANYTHING`, plus :data:`EXPECTED_WITHOUT_AN_INTEGER_DEFAULT`, plus ``FilterType`` -- together, -the population :meth:`test_the_sweep_size_is_pinned` counts. +the population :meth:`test_the_sweep_matches_the_census` holds to +:mod:`tests.const.census`. :class:`~pcapkit.const.pcapng.option_type.OptionType` and :class:`~pcapkit.const.reg.apptype.AppType` are outside this sweep because @@ -72,6 +73,7 @@ from aenum import IntEnum, IntFlag from tests._support import ISOLATED_PREFIXES, purge_modules, restore_modules, snapshot_modules +from tests.const import census #: Test tier (#1958): exercises metaclass, ABC or enum machinery, which differs by #: interpreter. @@ -289,15 +291,17 @@ def test_operation_and_linktype_get_honour_a_default(self) -> None: self.assertIs(Operation.get(99999, 1), Operation.REQUEST) self.assertIs(LinkType.get(-5, 1), LinkType.ETHERNET) - def test_the_sweep_size_is_pinned(self) -> None: + def test_the_sweep_matches_the_census(self) -> None: # If this drifts, a const enum was added, removed or renamed, and the - # exception sets below need a fresh look rather than a silent pass. - # GitHub issue #880's ngap.ProcedureCode and ngap.ProtocolIE are on + # exception sets below need a fresh look rather than a silent pass; the + # sweep is the census's 'int' and 'flag' registries (#2615). GitHub + # issue #880's ngap.ProcedureCode and ngap.ProtocolIE are on # EnumRegistry rather than hand-rolled IntEnum classes in # pcapkit.protocols.application.ngap; neither needs an exception-set # entry, since both take the base's own get(key, default). names = {_qualname(obj) for obj in self.enums} - self.assertEqual(len(self.enums), 147) + self.assertEqual(census.compare(map(_qualname, self.enums), census.names('int', 'flag')), + census.AGREES, census.ADVICE) for expected in (EXPECTED_TO_RESOLVE_ANYTHING, EXPECTED_WITHOUT_AN_INTEGER_DEFAULT, EXPECTED_WITHOUT_A_CACHEABLE_FALLBACK): self.assertTrue(expected.issubset(names), @@ -325,7 +329,10 @@ def test_every_integer_path_consults_the_default(self) -> None: if qualname in EXPECTED_WITHOUT_A_CACHEABLE_FALLBACK: # Covered by its own test below: it raises for UNRESOLVABLE # like the rest of this sweep, but has no real member to use - # as ``fallback`` -- see the set's own comment. + # as ``fallback`` -- see the set's own comment. Asserted, as + # above, so the set cannot quietly excuse a registry that has one. + self.assertEqual(list(obj), [], + f'{qualname} has a member to fall back to and belongs in the sweep') continue with self.subTest(enum=qualname): with self.assertRaises(ValueError): @@ -333,11 +340,12 @@ def test_every_integer_path_consults_the_default(self) -> None: fallback = next(iter(obj)).value self.assertIs(obj.get(UNRESOLVABLE, fallback), obj(fallback)) covered += 1 - # The sweep size ``test_the_sweep_size_is_pinned`` pins, less every - # registry excused above: ``EXPECTED_TO_RESOLVE_ANYTHING`` and - # ``EXPECTED_WITHOUT_A_CACHEABLE_FALLBACK``, with - # ``EXPECTED_WITHOUT_AN_INTEGER_DEFAULT`` empty. - self.assertEqual(covered, 144) + # The census's integer registries, less every one excused above. Each + # excusal asserts its own reason, here or in a test of its own, so the + # derived figure lets no registry out of the sweep unchecked (#2615). + excused = (EXPECTED_TO_RESOLVE_ANYTHING | EXPECTED_WITHOUT_AN_INTEGER_DEFAULT + | EXPECTED_WITHOUT_A_CACHEABLE_FALLBACK) + self.assertEqual(covered, len(census.names('int', 'flag') - excused)) def test_the_always_resolving_registries_have_nothing_to_fall_back_to(self) -> None: """The two registries excused from the sweep, and why. diff --git a/tests/const/test_const_enum_lookup.py b/tests/const/test_const_enum_lookup.py index 2999754dbc..2eea0fdc5f 100644 --- a/tests/const/test_const_enum_lookup.py +++ b/tests/const/test_const_enum_lookup.py @@ -69,6 +69,7 @@ from tests._support import (ISOLATED_PREFIXES, purge_modules, reimport_once_per_class, restore_modules, snapshot_modules) +from tests.const import census #: Test tier (#1958): exercises metaclass, ABC or enum machinery, which differs by #: interpreter. @@ -192,15 +193,16 @@ def setUpClass(cls) -> None: cls.enums = _iter_const_int_enums() def test_every_registry_enum_was_discovered(self) -> None: - # Pins the size of the sweep itself: if this drifts, a const enum was - # added, removed, or renamed, and EXPECTED_TO_REJECT_ZERO (and the - # analysis in GitHub issue #492) needs a fresh look rather than a - # silent pass. ``TransportProtocol`` has no ``IntFlag`` base (GitHub - # issue #808), so it is in this sweep rather than - # ``_iter_const_int_flags``'s, and so are GitHub issue #880's + # Holds the sweep to the census's 'int' registries (#2615): if this + # drifts, a const enum was added, removed, or renamed, and + # EXPECTED_TO_REJECT_ZERO (and the analysis in GitHub issue #492) needs + # a fresh look rather than a silent pass. ``TransportProtocol`` has no + # ``IntFlag`` base (GitHub issue #808), so it is in this sweep rather + # than ``_iter_const_int_flags``'s, as are GitHub issue #880's # ngap.ProcedureCode and ngap.ProtocolIE, which are on EnumRegistry. - names = {f'{obj.__module__}.{obj.__qualname__}' for obj in self.enums} - self.assertEqual(len(self.enums), 141) + found = [f'{obj.__module__}.{obj.__qualname__}' for obj in self.enums] + names = set(found) + self.assertEqual(census.compare(found, census.names('int')), census.AGREES, census.ADVICE) self.assertTrue(EXPECTED_TO_REJECT_ZERO.issubset(names), f'expected reject-list entries missing from the sweep: ' f'{EXPECTED_TO_REJECT_ZERO - names}') @@ -308,14 +310,16 @@ def setUpClass(cls) -> None: # snapshot rebinds it straight back to the pristine modules instead. cls.addClassCleanup(restore_modules, snapshot, ISOLATED_PREFIXES) - def test_the_sweep_size_is_pinned(self) -> None: + def test_the_sweep_matches_the_census(self) -> None: """A flag enum added or removed needs a fresh look, not a silent pass. + The sweep is the census's ``'flag'`` registries (#2615). ``TransportProtocol`` has no ``IntFlag`` base (GitHub issue #808), so it is in ``_iter_const_int_enums``'s sweep rather than this one. """ - names = {f'{obj.__module__}.{obj.__qualname__}' for obj in self.flags} - self.assertEqual(len(self.flags), 6) + found = [f'{obj.__module__}.{obj.__qualname__}' for obj in self.flags] + names = set(found) + self.assertEqual(census.compare(found, census.names('flag')), census.AGREES, census.ADVICE) expected = {f'pcapkit.const.mh.{stem}.{name}' for stem, name, _, _ in MH_FLAG_ENUMS} diff --git a/tests/const/test_const_registry_protocol.py b/tests/const/test_const_registry_protocol.py index 84aad8338d..d9b86c772a 100644 --- a/tests/const/test_const_registry_protocol.py +++ b/tests/const/test_const_registry_protocol.py @@ -20,8 +20,10 @@ """ from __future__ import annotations +import ast import importlib import pathlib +import re import unittest from typing import TYPE_CHECKING @@ -30,6 +32,7 @@ from pcapkit.corekit.enum import EnumRegistry from tests._support import (ISOLATED_PREFIXES, purge_modules, reimport_once_per_class, restore_modules, snapshot_modules) +from tests.const import census if TYPE_CHECKING: from typing import Any @@ -79,6 +82,20 @@ '_unregistered_member') +def _is_generated_shape(source: 'str') -> 'bool': + """Whether ``source`` reads as the default template's output. + + The checks + :meth:`GeneratedTemplateSourceInheritsTests.test_every_generated_const_module_is_accounted_for` + asserts one by one: the base imported, a registry declared on it over + :class:`~aenum.IntEnum`, and no copy of :data:`PROTOCOL`. + + """ + return ('from pcapkit.corekit.enum import EnumRegistry' in source + and re.search(r'class \w+\(EnumRegistry, IntEnum\):', source) is not None + and not any(f'def {method}(' in source for method in PROTOCOL)) + + def _unused_value(cls: 'Any') -> 'int': """First non-negative integer no member of ``cls`` carries. @@ -139,6 +156,33 @@ def test_converted_crawlers_do_not_spell_their_own_get(self) -> None: self.assertIn('from pcapkit.corekit.enum import EnumRegistry', source) self.assertNotIn("def get(key: 'int | str'", source) + def test_each_converted_entry_is_what_converted_means(self) -> None: + """Each :data:`CONVERTED` entry is a bespoke template whose members carry nothing extra. + + Its crawler, the one :data:`CONVERTED_VENDORS` pairs it with, spells a + module-level ``LINE`` template of its own rather than rendering the + default one, and neither that crawler nor the const module defines a + ``__new__``. So a default-template registry cannot be filed here in place + of the generated side of + :meth:`GeneratedTemplateSourceInheritsTests.test_every_generated_const_module_is_accounted_for` + (#2615). + + """ + crawlers = [relpath.replace('pcapkit/const/', 'pcapkit/vendor/', 1) + for _, _, relpath in CONVERTED] + self.assertEqual(sorted(crawlers), sorted(CONVERTED_VENDORS)) + for (_, name, relpath), crawler in zip(CONVERTED, crawlers): + with self.subTest(registry=name): + tree = ast.parse((REPO_ROOT / crawler).read_text()) + self.assertTrue(any(isinstance(node, ast.Assign) + and any(isinstance(target, ast.Name) and target.id == 'LINE' + for target in node.targets) + for node in tree.body), + f'{crawler} renders the default template, so {name} is not converted') + for path in (relpath, crawler): + self.assertNotIn('def __new__(', (REPO_ROOT / path).read_text(), + f'{path} attaches attributes in __new__, so {name} is not converted') + class ProtocolIsInheritedTests(unittest.TestCase): """All of the protocol must be present, and reached from the base class.""" @@ -846,8 +890,20 @@ def test_every_generated_const_module_is_accounted_for(self) -> None: generated = [path for path in all_files if path not in converted_relpaths and path not in EXCLUDED_FROM_INTENUM_CENSUS] - self.assertEqual(len(all_files), 151) - self.assertEqual(len(generated), 134) + # The tree's modules are the census's (#2615). The split is then exact + # while each converted or excluded path is one of them, no path is + # both, an excluded module is not the generated shape -- the reason it + # is excluded -- and a converted one is what CONVERTED means, which + # test_each_converted_entry_is_what_converted_means checks; so neither + # set can hide a generated module. + self.assertEqual(census.compare(all_files, census.paths()), census.AGREES, census.ADVICE) + self.assertEqual(sorted((converted_relpaths | EXCLUDED_FROM_INTENUM_CENSUS) - set(all_files)), + [], 'CONVERTED or EXCLUDED_FROM_INTENUM_CENSUS names a module not in the tree') + self.assertEqual(sorted(converted_relpaths & EXCLUDED_FROM_INTENUM_CENSUS), []) + for relpath in sorted(EXCLUDED_FROM_INTENUM_CENSUS): + with self.subTest(excluded=relpath): + self.assertFalse(_is_generated_shape((REPO_ROOT / relpath).read_text()), + f'{relpath} is the generated shape, so it is not excluded') for relpath in generated: with self.subTest(module=relpath): source = (REPO_ROOT / relpath).read_text() diff --git a/tests/project/test_dead_helpers_2318_unit.py b/tests/project/test_dead_helpers_2318_unit.py index 0a993b1c18..083eede9c2 100644 --- a/tests/project/test_dead_helpers_2318_unit.py +++ b/tests/project/test_dead_helpers_2318_unit.py @@ -4,7 +4,7 @@ A helper nothing reads answers a question nobody asks. Each is held to the rule that makes it worth keeping rather than to its name. -A ``Family`` in ``examples/generators/dispatch.py`` carries an ``enum`` only +A ``Family`` under ``examples/generators/dispatch_families/`` carries an ``enum`` only where :func:`code_name` can fall back to one, i.e. where the family's registry holds a key with no ``.name`` of its own; every key of ``Internet.__proto__`` carries one, so the ``internet`` family needs no ``enum``. A ``FAKE_GH`` arm diff --git a/tests/project/test_rfc_constants_one_home_1739_unit.py b/tests/project/test_rfc_constants_one_home_1739_unit.py index e8f22ae208..a71d3f2966 100644 --- a/tests/project/test_rfc_constants_one_home_1739_unit.py +++ b/tests/project/test_rfc_constants_one_home_1739_unit.py @@ -208,87 +208,136 @@ #: Sweep hits of a homed value that are not a copy of it, one expression each: #: the module, the expression (see :func:`_context`) and why. A hit here is in a #: module whose own protocol owns a constant of that value, or that reads one. +#: Sorted, so that entries for unrelated modules land apart (#2615). NOT_A_COPY = { - ('pcapkit/protocols/application/ssl.py', 'self._read_octets(size=32)'): - "a TLS/DTLS ClientHello/ServerHello Random is 32 octets (:rfc:`9846#section-4.2.2`), " - 'not an Interface Description Block minimum', - ('pcapkit/protocols/protocol.py', 'def _make_mac_addr(size=6)'): - 'a MAC address is 6 octets (IEEE 802), not the IPv6 Next Header offset', - ('pcapkit/protocols/misc/pcapng.py', 'def _make_mac_addr(size=6)'): - "the base MAC maker's 6-octet default, which the override repeats, not the IPv6 Next " - 'Header offset', - ('pcapkit/protocols/schema/application/dtls.py', 'NumberField(length=6)'): - "DTLS's 48-bit sequence_number (:rfc:`9147#section-4`), not the IPv6 Next " - 'Header offset', - ('pcapkit/protocols/schema/application/dtls.py', "{'epoch': (6, 2)}"): - "the EE bits of a DTLS 1.3 unified header, at bit 6 of its first octet " - '(:rfc:`9147#section-4.1`), not the IPv6 Next Header offset', - ('pcapkit/protocols/schema/link/frad.py', "{'dlci_hi': (0, 6)}"): - "the 6-bit width of a Q.922 address field's high-order DLCI bits " - '(:rfc:`2427#section-3`), not the IPv6 Next Header offset', - ('pcapkit/protocols/schema/link/frad.py', "{'cr': (6, 1)}"): - 'the C/R bit at bit offset 6 of a Q.922 address field (:rfc:`2427#section-3`), ' - 'not the IPv6 Next Header offset', - ('pcapkit/protocols/schema/link/frad.py', "{'fecn': (12, 1)}"): - 'the FECN bit at bit offset 12 of a Q.922 address field (:rfc:`2427#section-3`), ' - 'not the TCP Data Offset octet', + ('pcapkit/dumpkit/common.py', 'codepoint > 65535'): + 'above the Basic Multilingual Plane, needs a UTF-16 surrogate pair, not an IP length', + ('pcapkit/dumpkit/common.py', 'ord(char) > 65535'): + 'the last code point of the Basic Multilingual Plane, not an IP length', + ('pcapkit/foundation/engines/dpkt.py', 'pkt.ts_high << 32'): + 'a bit width: the high half of a 64-bit timestamp', + ('pcapkit/foundation/engines/dpkt.py', 'version == 6'): + 'the IP version number', + ('pcapkit/foundation/engines/dpkt.py', '{24: ip6}'): + 'the BSD loopback address family of IPv6 on NetBSD and OpenBSD', + ('pcapkit/foundation/engines/dpkt.py', '{28: ip6}'): + 'the BSD loopback address family of IPv6 on FreeBSD', + ('pcapkit/foundation/engines/pcap.py', 'head[8:12]'): + "a record header's incl_len field, octets 8 to 12", + ('pcapkit/foundation/engines/pcapng.py', 'head[8:12]'): + "the Byte-Order Magic field of a Section Header Block, octets 8 to 12", + ('pcapkit/foundation/engines/pcapng.py', 'self._peek(12)'): + "the first 12 octets of a block -- type, length and a Section Header Block's " + 'Byte-Order Magic -- not the generic block minimum', + ('pcapkit/foundation/engines/pcapng.py', 'self._peek(16)'): + "a Decryption Secrets Block up to its Secrets Length, not a block minimum", + ('pcapkit/foundation/engines/pcapng.py', 'self._peek(16)[12:16]'): + "a Decryption Secrets Block's Secrets Length field, octets 12 to 16", + ('pcapkit/foundation/engines/scapy.py', 'len(options) >= 12'): + 'an if_tsoffset option: 4 octets of header and 8 of value', + ('pcapkit/foundation/engines/scapy.py', 'options[4:12]'): + 'the 8-octet value of an if_tsoffset option', ('pcapkit/protocols/application/dtls.py', '_CIPHERTEXT_MIN_LENGTH = 16'): "the shortest DTLS 1.3 ciphertext, the record-number sample length " '(:rfc:`9147#section-4.2.3`), not a pcap record header', + ('pcapkit/protocols/application/ssl.py', 'self._read_octets(size=32)'): + "a TLS/DTLS ClientHello/ServerHello Random is 32 octets (:rfc:`9846#section-4.2.2`), " + 'not an Interface Description Block minimum', + ('pcapkit/protocols/application/sslv2.py', "b'\\x00' * 16"): + "the default length of an SSLv2 CLIENT-HELLO challenge or SERVER-HELLO " + "connection ID (the draft's own 16-32 octet minimum), not the PCAP classic " + 'record header length', + ('pcapkit/protocols/internet/icmp.py', 'version == 6'): + 'ICMPv6, the ICMP of IP version 6 (:rfc:`4443`), not the Next Header offset', + ('pcapkit/protocols/internet/ip.py', 'version == 6'): + 'the IP version number, not the Next Header offset', ('pcapkit/protocols/internet/ipv6.py', "schema.hextet['version'] != 6"): 'the IP version number, not the Next Header offset', ('pcapkit/protocols/internet/ipv6.py', 'self._import_next_layer(version=6)'): 'the IP version number, not the Next Header offset', - ('pcapkit/protocols/internet/ipv6.py', "{'version': 6}"): - 'the IP version number, not the Next Header offset', ('pcapkit/protocols/internet/ipv6.py', 'version != 6'): 'the IP version number, not the Next Header offset', - ('pcapkit/protocols/internet/ip.py', 'version == 6'): + ('pcapkit/protocols/internet/ipv6.py', "{'version': 6}"): 'the IP version number, not the Next Header offset', - ('pcapkit/protocols/internet/icmp.py', 'version == 6'): - 'ICMPv6, the ICMP of IP version 6 (:rfc:`4443`), not the Next Header offset', - ('pcapkit/protocols/schema/internet/ipv6.py', 'ptr + 6'): - 'the end of a Jumbo Payload option (2 octets of header, 4 of value)', + ('pcapkit/protocols/link/gre.py', 'return 16'): + "GRE's own longest possible header -- the 4-octet base plus all three optional " + 'extensions at 4 octets each (:rfc:`2784`/:rfc:`2890`) -- not a PCAP record header', ('pcapkit/protocols/link/linux_sll1.py', 'return 16'): 'the fixed length of the LINUX_SLL (v1) header, not the PCAP record header', ('pcapkit/protocols/link/linux_sll2.py', 'return 20'): 'the fixed length of the LINUX_SLL2 header, not the IPv4 or TCP minimum header', ('pcapkit/protocols/misc/pcap/frame.py', 'version == 6'): 'the IP version number, not the decimal places of a resolution', - ('pcapkit/protocols/misc/pcapng.py', 'version == 6'): - 'the IP version number, not the decimal places of a resolution', - ('pcapkit/protocols/misc/pcapng.py', 'schema.length != 6'): - 'a MAC address option is 6 octets', + ('pcapkit/protocols/misc/pcapng.py', '(body + bytes(16))[4:16]'): + 'an epb_hash option body padded to its 16-octet MD5 digest', + ('pcapkit/protocols/misc/pcapng.py', '(body + bytes(20))[:20]'): + 'an epb_hash option body padded to its 20-octet SHA-1 digest', + ('pcapkit/protocols/misc/pcapng.py', '16 + _utf8_length(records)'): + 'an IPv6 name record is 16 octets of address, not a block', ('pcapkit/protocols/misc/pcapng.py', 'Schema_IF_MACAddrOption(length=6)'): 'a MAC address option is 6 octets', - ('pcapkit/protocols/misc/pcapng.py', 'schema.length != 16'): - 'an IPv6 address option is 16 octets, not a Simple Packet Block', ('pcapkit/protocols/misc/pcapng.py', 'Schema_NS_DNSIP6AddrOption(length=16)'): 'an IPv6 address option is 16 octets, not a Simple Packet Block', - ('pcapkit/protocols/misc/pcapng.py', '(body + bytes(20))[:20]'): - 'an epb_hash option body padded to its 20-octet SHA-1 digest', - ('pcapkit/protocols/misc/pcapng.py', 'bytes(20)'): - 'an epb_hash option body padded to its 20-octet SHA-1 digest', - ('pcapkit/protocols/misc/pcapng.py', '(body + bytes(16))[4:16]'): - 'an epb_hash option body padded to its 16-octet MD5 digest', ('pcapkit/protocols/misc/pcapng.py', 'bytes(16)'): 'an epb_hash option body padded to its 16-octet MD5 digest', - ('pcapkit/protocols/misc/pcapng.py', '16 + _utf8_length(records)'): - 'an IPv6 name record is 16 octets of address, not a block', + ('pcapkit/protocols/misc/pcapng.py', 'bytes(20)'): + 'an epb_hash option body padded to its 20-octet SHA-1 digest', + ('pcapkit/protocols/misc/pcapng.py', 'def _make_mac_addr(size=6)'): + "the base MAC maker's 6-octet default, which the override repeats, not the IPv6 Next " + 'Header offset', + ('pcapkit/protocols/misc/pcapng.py', 'schema.addr_high << 16'): + 'a bit width: the high half of a 32-bit address', + ('pcapkit/protocols/misc/pcapng.py', 'schema.length != 16'): + 'an IPv6 address option is 16 octets, not a Simple Packet Block', + ('pcapkit/protocols/misc/pcapng.py', 'schema.length != 6'): + 'a MAC address option is 6 octets', + ('pcapkit/protocols/misc/pcapng.py', 'short_address >> 16'): + 'a bit width: the high half of a 32-bit address', ('pcapkit/protocols/misc/pcapng.py', 'timestamp_high << 32'): 'a bit width: the high half of a 64-bit timestamp', ('pcapkit/protocols/misc/pcapng.py', 'ts_info >> 32'): 'a bit width: the high half of a 64-bit timestamp', - ('pcapkit/protocols/misc/pcapng.py', 'short_address >> 16'): - 'a bit width: the high half of a 32-bit address', - ('pcapkit/protocols/misc/pcapng.py', 'schema.addr_high << 16'): - 'a bit width: the high half of a 32-bit address', - ('pcapkit/protocols/schema/misc/pcapng.py', 'BytesField(length=6)'): - 'a MAC address is 6 octets', + ('pcapkit/protocols/misc/pcapng.py', 'version == 6'): + 'the IP version number, not the decimal places of a resolution', + ('pcapkit/protocols/protocol.py', '2 ** 16'): + "the exponent past the Quick-Start rate code's four bits, N = 16 " + '(:rfc:`4782#section-3.1`), not a PCAP record header length', + ('pcapkit/protocols/protocol.py', '40 * 2 ** 16'): + "the first rate the Quick-Start rate code's four bits cannot carry, K * 2 ** 16 " + '(:rfc:`4782#section-3.1`), not the IPv6 header length', + ('pcapkit/protocols/protocol.py', '40 * 2 ** code'): + "the Quick-Start rate unit, K = 40 kbps (:rfc:`4782#section-3.1`), not the IPv6 " + 'header length', + ('pcapkit/protocols/protocol.py', 'def _make_mac_addr(size=6)'): + 'a MAC address is 6 octets (IEEE 802), not the IPv6 Next Header offset', + ('pcapkit/protocols/protocol.py', 'rate // 40'): + "the Quick-Start rate unit, K = 40 kbps (:rfc:`4782#section-3.1`), not the IPv6 " + 'header length', + ('pcapkit/protocols/schema/application/dtls.py', 'NumberField(length=6)'): + "DTLS's 48-bit sequence_number (:rfc:`9147#section-4`), not the IPv6 Next " + 'Header offset', + ('pcapkit/protocols/schema/application/dtls.py', "{'epoch': (6, 2)}"): + "the EE bits of a DTLS 1.3 unified header, at bit 6 of its first octet " + '(:rfc:`9147#section-4.1`), not the IPv6 Next Header offset', + ('pcapkit/protocols/schema/internet/ipv6.py', 'ptr + 6'): + 'the end of a Jumbo Payload option (2 octets of header, 4 of value)', + ('pcapkit/protocols/schema/link/frad.py', "{'cr': (6, 1)}"): + 'the C/R bit at bit offset 6 of a Q.922 address field (:rfc:`2427#section-3`), ' + 'not the IPv6 Next Header offset', + ('pcapkit/protocols/schema/link/frad.py', "{'dlci_hi': (0, 6)}"): + "the 6-bit width of a Q.922 address field's high-order DLCI bits " + '(:rfc:`2427#section-3`), not the IPv6 Next Header offset', + ('pcapkit/protocols/schema/link/frad.py', "{'fecn': (12, 1)}"): + 'the FECN bit at bit offset 12 of a Q.922 address field (:rfc:`2427#section-3`), ' + 'not the TCP Data Offset octet', + ('pcapkit/protocols/schema/link/gre.py', "{'reserved_2': (4, 9)}"): + "the 9-bit span bits 4-12 leave of RFC 2784's 12-bit Reserved0 once RFC 2890 " + 'carves ``K``/``S`` out of it (:rfc:`2890#section-2`), not a PCAP-NG resolution-places ' + 'table', ('pcapkit/protocols/schema/misc/pcapng.py', 'BytesField(length=16)'): 'an IPv6 address is 16 octets', - ('pcapkit/protocols/schema/misc/pcapng.py', "pkt['length'] - 16"): - 'an IPv6 name record is 16 octets of address, not a block', + ('pcapkit/protocols/schema/misc/pcapng.py', 'BytesField(length=6)'): + 'a MAC address is 6 octets', ('pcapkit/protocols/schema/misc/pcapng.py', 'magic == 12'): 'the Block Total Length of a 12-octet Section Header Block read where the ' 'Byte-Order Magic belongs, spelled as the octets read to pair with the ' @@ -296,84 +345,51 @@ ('pcapkit/protocols/schema/misc/pcapng.py', 'magic in (BYTE_ORDER_MAGIC, 12)'): 'the same octets read where the Byte-Order Magic belongs, paired with the ' 'byte-swapped 0x0C000000 beside it, as in the check above', - ('pcapkit/protocols/schema/misc/pcapng.py', "{'reserved': (12, 12)}"): - 'an epb_flags bit-field: 12 reserved bits from bit 12', + ('pcapkit/protocols/schema/misc/pcapng.py', "pkt['length'] - 16"): + 'an IPv6 name record is 16 octets of address, not a block', + ('pcapkit/protocols/schema/misc/pcapng.py', "{'byteorder': (32, 32)}"): + "a bit-field: a Section Header Block's Byte-Order Magic, the 32 bits after " + 'its Block Total Length, not a Packet Block', ('pcapkit/protocols/schema/misc/pcapng.py', "{'crc_error': (24, 1)}"): 'an epb_flags bit offset, not an Interface Statistics Block', + ('pcapkit/protocols/schema/misc/pcapng.py', "{'reserved': (12, 12)}"): + 'an epb_flags bit-field: 12 reserved bits from bit 12', ('pcapkit/protocols/schema/misc/pcapng.py', "{'unaligned_error': (28, 1)}"): 'an epb_flags bit offset, not a Section Header Block', - ('pcapkit/protocols/schema/misc/pcapng.py', "{'byteorder': (32, 32)}"): - "a bit-field: a Section Header Block's Byte-Order Magic, the 32 bits after " - 'its Block Total Length, not a Packet Block', + ('pcapkit/protocols/schema/transport/tcp.py', 'BytesField(length=20)'): + 'an MPTCP MP_JOIN SYN/ACK truncated HMAC is 20 octets', ('pcapkit/protocols/schema/transport/tcp.py', "pkt['length'] > 12"): 'an MP_CAPABLE option length, not the Data Offset octet', ('pcapkit/protocols/schema/transport/tcp.py', "pkt['length'] > 20"): 'an MP_CAPABLE option length, not the minimum header length', - ('pcapkit/protocols/schema/transport/tcp.py', 'BytesField(length=20)'): - 'an MPTCP MP_JOIN SYN/ACK truncated HMAC is 20 octets', ('pcapkit/protocols/schema/transport/tcp.py', "pkt['length'] in (20, 22, 28, 30)"): 'the MP_ADD_ADDR option lengths with an IPv6 address, not the minimum header length', ('pcapkit/protocols/schema/transport/tcp.py', "{'reserved': (4, 12)}"): 'an MPTCP bit-field: 12 reserved bits after the 4-bit subtype, not the Data ' 'Offset octet', - ('pcapkit/protocols/transport/tcp.py', 'schema.length != 12'): - 'an MPTCP option length, not the Data Offset octet', - ('pcapkit/protocols/transport/tcp.py', 'schema.length > 12'): - 'an MP_CAPABLE option length, not the Data Offset octet', - ('pcapkit/protocols/transport/tcp.py', 'schema.length > 20'): - 'a field offset within the MP_CAPABLE option, not the minimum header length', - ('pcapkit/protocols/transport/tcp.py', 'Schema_MPTCPJoinSYN(length=12)'): - 'an MP_JOIN SYN option is 12 octets', + ('pcapkit/protocols/transport/tcp.py', '(4, 12, 20, 22, 24)[present]'): + 'the MP_CAPABLE option lengths, :rfc:`8684#section-3.1`', ('pcapkit/protocols/transport/tcp.py', 'Schema_MPTCPFallback(length=12)'): 'an MP_FAIL option is 12 octets', ('pcapkit/protocols/transport/tcp.py', 'Schema_MPTCPFastclose(length=12)'): 'an MP_FASTCLOSE option is 12 octets', + ('pcapkit/protocols/transport/tcp.py', 'Schema_MPTCPJoinSYN(length=12)'): + 'an MP_JOIN SYN option is 12 octets', ('pcapkit/protocols/transport/tcp.py', 'bytes(20)'): 'an MP_JOIN SYN/ACK HMAC default of 20 octets', - ('pcapkit/protocols/transport/tcp.py', - "{20: ModuleDescriptor('pcapkit.protocols.application.ftp_data', 'FTP_DATA')}"): - 'the FTP data port, not the minimum header length', + ('pcapkit/protocols/transport/tcp.py', 'schema.length != 12'): + 'an MPTCP option length, not the Data Offset octet', + ('pcapkit/protocols/transport/tcp.py', 'schema.length > 12'): + 'an MP_CAPABLE option length, not the Data Offset octet', + ('pcapkit/protocols/transport/tcp.py', 'schema.length > 20'): + 'a field offset within the MP_CAPABLE option, not the minimum header length', ('pcapkit/protocols/transport/tcp.py', 'schema.length not in (4, 12, 20, 22, 24)'): 'the MP_CAPABLE option lengths, :rfc:`8684#section-3.1`', - ('pcapkit/protocols/transport/tcp.py', '(4, 12, 20, 22, 24)[present]'): - 'the MP_CAPABLE option lengths, :rfc:`8684#section-3.1`', ('pcapkit/protocols/transport/tcp.py', 'schema.length not in (8, 10, 16, 18, 20, 22, 28, 30)'): 'the MP_ADD_ADDR option lengths, :rfc:`8684#section-3.4.1`', - ('pcapkit/protocols/protocol.py', '40 * 2 ** code'): - "the Quick-Start rate unit, K = 40 kbps (:rfc:`4782#section-3.1`), not the IPv6 " - 'header length', - ('pcapkit/protocols/protocol.py', '40 * 2 ** 16'): - "the first rate the Quick-Start rate code's four bits cannot carry, K * 2 ** 16 " - '(:rfc:`4782#section-3.1`), not the IPv6 header length', - ('pcapkit/protocols/protocol.py', '2 ** 16'): - "the exponent past the Quick-Start rate code's four bits, N = 16 " - '(:rfc:`4782#section-3.1`), not a PCAP record header length', - ('pcapkit/protocols/protocol.py', 'rate // 40'): - "the Quick-Start rate unit, K = 40 kbps (:rfc:`4782#section-3.1`), not the IPv6 " - 'header length', - ('pcapkit/foundation/engines/dpkt.py', 'pkt.ts_high << 32'): - 'a bit width: the high half of a 64-bit timestamp', - ('pcapkit/foundation/engines/dpkt.py', 'version == 6'): - 'the IP version number', - ('pcapkit/foundation/engines/dpkt.py', '{24: ip6}'): - 'the BSD loopback address family of IPv6 on NetBSD and OpenBSD', - ('pcapkit/foundation/engines/dpkt.py', '{28: ip6}'): - 'the BSD loopback address family of IPv6 on FreeBSD', - ('pcapkit/foundation/engines/pcap.py', 'head[8:12]'): - "a record header's incl_len field, octets 8 to 12", - ('pcapkit/foundation/engines/pcapng.py', 'self._peek(12)'): - "the first 12 octets of a block -- type, length and a Section Header Block's " - 'Byte-Order Magic -- not the generic block minimum', - ('pcapkit/foundation/engines/pcapng.py', 'head[8:12]'): - "the Byte-Order Magic field of a Section Header Block, octets 8 to 12", - ('pcapkit/foundation/engines/pcapng.py', 'self._peek(16)[12:16]'): - "a Decryption Secrets Block's Secrets Length field, octets 12 to 16", - ('pcapkit/foundation/engines/pcapng.py', 'self._peek(16)'): - "a Decryption Secrets Block up to its Secrets Length, not a block minimum", - ('pcapkit/foundation/engines/scapy.py', 'len(options) >= 12'): - 'an if_tsoffset option: 4 octets of header and 8 of value', - ('pcapkit/foundation/engines/scapy.py', 'options[4:12]'): - 'the 8-octet value of an if_tsoffset option', + ('pcapkit/protocols/transport/tcp.py', + "{20: ModuleDescriptor('pcapkit.protocols.application.ftp_data', 'FTP_DATA')}"): + 'the FTP data port, not the minimum header length', ('pcapkit/toolkit/dpkt.py', '_fragment(6)'): 'the IP version number', ('pcapkit/toolkit/dpkt.py', 'version == 6'): @@ -384,123 +400,109 @@ 'a MAC address is 6 octets', ('pcapkit/toolkit/pypcapfile.py', '{Enum_TransType.IPv6: 6}'): 'the IP version an IPv6-in-IP tunnel carries, not the Next Header offset', - ('pcapkit/dumpkit/common.py', 'ord(char) > 65535'): - 'the last code point of the Basic Multilingual Plane, not an IP length', - ('pcapkit/dumpkit/common.py', 'codepoint > 65535'): - 'above the Basic Multilingual Plane, needs a UTF-16 surrogate pair, not an IP length', - ('pcapkit/protocols/application/sslv2.py', "b'\\x00' * 16"): - "the default length of an SSLv2 CLIENT-HELLO challenge or SERVER-HELLO " - "connection ID (the draft's own 16-32 octet minimum), not the PCAP classic " - 'record header length', - ('pcapkit/protocols/link/gre.py', 'return 16'): - "GRE's own longest possible header -- the 4-octet base plus all three optional " - 'extensions at 4 octets each (:rfc:`2784`/:rfc:`2890`) -- not a PCAP record header', - ('pcapkit/protocols/schema/link/gre.py', "{'reserved_2': (4, 9)}"): - "the 9-bit span bits 4-12 leave of RFC 2784's 12-bit Reserved0 once RFC 2890 " - 'carves ``K``/``S`` out of it (:rfc:`2890#section-2`), not a PCAP-NG resolution-places ' - 'table', } #: Sweep hits of a homed value in a module whose own protocol owns no constant #: of that value, nor reads one: the module mapped to the values left there and #: why. A value not listed still fails, so a copy added later is still caught. +#: Sorted, so that entries for unrelated modules land apart (#2615). UNRELATED = { + 'pcapkit/foundation/engines/pypcapfile.py': ((12,), 'the Python 3.12 version ceiling of ' + 'PyPCAPFile, not a length'), 'pcapkit/protocols/application/http.py': ((6, 9, 12), 'the length of an HTTP/1 version ' 'token, and HTTP/2 frame types and flag masks'), 'pcapkit/protocols/application/httpv2.py': ((6, 9), 'HTTP/2 frame header and setting sizes'), - 'pcapkit/protocols/internet/ah.py': ((20,), "AH's length estimate, its 12-octet fixed part " - 'plus a minimal ICV, not the fixed part itself'), - 'pcapkit/protocols/schema/internet/ipv6_option.py': ((16,), 'an MPL Seed-ID of 16 octets'), - 'pcapkit/protocols/schema/application/httpv2.py': ((6, 32), - 'an HTTP/2 setting size and a flag bit'), 'pcapkit/protocols/application/ospf.py': ((16, 24), 'OSPF header length; 16 is the bit width of the ' 'packet length, and the offset of the ' 'Authentication field the checksum leaves out ' '(RFC 2328 appendix A.3.1), not a PCAP record ' 'header'), - 'pcapkit/protocols/schema/application/ospf.py': ((24,), 'the OSPF header length'), + 'pcapkit/protocols/internet/ah.py': ((20,), "AH's length estimate, its 12-octet fixed part " + 'plus a minimal ICV, not the fixed part itself'), 'pcapkit/protocols/internet/esp.py': ((12, 16, 20, 24, 32), 'ESP ICV and key sizes'), - 'pcapkit/protocols/internet/igmpv3.py': ((16,), "the floating-point Max Resp Code/QQIC " - "decode's mantissa mask (RFC 3376 section 4.1.1's " - "own 0x10), not the PCAP record header length"), - 'pcapkit/protocols/internet/igmp.py': ((12,), "the minimum overall length (RFC 3376 " - "section 7.1) at which a Membership Query carries " - "IGMPv3's source-list extension rather than being " - "IGMPv1/v2; unrelated to AH_HEADER_LEN, " - "TCP_DATA_OFFSET_OCTET or GENERIC_BLOCK_MINIMUM, " - "which happen to share the value"), 'pcapkit/protocols/internet/hip.py': ((6, 12, 16, 20, 32, 40, 65535), 'HIP parameter fields and sizes; 65535 bounds the ' 'two-octet TLV Length a TRANSACTION_ID or OVERLAY_ID ' 'may declare (#2271), as it does in the schema module, ' 'and is not an IP total length'), - 'pcapkit/protocols/schema/internet/hip.py': ((12, 16, 20, 65535), - 'HIP parameter fields, sizes and bit-fields'), 'pcapkit/protocols/internet/hopopt.py': ((6, 16, 32), 'HOPOPT option fields, widths and sizes'), - 'pcapkit/protocols/schema/internet/hopopt.py': ((6, 16), 'HOPOPT option field sizes ' - 'and bit-fields'), + 'pcapkit/protocols/internet/igmp.py': ((12,), "the minimum overall length (RFC 3376 " + "section 7.1) at which a Membership Query carries " + "IGMPv3's source-list extension rather than being " + "IGMPv1/v2; unrelated to AH_HEADER_LEN, " + "TCP_DATA_OFFSET_OCTET or GENERIC_BLOCK_MINIMUM, " + "which happen to share the value"), + 'pcapkit/protocols/internet/igmpv3.py': ((16,), "the floating-point Max Resp Code/QQIC " + "decode's mantissa mask (RFC 3376 section 4.1.1's " + "own 0x10), not the PCAP record header length"), + 'pcapkit/protocols/internet/ipv4.py': ((9, 12, 16, 40), 'IPv4 option counts and sizes; ' + '40 is the option area maximum, not an IPv6 header'), + 'pcapkit/protocols/internet/ipv6.py': ((16,), 'a hex base and an IPv6 address size'), + 'pcapkit/protocols/internet/ipv6_ext.py': ((6,), 'the IP version number'), 'pcapkit/protocols/internet/ipv6_opts.py': ((6, 16, 32), 'IPv6-Opts option fields, widths and sizes'), - 'pcapkit/protocols/schema/internet/ipv6_opts.py': ((6, 16), 'IPv6-Opts option field sizes ' - 'and bit-fields'), - 'pcapkit/protocols/internet/ipv6_ext.py': ((6,), 'the IP version number'), 'pcapkit/protocols/internet/ipv6_route.py': ((6, 16), 'Routing header address sizes'), - 'pcapkit/protocols/schema/internet/ipv6_route.py': ((16, 20), 'an IPv6 address is 16 ' - 'octets; a 20-bit reserved bit-field'), - 'pcapkit/protocols/schema/internet/ipv4.py': ((6, 16), 'the ECN and option-function ' - 'bit-field offsets'), - 'pcapkit/protocols/schema/internet/ipv6.py': ((12, 20), 'the 20-bit Flow Label from bit 12 ' - 'of the first 32 bits'), - 'pcapkit/protocols/internet/ipv6.py': ((16,), 'a hex base and an IPv6 address size'), - 'pcapkit/protocols/internet/ipv4.py': ((9, 12, 16, 40), 'IPv4 option counts and sizes; ' - '40 is the option area maximum, not an IPv6 header'), 'pcapkit/protocols/internet/ipx.py': ((12,), 'an IPX address is 12 octets'), - 'pcapkit/protocols/schema/internet/ipx.py': ((12,), 'an IPX address is 12 octets'), 'pcapkit/protocols/internet/mh.py': ((6, 16, 20, 24, 32), 'Mobility Header option fields and sizes'), - 'pcapkit/protocols/schema/internet/mh.py': ((6, 9, 12, 16, 20, 24, 32), - 'Mobility Header option fields, sizes and bit-fields'), 'pcapkit/protocols/link/arp.py': ((6, 9, 16, 28), 'address sizes, InARP operation codes, ' 'and the 28-octet IPv4 ARP length'), - 'pcapkit/protocols/link/loopback.py': ((24, 28), 'the BSD loopback address families of IPv6 ' - '(NetBSD and OpenBSD 24, FreeBSD 28)'), 'pcapkit/protocols/link/ieee802_11.py': ((6, 9, 12), 'a 6-octet MAC address, and ' 'Frame Control bit offsets'), 'pcapkit/protocols/link/l2tpv2.py': ((6, 12, 16), 'L2TPv2 header flags, widths and length'), 'pcapkit/protocols/link/linux_sll1.py': ((65535,), "the max value of LINUX_SLL's own " '2-octet link_addr_length field, not an IP total ' 'length'), + 'pcapkit/protocols/link/loopback.py': ((24, 28), 'the BSD loopback address families of IPv6 ' + '(NetBSD and OpenBSD 24, FreeBSD 28)'), 'pcapkit/protocols/link/netlink.py': ((16,), 'the nlmsghdr fixed header length (16 ' 'octets), not a PCAP record header'), 'pcapkit/protocols/link/radiotap.py': ((65535,), "the max value of radiotap's own " '2-octet it_len field, not an IP total length'), + 'pcapkit/protocols/misc/pcapng.py': ((65535,), 'a 16-bit mask'), + 'pcapkit/protocols/schema/application/httpv2.py': ((6, 32), + 'an HTTP/2 setting size and a flag bit'), + 'pcapkit/protocols/schema/application/ospf.py': ((24,), 'the OSPF header length'), + 'pcapkit/protocols/schema/internet/hip.py': ((12, 16, 20, 65535), + 'HIP parameter fields, sizes and bit-fields'), + 'pcapkit/protocols/schema/internet/hopopt.py': ((6, 16), 'HOPOPT option field sizes ' + 'and bit-fields'), + 'pcapkit/protocols/schema/internet/ipv4.py': ((6, 16), 'the ECN and option-function ' + 'bit-field offsets'), + 'pcapkit/protocols/schema/internet/ipv6.py': ((12, 20), 'the 20-bit Flow Label from bit 12 ' + 'of the first 32 bits'), + 'pcapkit/protocols/schema/internet/ipv6_option.py': ((16,), 'an MPL Seed-ID of 16 octets'), + 'pcapkit/protocols/schema/internet/ipv6_opts.py': ((6, 16), 'IPv6-Opts option field sizes ' + 'and bit-fields'), + 'pcapkit/protocols/schema/internet/ipv6_route.py': ((16, 20), 'an IPv6 address is 16 ' + 'octets; a 20-bit reserved bit-field'), + 'pcapkit/protocols/schema/internet/ipx.py': ((12,), 'an IPX address is 12 octets'), + 'pcapkit/protocols/schema/internet/mh.py': ((6, 9, 12, 16, 20, 24, 32), + 'Mobility Header option fields, sizes and bit-fields'), 'pcapkit/protocols/schema/link/ethernet.py': ((6,), 'a MAC address is 6 octets'), 'pcapkit/protocols/schema/link/ieee802_11.py': ((6, 9), 'a MAC address is 6 octets, and ' 'the From DS bit offset of Frame Control'), 'pcapkit/protocols/schema/link/l2tp.py': ((6, 12), 'an L2TP header field offset, and the ' 'version bit-field'), 'pcapkit/protocols/schema/link/vlan.py': ((12,), 'the 12-bit VLAN identifier'), - 'pcapkit/protocols/misc/pcapng.py': ((65535,), 'a 16-bit mask'), 'pcapkit/protocols/schema/misc/pcapng.py': ((9, 65535), 'an epb_flags bit offset, and a ' '16-bit field default'), - 'pcapkit/protocols/transport/sctp.py': ((12, 16, 20), 'SCTP chunk and parameter sizes'), + 'pcapkit/protocols/schema/transport/dccp.py': ((6,), 'a 48-bit DCCP sequence or ' + 'acknowledgement number, and a Timestamp Echo ' + 'option without its Elapsed Time'), 'pcapkit/protocols/schema/transport/sctp.py': ((6, 16, 20), 'SCTP chunk and parameter sizes, ' 'and a flag bit'), + 'pcapkit/protocols/schema/transport/tcp.py': ((6, 16, 28), 'TCP and MPTCP option sizes, and ' + 'the MP_ADD_ADDR lengths'), 'pcapkit/protocols/transport/dccp.py': ((6, 65535), 'DCCP option lengths, the 2-bit shift of ' 'an Ack Vector state, and the largest 2-octet ' 'Elapsed Time'), - 'pcapkit/protocols/schema/transport/dccp.py': ((6,), 'a 48-bit DCCP sequence or ' - 'acknowledgement number, and a Timestamp Echo ' - 'option without its Elapsed Time'), + 'pcapkit/protocols/transport/sctp.py': ((12, 16, 20), 'SCTP chunk and parameter sizes'), 'pcapkit/protocols/transport/tcp.py': ((6, 16, 24, 32, 65535), 'TCP and MPTCP option sizes, ' 'and the 16-bit window and MSS defaults'), - 'pcapkit/protocols/schema/transport/tcp.py': ((6, 16, 28), 'TCP and MPTCP option sizes, and ' - 'the MP_ADD_ADDR lengths'), 'pcapkit/toolkit/pyshark.py': ((6, 9, 20, 24, 28), 'Wireshark encapsulation numbers mapped ' 'to link types'), - 'pcapkit/foundation/engines/pypcapfile.py': ((12,), 'the Python 3.12 version ceiling of ' - 'PyPCAPFile, not a length'), } #: Values equal to one of the homes' but meaning something else, so left where @@ -837,6 +839,13 @@ def test_every_entry_has_a_reason(self) -> 'None': reasons = list(NOT_A_COPY.values()) + [reason for _, reason in UNRELATED.values()] self.assertTrue(all(reason.strip() for reason in reasons)) + def test_the_tables_are_sorted(self) -> 'None': + # Sorted by module, so that entries for unrelated modules land apart + # rather than at a shared tail (#2615). + for name, table in (('NOT_A_COPY', NOT_A_COPY), ('UNRELATED', UNRELATED)): + with self.subTest(table=name): + self.assertEqual(list(table), sorted(table)) + def test_distinct_values_stay_apart(self) -> 'None': homes = {name for _, name, _ in HOMES} for module, name, value in DISTINCT: diff --git a/tests/protocols/internet/test_hip_from_data_alias_unit.py b/tests/protocols/internet/test_hip_from_data_alias_unit.py index 2705114f2e..1da8208f4f 100644 --- a/tests/protocols/internet/test_hip_from_data_alias_unit.py +++ b/tests/protocols/internet/test_hip_from_data_alias_unit.py @@ -25,7 +25,7 @@ HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) #: Header fields shared by every HIP packet built here, as in -#: :data:`examples.generators.options.HIP_BASE`. +#: :data:`examples.generators.option_families.hip.HIP_BASE`. HIP_BASE = { 'next': 59, 'packet': 1, 'checksum': b'\x00\x00', 'controls_anonymous': False, 'shit': 0, 'rhit': 0, 'payload': b'', diff --git a/tests/protocols/internet/test_hip_host_id_di_width_unit.py b/tests/protocols/internet/test_hip_host_id_di_width_unit.py index 04c94d437a..9096b18740 100644 --- a/tests/protocols/internet/test_hip_host_id_di_width_unit.py +++ b/tests/protocols/internet/test_hip_host_id_di_width_unit.py @@ -24,7 +24,7 @@ HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) #: Header fields shared by every HIP packet built here, as in -#: :data:`examples.generators.options.HIP_BASE`. +#: :data:`examples.generators.option_families.hip.HIP_BASE`. HIP_BASE = { 'next': 6, 'packet': 1, 'checksum': b'\x00\x00', 'controls_anonymous': False, 'shit': 0, 'rhit': 0, 'payload': b'', diff --git a/tests/protocols/internet/test_hip_list_argument_str_names_2394_unit.py b/tests/protocols/internet/test_hip_list_argument_str_names_2394_unit.py index 0c70f23335..c3684c3e9b 100644 --- a/tests/protocols/internet/test_hip_list_argument_str_names_2394_unit.py +++ b/tests/protocols/internet/test_hip_list_argument_str_names_2394_unit.py @@ -53,7 +53,7 @@ } #: HIP version each version-bound parameter demands: ``HIP_TRANSFORM`` (577) is -#: :rfc:`5201` only, as :data:`examples.generators.options.HIP_VERSION` records, +#: :rfc:`5201` only, as :data:`examples.generators.option_families.hip.HIP_VERSION` records, #: and the rest are refused at version 1 or indifferent to it. HIP_VERSION = {577: 1} diff --git a/tests/protocols/internet/test_hip_locator_set_length_unit.py b/tests/protocols/internet/test_hip_locator_set_length_unit.py index 1c5fa03f8f..7bcf81568f 100644 --- a/tests/protocols/internet/test_hip_locator_set_length_unit.py +++ b/tests/protocols/internet/test_hip_locator_set_length_unit.py @@ -83,7 +83,7 @@ HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) #: Header fields shared by every HIP packet built here, as in -#: :data:`examples.generators.options.HIP_BASE`. +#: :data:`examples.generators.option_families.hip.HIP_BASE`. HIP_BASE = { 'next': 6, 'packet': 1, 'version': 2, 'checksum': b'\x00\x00', 'controls_anonymous': False, 'shit': 0, 'rhit': 0, 'payload': b'', diff --git a/tests/protocols/internet/test_hip_param_data_rebuild_2352_unit.py b/tests/protocols/internet/test_hip_param_data_rebuild_2352_unit.py index 4472e30632..7eddf48232 100644 --- a/tests/protocols/internet/test_hip_param_data_rebuild_2352_unit.py +++ b/tests/protocols/internet/test_hip_param_data_rebuild_2352_unit.py @@ -70,7 +70,7 @@ #: HIP version each version-bound parameter demands: ``HIP_TRANSFORM`` (577) #: and ``R1_Counter`` (128) are :rfc:`5201` only, as -#: :data:`examples.generators.options.HIP_VERSION` records. +#: :data:`examples.generators.option_families.hip.HIP_VERSION` records. HIP_VERSION = {128: 1, 577: 1} #: Fields a constructor derives rather than accepting, so that a data model diff --git a/tests/protocols/internet/test_hip_r1_counter_width_unit.py b/tests/protocols/internet/test_hip_r1_counter_width_unit.py index 3255bb29cd..e9f5fd3a19 100644 --- a/tests/protocols/internet/test_hip_r1_counter_width_unit.py +++ b/tests/protocols/internet/test_hip_r1_counter_width_unit.py @@ -68,7 +68,7 @@ HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) #: Header fields shared by every HIP packet built here. The same set -#: :data:`examples.generators.options.HIP_BASE` uses, so a failure here is not a +#: :data:`examples.generators.option_families.hip.HIP_BASE` uses, so a failure here is not a #: failure to drive the constructor. HIP_BASE = { 'next': 6, 'packet': 1, 'checksum': b'\x00\x00', @@ -224,7 +224,7 @@ def test_the_generator_gives_r1_counter_a_non_zero_counter(self) -> None: from tests.protocols.test_option_roundtrip_unit import _load_generator - overrides = _load_generator()._hip_overrides() + overrides = _load_generator().FAMILY_MODULES['hip']._hip_overrides() for code in (Parameter.R1_Counter, Parameter.R1_COUNTER): with self.subTest(code=int(code), name=code.name): self.assertIn(code, overrides) diff --git a/tests/protocols/link/test_l2tp_version_unit.py b/tests/protocols/link/test_l2tp_version_unit.py index 759cea66ec..1a4c7e99f8 100644 --- a/tests/protocols/link/test_l2tp_version_unit.py +++ b/tests/protocols/link/test_l2tp_version_unit.py @@ -49,7 +49,7 @@ def l2tp_data(version: 'int' = 2) -> bytes: """An L2TPv2 data message with every optional field absent. Matches :func:`tests.protocols.test_dispatch_bindings_unit.l2tp_data` and - :data:`examples.generators.dispatch._L2TP_DATA` at ``version=2``; the + :data:`examples.generators.dispatch_families.udp._L2TP_DATA` at ``version=2``; the parameter exists so a case can put a *different* nibble in bits 12-15 while holding every other octet identical. diff --git a/tests/protocols/test_construction_keyword_check_unit.py b/tests/protocols/test_construction_keyword_check_unit.py index 50a96b8f87..d5d92504ec 100644 --- a/tests/protocols/test_construction_keyword_check_unit.py +++ b/tests/protocols/test_construction_keyword_check_unit.py @@ -8,7 +8,7 @@ nothing says so, and the mapping that produced them reads correctly. The cost is not hypothetical. Issue #602 was -:data:`examples.generators.options.TCP_BASE` asking for ``seq=1`` where +:data:`examples.generators.option_families.tcp.TCP_BASE` asking for ``seq=1`` where :meth:`TCP.make ` spells the parameter ``seq_no``. Measured without the fix:: @@ -328,7 +328,7 @@ def test_hip_declares_extension_on_read_and_not_on_make(self) -> None: ``extension``, :meth:`HIP.make ` does not, and :meth:`HIP.__post_init__ ` forwards it to both -- - so ``examples/generators/options.py``'s ``_hip_build`` passing + so ``examples/generators/option_families/hip.py``'s ``_hip_build`` passing ``extension=True`` on the construction path is correct code that a check against ``make`` alone would have broken. Asserted from the signatures, so it fails if either side of the asymmetry moves. diff --git a/tests/protocols/test_dispatch_registry_unit.py b/tests/protocols/test_dispatch_registry_unit.py index d22580347c..80a251da56 100644 --- a/tests/protocols/test_dispatch_registry_unit.py +++ b/tests/protocols/test_dispatch_registry_unit.py @@ -26,7 +26,10 @@ """ from __future__ import annotations +import collections +import importlib import importlib.util +import pkgutil import sys import types import unittest @@ -48,6 +51,44 @@ #: milliseconds; the point of the deadline is a case that never finishes. CASE_TIMEOUT = 30 +#: The non-empty ``__proto__`` tables no family probes, by ``module.Class`` +#: path, and why. Every other one is some family's registry (#2615). +UNCOVERED = { + 'pcapkit.protocols.link.linux_sll.LinuxSLL.__proto__': + 'its ARPHRD entries are pinned as shipped and probed one by one by ' + 'tests/protocols/link/test_linux_sll_arphrd_registry_unit.py (#2510)', +} + + +def _proto_tables() -> 'dict[str, dict[Any, Any]]': + """Every non-empty ``__proto__`` table a protocol class defines itself, by path. + + Found by walking every :class:`~pcapkit.protocols.protocol.ProtocolBase` + descendant once every shipped protocol module is imported, as + :mod:`tests.foundation.registry.test_registry_wrapper_coverage_2511_unit` + finds its maps. + + """ + import pcapkit.protocols as package + from pcapkit.protocols.protocol import ProtocolBase + + for info in pkgutil.walk_packages(package.__path__, 'pcapkit.protocols.'): + if 'NotImplemented' not in info.name: + importlib.import_module(info.name) + found = {} # type: dict[str, dict[Any, Any]] + seen = set() # type: set[type] + pending = [ProtocolBase] # type: list[type] + while pending: + klass = pending.pop() + if klass in seen: + continue + seen.add(klass) + pending.extend(klass.__subclasses__()) + table = vars(klass).get('__proto__') + if isinstance(table, dict) and table: + found[f'{klass.__module__}.{klass.__qualname__}.__proto__'] = table + return found + class Degrade(NamedTuple): """One code known to degrade to :class:`~pcapkit.protocols.misc.raw.Raw`.""" @@ -160,7 +201,7 @@ def test_every_registered_code_has_a_case(self) -> None: self.assertEqual( set(registry), covered, f'{family.label}: every code in the registry needs a case in ' - f'examples/generators/dispatch.py' + f'its module under examples/generators/dispatch_families/' ) def test_every_case_has_a_pinned_target(self) -> None: @@ -195,7 +236,7 @@ def test_registry_matches_pinned_target(self) -> None: pointing ``Internet.__proto__[TransType.AH]`` at :class:`~pcapkit.protocols.misc.raw.Raw` instead of :class:`~pcapkit.protocols.internet.ah.AH` changes nothing - :meth:`test_cases_cover_every_table_named_in_the_issue` looks at, nor + :meth:`test_cases_match_the_pinned_targets` looks at, nor anything a registry-derived dispatch check would, and both pass. This test does not: it compares :attr:`Case.registered` -- what the table says at run time -- against @@ -215,41 +256,68 @@ def test_registry_matches_pinned_target(self) -> None: f'deliberate change.' ) - def test_cases_cover_every_table_named_in_the_issue(self) -> None: - """The enumeration finds every entry, across the tables of #496, #1574 and #2560. + def test_cases_match_the_pinned_targets(self) -> None: + """The enumeration finds exactly the entries :data:`PINNED_TARGETS` records. - A guard on the shape of the result rather than on any one case: if the - registries grow or shrink without this module noticing, the per-family - counts below catch it even if every individual case still passes. + A guard on the shape of the result rather than on any one case: a + table that loses an entry, or a family whose table goes missing, leaves + a pinned target that no case reaches, and a table that gains one leaves + a case that nothing pins -- even if every individual case still + passes. Each family module records its own table's entries beside the + family (#2615), so the expected shape is derived from those records + rather than pinned here as counts, and a failure names the entry. """ - cases = self.dispatch.cases() - # The tables of #496, including #904's Internet.__proto__[TransType.Shim6] - # registered at IPv6_Ext rather than defaulting to Raw, plus #1574's - # LinkType.NULL and LinkType.LOOP in both link-type tables, the four - # address families of Loopback.__proto__, #2438's - # Frame.__proto__[LinkType.LINUX_SLL]/[LinkType.LINUX_SLL2] in both the - # pcap and pcapng frame tables, #2439's IGMP, #2436's - # Internet.__proto__[TransType.ICMP] and [TransType.IPv6_ICMP], #2489's - # Internet.__proto__[TransType.DCCP], #2560's two NLPIDs of - # FRAD.__proto__, #2575's Radiotap.__proto__[LinkType.IEEE802_11], and - # #2608's LinkType.IEEE802_11 and LinkType.IEEE802_11_RADIOTAP in both - # link-type tables. - self.assertEqual(len(cases), 64, - 'expected exactly 64 entries across the seven __proto__ ' - 'tables named in GitHub issue #496, the loopback table ' - 'of #1574, the FRAD table of #2560 and the radiotap table ' - 'of #2575; a different count means a registry changed ' - 'shape and this module was not updated to match') - - by_family = {} # type: dict[str, int] - for case in cases: - by_family[case.family] = by_family.get(case.family, 0) + 1 - self.assertEqual(by_family, { - 'link': 7, 'internet': 21, 'tcp': 5, 'udp': 4, 'sctp': 2, - 'pcap-frame': 9, 'pcapng-frame': 9, 'loopback': 4, 'frad': 2, - 'radiotap': 1, - }) + labels = [case.label for case in self.dispatch.cases()] + pinned = set(self.dispatch.PINNED_TARGETS) + found = collections.Counter(labels) + self.assertEqual( + {'found twice': sorted(label for label, count in found.items() if count > 1), + 'not pinned': sorted(set(found) - pinned), + 'pinned but not found': sorted(pinned - set(found))}, + {'found twice': [], 'not pinned': [], 'pinned but not found': []}, + 'the __proto__ tables and the PINNED_TARGETS of their modules under ' + 'examples/generators/dispatch_families/ disagree; a registry changed ' + 'shape and its family module was not updated to match') + + families = {family.label for family in self.dispatch.FAMILIES} + self.assertEqual(sorted(families - {case.family for case in self.dispatch.cases()}), [], + 'a family whose table holds no entry probes nothing') + + def test_each_family_module_pins_its_own_families(self) -> None: + """A family module pins its own families' entries, under labels no other module uses. + + Merging the modules' tables would otherwise let one module's entry + replace another's without a word, and a family's entries filed in the + wrong module would put two changes to one family in two files. + + """ + labels = [family.label for family in self.dispatch.FAMILIES] + self.assertEqual(sorted(labels), sorted(set(labels)), 'a family label is used twice') + for stem, module in self.dispatch.FAMILY_MODULES.items(): + own = {family.label for family in module.FAMILIES} + with self.subTest(module=stem): + self.assertTrue(own, f'{stem} adds no family') + strays = sorted(label for label in module.PINNED_TARGETS + if label.split('/', 1)[0] not in own) + self.assertEqual(strays, [], f'{stem} pins entries of another module\'s family') + + def test_every_dispatch_table_has_a_family(self) -> None: + """Every non-empty ``__proto__`` table is a family's registry, or in :data:`UNCOVERED`. + + Derived from the tree's own tables rather than from the family modules, + so a family module that goes missing, its pinned targets with it, still + fails here, as does a table no family probes. Checked both ways, so an + :data:`UNCOVERED` entry whose table gains a family, or empties, is stale. + + """ + covered = {id(family.registry()) for family in self.dispatch.FAMILIES} + uncovered = sorted(path for path, table in _proto_tables().items() + if id(table) not in covered) + self.assertEqual(uncovered, sorted(UNCOVERED), + 'a non-empty __proto__ table needs a family module under ' + 'examples/generators/dispatch_families/, or an UNCOVERED ' + 'entry saying what covers it instead') def test_dispatch_reaches_target_or_is_a_recorded_degrade(self) -> None: """Every code either dispatches to its registered class, or degrades exactly as recorded. diff --git a/tests/protocols/test_option_generator_tcp_base_unit.py b/tests/protocols/test_option_generator_tcp_base_unit.py index 5be79adf18..f56759e270 100644 --- a/tests/protocols/test_option_generator_tcp_base_unit.py +++ b/tests/protocols/test_option_generator_tcp_base_unit.py @@ -2,8 +2,8 @@ """``TCP_BASE`` builds the segment it describes. Every frame in :file:`examples/captures/options-tcp.pcap` is a TCP segment whose -header comes from :data:`examples.generators.options.TCP_BASE` and whose only -per-case difference is the one option it carries. So that mapping is the +header comes from :data:`examples.generators.option_families.tcp.TCP_BASE` and +whose only per-case difference is the one option it carries. So that mapping is the description of every one of those fixture frames, and a key it spells wrongly is a description that does not match the octets. @@ -202,7 +202,7 @@ def test_every_key_is_a_parameter_make_declares(self) -> None: parameter.KEYWORD_ONLY) } - undeclared = sorted(set(options.TCP_BASE) - declared) + undeclared = sorted(set(options.FAMILY_MODULES['tcp'].TCP_BASE) - declared) self.assertEqual(undeclared, [], ( f'TCP_BASE passes {undeclared} to TCP.make, which does not declare ' f'them; they are absorbed by **kwargs and silently dropped, so the ' @@ -242,7 +242,7 @@ def test_tcp_base_is_the_expected_header(self) -> None: """ options = _load_generator() - self.assertEqual(dict(options.TCP_BASE), EXPECTED_HEADER) + self.assertEqual(dict(options.FAMILY_MODULES['tcp'].TCP_BASE), EXPECTED_HEADER) def test_an_undeclared_keyword_is_refused_rather_than_absorbed(self) -> None: """Constructing with an undeclared keyword raises. @@ -298,7 +298,7 @@ def test_each_tcp_base_misspelling_is_refused(self) -> None: options = _load_generator() for wrong, right in (('seq', 'seq_no'), ('ack_flag', 'ack'), ('urgent_pointer', 'urgent')): - keywords = dict(options.TCP_BASE) + keywords = dict(options.FAMILY_MODULES['tcp'].TCP_BASE) keywords[wrong] = keywords.pop(right) with self.subTest(keyword=wrong): @@ -330,7 +330,7 @@ def _build(self, **overrides: 'Any') -> 'Any': """ from pcapkit.protocols.transport.tcp import TCP - return TCP(**{**self.options.TCP_BASE, **overrides}) + return TCP(**{**self.options.FAMILY_MODULES['tcp'].TCP_BASE, **overrides}) def test_the_header_fields_are_the_ones_expected(self) -> None: """Every value the mapping states reaches the field it names. diff --git a/tests/protocols/test_option_roundtrip_unit.py b/tests/protocols/test_option_roundtrip_unit.py index 74280ecd11..a6f6640989 100644 --- a/tests/protocols/test_option_roundtrip_unit.py +++ b/tests/protocols/test_option_roundtrip_unit.py @@ -71,7 +71,9 @@ from __future__ import annotations +import importlib import importlib.util +import pkgutil import sys import types import unittest @@ -94,6 +96,43 @@ #: never finish at all. CASE_TIMEOUT = 30 +#: The non-empty option-like maps no family exercises, by ``module.Class.__map__`` +#: path, and why. Empty: every one is some family's registry (#2615). +UNCOVERED = {} # type: dict[str, str] + + +def _option_registries() -> 'dict[str, dict[Any, Any]]': + """Every non-empty registry map a protocol class defines itself, by path, but ``__proto__``. + + The dunder :class:`dict` maps in each + :class:`~pcapkit.protocols.protocol.ProtocolBase` descendant's own + :func:`vars`, found once every shipped protocol module is imported, as + :mod:`tests.foundation.registry.test_registry_wrapper_coverage_2511_unit` + finds them. ``__proto__`` is the dispatch harness's, in + :mod:`tests.protocols.test_dispatch_registry_unit`. + + """ + import pcapkit.protocols as package + from pcapkit.protocols.protocol import ProtocolBase + + for info in pkgutil.walk_packages(package.__path__, 'pcapkit.protocols.'): + if 'NotImplemented' not in info.name: + importlib.import_module(info.name) + found = {} # type: dict[str, dict[Any, Any]] + seen = set() # type: set[type] + pending = [ProtocolBase] # type: list[type] + while pending: + klass = pending.pop() + if klass in seen: + continue + seen.add(klass) + pending.extend(klass.__subclasses__()) + for name, value in vars(klass).items(): + if (name.startswith('__') and name.endswith('__') and isinstance(value, dict) + and name not in ('__annotations__', '__proto__') and value): + found[f'{klass.__module__}.{klass.__qualname__}.{name}'] = value + return found + class Gap(NamedTuple): """One cycle that does not close, and the defect that stops it.""" @@ -135,8 +174,8 @@ class Gap(NamedTuple): # :mod:`tests.protocols.transport.test_tcp_mptcp_length_arithmetic_unit` # (#576); # * the three MP_JOIN layouts of RFC 8684 section 3.2, of which this suite - # reaches only the SYN form that ``examples.generators.options``' - # ``TCP_BASE`` selects, in + # reaches only the SYN form that the generator's ``TCP_BASE`` + # (examples/generators/option_families/tcp.py) selects, in # :mod:`tests.protocols.transport.test_tcp_mptcp_join_flag_ordering_unit` # (#587); # * the IPv4 ``SID`` option width, against wire octets, in @@ -266,8 +305,9 @@ def test_every_registered_code_has_a_case(self) -> None: } self.assertEqual( set(registry), covered | skipped, - f'{family.label}: every code in the registry needs a case in ' - f'examples/generators/options.py, or an entry in its SKIP table' + f'{family.label}: every code in the registry needs a case, or ' + f'an entry in the SKIP table of its module under ' + f'examples/generators/option_families/' ) def test_every_registered_code_resolves_without_growing_the_registry(self) -> None: @@ -324,6 +364,41 @@ def test_expected_failures_name_real_cases(self) -> None: f'exist; delete them, or fix the label' ) + def test_each_family_module_owns_its_entries(self) -> None: + """A family module adds families no other labels, and skips only their codes. + + The modules under :file:`examples/generators/option_families/` are + merged into ``FAMILIES`` and ``SKIP`` (#2615), and a merge would + otherwise let one module's family or entry replace another's without a + word. + + """ + labels = [family.label for family in self.options.FAMILIES] + self.assertEqual(sorted(labels), sorted(set(labels)), 'a family label is used twice') + for stem, module in self.options.FAMILY_MODULES.items(): + own = {family.label for family in module.FAMILIES} + with self.subTest(module=stem): + self.assertTrue(own, f'{stem} adds no family') + strays = sorted(key for key in getattr(module, 'SKIP', {}) if key[0] not in own) + self.assertEqual(strays, [], f"{stem} skips codes of another module's family") + + def test_every_option_registry_has_a_family(self) -> None: + """Every non-empty option-like map is a family's registry, or in :data:`UNCOVERED`. + + Derived from the tree's own maps rather than from the family modules, + so a family module that goes missing still fails here, as does a map + no family exercises (#2615). Checked both ways, so an + :data:`UNCOVERED` entry whose map gains a family, or empties, is stale. + + """ + covered = {id(family.registry()) for family in self.options.FAMILIES} + uncovered = sorted(path for path, table in _option_registries().items() + if id(table) not in covered) + self.assertEqual(uncovered, sorted(UNCOVERED), + 'a non-empty registry map needs a family in a module under ' + 'examples/generators/option_families/, or an UNCOVERED ' + 'entry saying what covers it instead') + def test_round_trip_is_identity_or_a_recorded_gap(self) -> None: """Every case either closes the cycle, or fails exactly as recorded. @@ -408,7 +483,7 @@ def test_a_hip_packet_carrying_one_parameter_round_trips(self) -> None: from pcapkit.const.hip.parameter import Parameter from pcapkit.protocols.internet.hip import HIP - base = dict(self.options.HIP_BASE) + base = dict(self.options.FAMILY_MODULES['hip'].HIP_BASE) one = [(Parameter.SEQ, {})] # type: list[tuple[Any, dict[str, Any]]] for copies in (1, 2): diff --git a/tests/protocols/transport/test_tcp_mptcp_capable_length_unit.py b/tests/protocols/transport/test_tcp_mptcp_capable_length_unit.py index 6e162f20f9..a71c47ad99 100644 --- a/tests/protocols/transport/test_tcp_mptcp_capable_length_unit.py +++ b/tests/protocols/transport/test_tcp_mptcp_capable_length_unit.py @@ -229,7 +229,7 @@ class TCPMPTCPCapablePublicConstructorUnitTests(unittest.TestCase): """Both RFC 8684 forms round-trip through the public ``TCP`` convenience constructor.""" #: Header fields shared by every constructed TCP segment in this class, matching - #: :data:`examples.generators.options.TCP_BASE`. + #: :data:`examples.generators.option_families.tcp.TCP_BASE`. TCP_BASE = { 'srcport': 50000, 'dstport': 80, 'seq_no': 1, 'ack_no': 0, 'ns': False, 'cwr': False, 'ece': False, 'urg': False, 'ack': False, diff --git a/tests/protocols/transport/test_tcp_mptcp_error_message_unit.py b/tests/protocols/transport/test_tcp_mptcp_error_message_unit.py index ff16a087fb..3deadef06c 100644 --- a/tests/protocols/transport/test_tcp_mptcp_error_message_unit.py +++ b/tests/protocols/transport/test_tcp_mptcp_error_message_unit.py @@ -49,7 +49,7 @@ #: Header fields for a segment built through the public constructor, spelled with #: the parameter names :meth:`TCP.make ` #: actually declares. The same mapping, and the same reason for not reusing -#: :data:`examples.generators.options.TCP_BASE`, as +#: :data:`examples.generators.option_families.tcp.TCP_BASE`, as #: :data:`tests.protocols.transport.test_tcp_mptcp_join_flag_ordering_unit.TCP_HEADER`. TCP_HEADER = { 'srcport': 50000, 'dstport': 80, 'seq_no': 1, 'ack_no': 0, diff --git a/tests/protocols/transport/test_tcp_mptcp_join_flag_ordering_unit.py b/tests/protocols/transport/test_tcp_mptcp_join_flag_ordering_unit.py index d23dbd0584..9db7504d97 100644 --- a/tests/protocols/transport/test_tcp_mptcp_join_flag_ordering_unit.py +++ b/tests/protocols/transport/test_tcp_mptcp_join_flag_ordering_unit.py @@ -122,7 +122,7 @@ #: Header fields shared by every constructed segment here, spelled with the parameter #: names :meth:`TCP.make ` actually declares. #: -#: Deliberately *not* a copy of :data:`examples.generators.options.TCP_BASE`, which +#: Deliberately *not* a copy of :data:`examples.generators.option_families.tcp.TCP_BASE`, which #: sibling modules in this directory mirror -- and, unlike theirs, this dict omits #: ``syn``/``ack`` altogether, since every case here passes those two explicitly to #: select a layout, where a shared constant could not. The names matter: ``ack`` is the diff --git a/tests/protocols/transport/test_tcp_mptcp_length_arithmetic_unit.py b/tests/protocols/transport/test_tcp_mptcp_length_arithmetic_unit.py index 52771910bd..54f8dd03bf 100644 --- a/tests/protocols/transport/test_tcp_mptcp_length_arithmetic_unit.py +++ b/tests/protocols/transport/test_tcp_mptcp_length_arithmetic_unit.py @@ -135,7 +135,7 @@ class level, through a :class:`~pcapkit.corekit.fields.misc.SwitchField` -- the __tier__ = 'ALL' #: Header fields shared by the constructed TCP segments here, matching -#: :data:`examples.generators.options.TCP_BASE` and +#: :data:`examples.generators.option_families.tcp.TCP_BASE` and #: :data:`tests.protocols.transport.test_tcp_mptcp_subtype_unit.TCP_BASE`. TCP_BASE = { 'srcport': 50000, 'dstport': 80, 'seq_no': 1, 'ack_no': 0, diff --git a/tests/protocols/transport/test_tcp_mptcp_subtype_unit.py b/tests/protocols/transport/test_tcp_mptcp_subtype_unit.py index a99d48226b..e4669662bf 100644 --- a/tests/protocols/transport/test_tcp_mptcp_subtype_unit.py +++ b/tests/protocols/transport/test_tcp_mptcp_subtype_unit.py @@ -85,7 +85,7 @@ from typing import Optional #: Header fields shared by every constructed TCP segment in this module, matching -#: :data:`examples.generators.options.TCP_BASE` so these cases build through exactly the +#: :data:`examples.generators.option_families.tcp.TCP_BASE` so these cases build through exactly the #: keyword shape the fixture generator uses. TCP_BASE = { 'srcport': 50000, 'dstport': 80, 'seq_no': 1, 'ack_no': 0, diff --git a/tests/vendor/test_vendor_missing_body_unit.py b/tests/vendor/test_vendor_missing_body_unit.py index d50d86e129..2f12cfe930 100644 --- a/tests/vendor/test_vendor_missing_body_unit.py +++ b/tests/vendor/test_vendor_missing_body_unit.py @@ -69,27 +69,61 @@ #: ``pcapkit`` an editable install happens to resolve. VENDOR_ROOT = pathlib.Path(__file__).resolve().parents[2] / 'pcapkit' / 'vendor' -#: How many crawlers emit at least one *hard-coded* line into ``miss`` as a -#: string-constant list literal. Pinned so a crawler that leaves this shape -- -#: switching to ``.append()``, or dropping its unassigned range -- is visible -#: here rather than silently shrinking the sweep below. It does **not** catch -#: the opposite drift: a brand-new crawler that starts in a shape this guard -#: never covered joins invisibly, because nothing here shrinks when one is -#: added. -#: -#: Only those crawlers are visible to this guard. Every other crawler that -#: defines ``process()`` -- one that builds ``miss`` by ``.append()``, or never -#: returns an ``(enum, miss)`` pair at all -- is invisible to it, not merely the -#: crawlers named as examples in :func:`_constant_miss_lines`'s docstring -#: below. A crawler that assigns ``miss`` from a comprehension, from a variable, -#: or under a different variable name entirely is skipped by construction too, -#: same as one built by ``.append()``. Those are covered behaviourally, not -#: statically, by :file:`tests/const/test_const_enum_lookup.py`. -#: The ICMP, ICMPv6 and IGMP ``type`` crawlers seed ``miss`` with an empty -#: list or build their bodies dynamically, so they do not all count here. The -#: constant-body crawlers still present in the merged tree are the updated count -#: that the static sweep pins. -EXPECTED_CRAWLERS_WITH_CONSTANT_MISS_LINES = 45 +#: Every crawler that emits at least one *hard-coded* line into ``miss`` as a +#: string-constant list literal, sorted, one per line, so that unrelated +#: additions land on different lines (#2615). Listed so a crawler that leaves +#: this shape -- switching to ``.append()``, or dropping its unassigned range -- +#: or joins it fails by name, rather than shrinking or growing the sweep below +#: silently. A crawler in any other shape is invisible to this guard; see +#: :func:`_constant_miss_lines` for which, and +#: :file:`tests/const/test_const_enum_lookup.py` for their behavioural cover. +CRAWLERS_WITH_CONSTANT_MISS_LINES = ( + 'pcapkit/vendor/icmp/destination_unreachable_code.py', + 'pcapkit/vendor/icmp/parameter_problem_code.py', + 'pcapkit/vendor/icmp/time_exceeded_code.py', + 'pcapkit/vendor/icmpv6/destination_unreachable_code.py', + 'pcapkit/vendor/icmpv6/parameter_problem_code.py', + 'pcapkit/vendor/icmpv6/time_exceeded_code.py', + 'pcapkit/vendor/igmp/record_type.py', + 'pcapkit/vendor/ipv4/classification_level.py', + 'pcapkit/vendor/ipv4/option_class.py', + 'pcapkit/vendor/ipv4/option_number.py', + 'pcapkit/vendor/ipv4/protection_authority.py', + 'pcapkit/vendor/ipv4/qs_function.py', + 'pcapkit/vendor/ipv4/tos_del.py', + 'pcapkit/vendor/ipv4/tos_ecn.py', + 'pcapkit/vendor/ipv4/tos_pre.py', + 'pcapkit/vendor/ipv4/tos_rel.py', + 'pcapkit/vendor/ipv4/tos_thr.py', + 'pcapkit/vendor/ipv4/ts_flag.py', + 'pcapkit/vendor/ipv6/option.py', + 'pcapkit/vendor/ipv6/option_action.py', + 'pcapkit/vendor/ipv6/qs_function.py', + 'pcapkit/vendor/ipv6/seed_id.py', + 'pcapkit/vendor/ipv6/smf_dpd_mode.py', + 'pcapkit/vendor/ipx/packet.py', + 'pcapkit/vendor/l2tp/type.py', + 'pcapkit/vendor/linux/arphrd.py', + 'pcapkit/vendor/mh/auth_subtype.py', + 'pcapkit/vendor/mh/binding_error.py', + 'pcapkit/vendor/mh/cga_type.py', + 'pcapkit/vendor/mh/dhcp_support_mode.py', + 'pcapkit/vendor/mh/enumerating_algorithm.py', + 'pcapkit/vendor/mh/lla_code.py', + 'pcapkit/vendor/mh/option.py', + 'pcapkit/vendor/mh/packet.py', + 'pcapkit/vendor/pcapng/filter_type.py', + 'pcapkit/vendor/pcapng/hash_algorithm.py', + 'pcapkit/vendor/pcapng/option_type.py', + 'pcapkit/vendor/pcapng/record_type.py', + 'pcapkit/vendor/pcapng/secrets_type.py', + 'pcapkit/vendor/pcapng/verdict_type.py', + 'pcapkit/vendor/reg/linktype.py', + 'pcapkit/vendor/sslv2/handshake_type.py', + 'pcapkit/vendor/sslv2/version.py', + 'pcapkit/vendor/tcp/checksum.py', + 'pcapkit/vendor/vlan/priority_level.py', +) def _constant_miss_lines() -> 'dict[str, list[str]]': @@ -98,13 +132,13 @@ def _constant_miss_lines() -> 'dict[str, list[str]]': Returns a mapping of repository-relative crawler path to the list of string constants in its ``miss`` list literal. A crawler that builds ``miss`` by ``.append()``, assigns it from a comprehension or a variable, - or never returns an ``(enum, miss)`` pair at all, is absent from the - returned mapping entirely -- not present with an empty list. That is - every other crawler module defining ``process()``, not just the + names it otherwise, or never returns an ``(enum, miss)`` pair at all, is + absent from the returned mapping entirely -- not present with an empty + list. That is every other crawler module defining ``process()``, not just the examples this docstring names (:mod:`pcapkit.vendor.reg.ethertype`, :mod:`pcapkit.vendor.ipx.socket`); - see :data:`EXPECTED_CRAWLERS_WITH_CONSTANT_MISS_LINES` for the - breakdown. They are covered behaviourally, not statically, by + see :data:`CRAWLERS_WITH_CONSTANT_MISS_LINES` for the ones present. + They are covered behaviourally, not statically, by :file:`tests/const/test_const_enum_lookup.py`. """ @@ -132,13 +166,20 @@ class VendorMissingBodyTests(unittest.TestCase): def setUpClass(cls) -> None: cls.miss_lines = _constant_miss_lines() # type: ignore[attr-defined] - def test_the_sweep_size_is_pinned(self) -> None: + def test_the_sweep_matches_the_list(self) -> None: """A crawler leaving or joining the sweep must be deliberate.""" - self.assertEqual(len(self.miss_lines), # type: ignore[attr-defined] - EXPECTED_CRAWLERS_WITH_CONSTANT_MISS_LINES, - 'the set of crawlers emitting a constant ``_missing_`` body ' - 'changed; update EXPECTED_CRAWLERS_WITH_CONSTANT_MISS_LINES ' - 'once you have checked the new ones still return') + found = set(self.miss_lines) # type: ignore[attr-defined] + listed = set(CRAWLERS_WITH_CONSTANT_MISS_LINES) + self.assertEqual({'joined': sorted(found - listed), 'left': sorted(listed - found)}, + {'joined': [], 'left': []}, + 'the crawlers that emit a constant ``_missing_`` body differ ' + 'from CRAWLERS_WITH_CONSTANT_MISS_LINES, which lists each one ' + 'whose body has been checked to end in a return') + + def test_the_list_is_sorted_and_unique(self) -> None: + """Sorted, so that unrelated additions land on different lines (#2615).""" + self.assertEqual(list(CRAWLERS_WITH_CONSTANT_MISS_LINES), + sorted(set(CRAWLERS_WITH_CONSTANT_MISS_LINES))) def test_the_hard_coded_body_ends_in_a_return(self) -> None: """The invariant #866 is about, restated to the shape that is true.