From 14be18aa819ea0caefe37ffde9fab772eaa6d18c Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Sat, 31 Oct 2026 16:14:17 -0400 Subject: [PATCH] feat(vendor): add the TLS content-type/handshake-type crawlers, close pairing gap (#2495) - pcapkit/vendor/tls/content_type.py, handshake_type.py: new LINK-and-scrape crawlers against IANA's tls-parameters-5.csv and -7.csv. Both override process()/count() for the registry's 5-column CSV, lower-case the wire name into the member identifier, and for HandshakeType strip IANA's _RESERVED suffix from identifiers while keeping it out of the comment. - Regenerated pcapkit/const/tls/content_type.py, handshake_type.py from the crawlers. Fixes a real behaviour bug: ContentType resolved 32-63 to Unassigned via a flat _missing_ fallback where IANA (and the crawler) call that band Reserved (RFC 9147) -- the committed docstring already said "Reserved", the code did not. Drops ContentType.invalid=0 (no IANA counterpart, unused anywhere); value 0 now resolves into the 0-19 Unassigned band like every other unassigned value there, which SSL's wire-type allowlist already excluded by construction, so this is a behavioural no-op in the protocol layer. HandshakeType gains explicit unassigned_7/unassigned_255 members (IANA's two single-row Unassigned entries) instead of leaving them to _missing_, and several members' RFC citations now match IANA's current RFC 9846 instead of the superseded 5246/6066/4680. - Moved the hand-written protocol-specific prose (record-layer header citations, Unassigned/Reserved band rationale, _RESERVED-suffix handling) from the const modules' docstrings into the crawlers' own docstrings, matching pcapkit.vendor.linux.arphrd's pattern; the generated const modules now carry only the plain templated docstring. - pcapkit/protocols/application/ssl.py: updated two comments that named the now-removed invalid(0) member. - docs/.../const-vendor-pairing.rst: rewrote the TLS section -- running either crawler is now a true no-op against the committed const files, closing the gap the previous revision left open as follow-up work. - tests/const/test_const_tls_vendor_pairing_2495_unit.py: new, pins the Reserved-band fix, the dropped invalid member and the two new unassigned members; all five assertions fail against the pre-fix files. Verified: tests/vendor, tests/const, test_ssl_tls_dtls_unit.py and test_tls_dtls_roundtrip_unit.py all green (651 passed, 26949 subtests). Regenerating both const modules twice in a row produces byte-identical output. --- .../conventions/const-vendor-pairing.rst | 52 ++++-- pcapkit/const/tls/content_type.py | 61 +++---- pcapkit/const/tls/handshake_type.py | 100 ++++++----- pcapkit/protocols/application/ssl.py | 12 +- pcapkit/vendor/__init__.py | 3 + pcapkit/vendor/tls/__init__.py | 26 +++ pcapkit/vendor/tls/content_type.py | 142 ++++++++++++++++ pcapkit/vendor/tls/handshake_type.py | 157 ++++++++++++++++++ ...test_const_tls_vendor_pairing_2495_unit.py | 76 +++++++++ .../application/test_ssl_tls_dtls_unit.py | 7 +- 10 files changed, 544 insertions(+), 92 deletions(-) create mode 100644 pcapkit/vendor/tls/__init__.py create mode 100644 pcapkit/vendor/tls/content_type.py create mode 100644 pcapkit/vendor/tls/handshake_type.py create mode 100644 tests/const/test_const_tls_vendor_pairing_2495_unit.py diff --git a/docs/source/contributing/conventions/const-vendor-pairing.rst b/docs/source/contributing/conventions/const-vendor-pairing.rst index c7af54ed65..a8fa09dfee 100644 --- a/docs/source/contributing/conventions/const-vendor-pairing.rst +++ b/docs/source/contributing/conventions/const-vendor-pairing.rst @@ -40,15 +40,47 @@ module docstring and comments, alongside the ``DATA`` literal it documents; the generated ``const`` module carries only the plain, templated docstring every other generated module does. -Known Violation -~~~~~~~~~~~~~~~ +``LINK``-and-Scrape Pairing: TLS Content Types and HandshakeType +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ :class:`~pcapkit.const.tls.content_type.ContentType` and -:class:`~pcapkit.const.tls.handshake_type.HandshakeType` (:pr:`2441`) both cite a live -IANA registry and claim to name every value IANA assigns, with no -:mod:`pcapkit.vendor` crawler behind either one -- no ``pcapkit/vendor/tls/`` -directory exists, and neither class appears in :data:`pcapkit.vendor.__all__`. This -was flagged in that PR's own review and merged anyway on the stated reasoning that the -crawler would follow later. Tracked as :issue:`2495`. Unlike ARPHRD, TLS's source -*is* a live, web-published IANA table, so its eventual crawler takes the normal -``LINK``-and-scrape shape, not the ``DATA`` attribute. +:class:`~pcapkit.const.tls.handshake_type.HandshakeType` (:pr:`2441`) now have +matching crawlers -- :mod:`pcapkit.vendor.tls.content_type` and +:mod:`pcapkit.vendor.tls.handshake_type`, exported as ``TLS_ContentType`` and +``TLS_HandshakeType`` -- closing the gap :issue:`2495` tracked. Both take the +normal ``LINK``-and-scrape shape against IANA's ``tls-parameters-5.csv`` and +``tls-parameters-7.csv``, as this doc said the eventual crawler would; unlike +ARPHRD, TLS's source is a live, web-published table, so neither reads a +``DATA`` literal. + +Running either crawler is a true no-op against the committed ``const`` +modules: regenerating both reproduces the tree byte-for-byte, the same +guarantee ARPHRD and the ``sslv2`` restack carry. Getting there meant +regenerating the ``const`` modules committed alongside :pr:`2441`, which +predated the crawler and had drifted from it in three ways, caught in +cross-review of :pr:`2503`: + +* the committed ``ContentType`` resolved values 32-63 to ``Unassigned`` via a + flat ``_missing_`` fallback, while IANA's registry -- and the crawler -- + call that band ``Reserved`` (:rfc:`9147`); the committed docstring already + said "Reserved", the code did not. Regenerating fixes the resolved name for + that band. +* the committed ``ContentType`` carried an ``invalid = 0`` sentinel with no + IANA counterpart and no reference anywhere in the codebase (IANA leaves 0 + inside its 0-19 ``Unassigned`` band, unnamed); it is dropped, and value 0 + now resolves to ``Unassigned`` like the rest of that band. + :class:`~pcapkit.protocols.application.ssl.SSL` never checked this member + by name -- its wire-type allowlist already excluded value 0 by construction + -- so the sentinel's removal is behaviourally a no-op there too. +* the committed ``HandshakeType`` cited RFC 5246, 6066 and 4680 for several + members where IANA's registry now cites :rfc:`9846`, and left IANA's two + single-row ``Unassigned`` entries (values 7 and 255) to ``_missing_`` + rather than naming them; the crawler renders both as explicit members + (``unassigned_7``, ``unassigned_255``), matching the registry. + +The protocol-specific prose the hand-curated modules carried -- the +record-layer header citations, the Unassigned/Reserved band rationale, the +``_RESERVED``-suffix handling -- moved into the crawlers' own module +docstrings and comments, the way :mod:`pcapkit.vendor.linux.arphrd` already +does for ARPHRD; the generated ``const`` modules carry only the plain, +templated docstring every other generated module does. diff --git a/pcapkit/const/tls/content_type.py b/pcapkit/const/tls/content_type.py index 2adb95e502..0895815e21 100644 --- a/pcapkit/const/tls/content_type.py +++ b/pcapkit/const/tls/content_type.py @@ -6,17 +6,7 @@ .. module:: pcapkit.const.tls.content_type This module contains the constant enumeration for **TLS/DTLS Content Types**, -as assigned by IANA in the ``TLS ContentType`` registry [*]_. The record-layer -header -- :rfc:`9846#section-5.1` for TLS, :rfc:`9147#section-4`'s -``DTLSPlaintext`` for DTLS -- opens with this type in both. - -Every value IANA assigns is named below. Everything else in the 0-255 range it -lists as Unassigned (0-19 and 28-31, plus 64-255, which :rfc:`9443` reserves -for multiplexing with other protocols on the same port) or as Reserved (32-63, -:rfc:`9147`), and :meth:`ContentType._missing_` resolves those to an -``Unassigned`` member without growing the registry. - -.. [*] https://www.iana.org/assignments/tls-parameters/tls-parameters.xhtml#tls-parameters-5 +which is automatically generated from :class:`pcapkit.vendor.tls.content_type.ContentType`. """ @@ -36,45 +26,47 @@ class ContentType(EnumRegistry, IntEnum): """[ContentType] TLS/DTLS Content Types""" - #: Invalid -- a sentinel :rfc:`9846#section-5.1` names so that a record - #: carrying it can be rejected, never a value sent on the wire; IANA keeps - #: 0 inside its Unassigned 0-19 band rather than assigning it [:rfc:`9846`] - invalid = 0 - - #: ChangeCipherSpec [:rfc:`9846`] + #: change_cipher_spec [:rfc:`9846`] change_cipher_spec = 20 - #: Alert [:rfc:`9846`] + #: alert [:rfc:`9846`] alert = 21 - #: Handshake [:rfc:`9846`] + #: handshake [:rfc:`9846`] handshake = 22 - #: Application Data [:rfc:`9846`] + #: application_data [:rfc:`9846`] application_data = 23 - #: Heartbeat [:rfc:`6520`] + #: heartbeat [:rfc:`6520`] heartbeat = 24 - #: ``tls12_cid`` [:rfc:`9146`] + #: tls12_cid [:rfc:`9146`] tls12_cid = 25 - #: ACK -- DTLS 1.3 only [:rfc:`9147`] + #: ACK [:rfc:`9147`] ack = 26 - #: ``return_routability_check`` -- DTLS connection-ID path validation - #: [:rfc:`9853`] + #: return_routability_check [:rfc:`9853`] return_routability_check = 27 @classmethod def _validate_value(cls, value: 'Any') -> 'None': """Reject ``value`` unless it passes the range check :meth:`_missing_` makes. + Overrides :meth:`~pcapkit.corekit.enum.EnumLookup._validate_value` + with that check and the message it raises, so + :meth:`~pcapkit.corekit.enum.EnumRegistry.register` refuses exactly the + values :meth:`~pcapkit.corekit.enum.EnumLookup.get` refuses, in the + same words (:issue:`1816`). + Args: value: Value about to be looked up or minted. Raises: - EnumValueError: If ``value`` fails that check. + EnumValueError: If ``value`` fails that check. A + :exc:`ValueError`, so an ``except ValueError`` caller is + unaffected. """ if not (isinstance(value, int) and 0 <= value <= 255): @@ -90,5 +82,18 @@ def _missing_(cls, value: 'int') -> 'ContentType': """ if not (isinstance(value, int) and 0 <= value <= 255): raise ValueError('%r is not a valid %s' % (value, cls.__name__)) - #: Unassigned - return cls._unregistered_member(value, 'Unassigned') + if 0 <= value <= 19: + #: Unassigned (Requires coordination; see [RFC 9443]) + #: [:rfc:`5764`][:rfc:`9443`] + return cls._unregistered_member(value, 'Unassigned') + if 28 <= value <= 31: + #: Unassigned + return cls._unregistered_member(value, 'Unassigned') + if 32 <= value <= 63: + #: Reserved [:rfc:`9147`] + return cls._unregistered_member(value, 'Reserved') + if 64 <= value <= 255: + #: Unassigned (Requires coordination; see [RFC 9443]) + #: [:rfc:`5764`][:rfc:`9443`] + return cls._unregistered_member(value, 'Unassigned') + return super()._missing_(value) diff --git a/pcapkit/const/tls/handshake_type.py b/pcapkit/const/tls/handshake_type.py index 223c87613d..763377541a 100644 --- a/pcapkit/const/tls/handshake_type.py +++ b/pcapkit/const/tls/handshake_type.py @@ -6,19 +6,7 @@ .. module:: pcapkit.const.tls.handshake_type This module contains the constant enumeration for **TLS/DTLS HandshakeType**, -as assigned by IANA in the ``TLS HandshakeType`` registry [*]_, including the -DTLS-only connection-ID messages :rfc:`9147#section-5.2` adds to that same -registry. Every value IANA assigns is named below; the ranges it lists as -Unassigned fall through to :meth:`_missing_` like any other unassigned value. - -Each member's name is the message name the protocol uses on the wire. IANA -suffixes ``_RESERVED`` onto the ones TLS 1.3 retired -- it keeps the code point -reserved rather than reassigning it -- which is a statement about the registry -rather than a different message name, so the suffix is not carried into the -member name and each such member instead records IANA's own comment on why the -assignment is retired. - -.. [*] https://www.iana.org/assignments/tls-parameters/tls-parameters.xhtml#tls-parameters-7 +which is automatically generated from :class:`pcapkit.vendor.tls.handshake_type.HandshakeType`. """ @@ -38,95 +26,110 @@ class HandshakeType(EnumRegistry, IntEnum): """[HandshakeType] TLS/DTLS HandshakeType""" - #: ``hello_request`` -- TLS 1.2 and earlier only [:rfc:`5246`] + #: hello_request [:rfc:`9846`] -- Used in TLS versions prior to 1.3. hello_request = 0 - #: ``client_hello`` [:rfc:`9846`] + #: client_hello [:rfc:`9846`] client_hello = 1 - #: ``server_hello`` [:rfc:`9846`] + #: server_hello [:rfc:`9846`] server_hello = 2 - #: ``hello_verify_request`` -- DTLS 1.2 and earlier only; DTLS 1.3 moved - #: its return-routability role to the ``cookie`` extension - #: (:rfc:`9147#section-5.1`) [:rfc:`6347`] + #: hello_verify_request [:rfc:`6347`][:rfc:`9846`] -- Assigned for interim + #: draft, but the functionality was moved to a different message. hello_verify_request = 3 - #: ``new_session_ticket`` [:rfc:`9846`] + #: new_session_ticket [:rfc:`4507`][:rfc:`9846`][:rfc:`8447`][RFC Errata 7250] new_session_ticket = 4 - #: ``end_of_early_data`` [:rfc:`9846`] + #: end_of_early_data [:rfc:`9846`] end_of_early_data = 5 - #: ``hello_retry_request`` -- historical; the functionality moved to an - #: extension, and TLS 1.3 carries it on a ``server_hello`` instead - #: (:rfc:`9846#section-4.2.4`) [:rfc:`9846`] + #: hello_retry_request [:rfc:`9846`] -- Assigned for interim draft, but the + #: functionality was moved to an extension. hello_retry_request = 6 - #: ``encrypted_extensions`` [:rfc:`9846`] + #: Unassigned + unassigned_7 = 7 + + #: encrypted_extensions [:rfc:`9846`] encrypted_extensions = 8 - #: ``request_connection_id`` -- DTLS 1.3 only [:rfc:`9147`] + #: request_connection_id [:rfc:`9147`] request_connection_id = 9 - #: ``new_connection_id`` -- DTLS 1.3 only [:rfc:`9147`] + #: new_connection_id [:rfc:`9147`] new_connection_id = 10 - #: ``certificate`` [:rfc:`9846`] + #: certificate [:rfc:`9846`] certificate = 11 - #: ``server_key_exchange`` -- TLS 1.2 and earlier only [:rfc:`5246`] + #: server_key_exchange [:rfc:`9846`] -- Used in TLS versions prior to 1.3. server_key_exchange = 12 - #: ``certificate_request`` [:rfc:`9846`] + #: certificate_request [:rfc:`9846`] certificate_request = 13 - #: ``server_hello_done`` -- TLS 1.2 and earlier only [:rfc:`5246`] + #: server_hello_done [:rfc:`9846`] -- Used in TLS versions prior to 1.3. server_hello_done = 14 - #: ``certificate_verify`` [:rfc:`9846`] + #: certificate_verify [:rfc:`9846`] certificate_verify = 15 - #: ``client_key_exchange`` -- TLS 1.2 and earlier only [:rfc:`5246`] + #: client_key_exchange [:rfc:`9846`] -- Used in TLS versions prior to 1.3. client_key_exchange = 16 - #: ``client_certificate_request`` -- TLS 1.2 and earlier only, per IANA's - #: own comment on this assignment [:rfc:`9261`] + #: client_certificate_request [:rfc:`9261`] -- Used in TLS versions prior to + #: 1.3. client_certificate_request = 17 - #: ``finished`` [:rfc:`9846`] + #: finished [:rfc:`9846`] finished = 20 - #: ``certificate_url`` -- TLS 1.2 and earlier only [:rfc:`6066`] + #: certificate_url [:rfc:`6066`][:rfc:`9846`] -- Used in TLS versions prior to + #: 1.3. certificate_url = 21 - #: ``certificate_status`` -- TLS 1.2 and earlier only [:rfc:`6066`] + #: certificate_status [:rfc:`6066`][:rfc:`9846`] -- Used in TLS versions prior + #: to 1.3. certificate_status = 22 - #: ``supplemental_data`` -- TLS 1.2 and earlier only [:rfc:`4680`] + #: supplemental_data [:rfc:`4680`][:rfc:`9846`] -- Used in TLS versions prior + #: to 1.3. supplemental_data = 23 - #: ``key_update`` [:rfc:`9846`] + #: key_update [:rfc:`9846`] key_update = 24 - #: ``compressed_certificate`` [:rfc:`8879`] + #: compressed_certificate [:rfc:`8879`] compressed_certificate = 25 - #: ``ekt_key`` [:rfc:`8870`] + #: ekt_key [:rfc:`8870`] ekt_key = 26 - #: ``message_hash`` [:rfc:`9846`] + #: message_hash [:rfc:`9846`] message_hash = 254 + #: Unassigned + unassigned_255 = 255 + @classmethod def _validate_value(cls, value: 'Any') -> 'None': """Reject ``value`` unless it passes the range check :meth:`_missing_` makes. + Overrides :meth:`~pcapkit.corekit.enum.EnumLookup._validate_value` + with that check and the message it raises, so + :meth:`~pcapkit.corekit.enum.EnumRegistry.register` refuses exactly the + values :meth:`~pcapkit.corekit.enum.EnumLookup.get` refuses, in the + same words (:issue:`1816`). + Args: value: Value about to be looked up or minted. Raises: - EnumValueError: If ``value`` fails that check. + EnumValueError: If ``value`` fails that check. A + :exc:`ValueError`, so an ``except ValueError`` caller is + unaffected. """ if not (isinstance(value, int) and 0 <= value <= 255): @@ -142,5 +145,10 @@ def _missing_(cls, value: 'int') -> 'HandshakeType': """ if not (isinstance(value, int) and 0 <= value <= 255): raise ValueError('%r is not a valid %s' % (value, cls.__name__)) - #: Unassigned - return cls._unregistered_member(value, 'Unassigned') + if 18 <= value <= 19: + #: Unassigned + return cls._unregistered_member(value, 'Unassigned') + if 27 <= value <= 253: + #: Unassigned + return cls._unregistered_member(value, 'Unassigned') + return super()._missing_(value) diff --git a/pcapkit/protocols/application/ssl.py b/pcapkit/protocols/application/ssl.py index 17bb980ab8..b88bbd172e 100644 --- a/pcapkit/protocols/application/ssl.py +++ b/pcapkit/protocols/application/ssl.py @@ -60,9 +60,10 @@ _DTLS_VERSION_MAJOR = 0xFE #: The content types a genuine record may carry. An allowlist rather than -#: every :class:`~pcapkit.const.tls.content_type.ContentType` member: the -#: registry names ``invalid(0)``, a sentinel :rfc:`9846#section-5.1` rejects, -#: and mints a member for any unassigned value it is asked for. +#: every :class:`~pcapkit.const.tls.content_type.ContentType` member: 0 is +#: never assigned, a value :rfc:`9846#section-5.1` rejects outright, and the +#: enum mints an ``Unassigned``/``Reserved`` member for any value IANA has +#: not named. #: #: ``tls12_cid`` (25, :rfc:`9146#section-4`) is left out: its connection ID #: precedes ``length`` and has a negotiated length the record does not carry, @@ -235,8 +236,9 @@ def _validate_tls_record(self, schema: 'Any', *, available: 'int', dtls: 'bool') """ # NOTE: ``make()`` leaves ``content_type`` a bare :obj:`int`, not the enum member a # parse yields, so resolve it through the registry to cover both. - # NOTE: checked against ``_WIRE_CONTENT_TYPES``, not by name: ``invalid(0)`` is - # named yet never a record's content type. + # NOTE: checked against ``_WIRE_CONTENT_TYPES``, not by name: an + # unassigned or reserved value still resolves to a named member via + # ``_missing_``, yet is never a record's genuine content type. content_type = Enum_ContentType.get(int(schema.content_type)) if content_type not in _WIRE_CONTENT_TYPES: raise ProtocolError( diff --git a/pcapkit/vendor/__init__.py b/pcapkit/vendor/__init__.py index 9febaf3742..a64bb0c0c4 100644 --- a/pcapkit/vendor/__init__.py +++ b/pcapkit/vendor/__init__.py @@ -120,6 +120,7 @@ def __getattr__(name: 'str') -> 'typing.NoReturn': from pcapkit.vendor.ospf import * from pcapkit.vendor.sctp import * from pcapkit.vendor.tcp import * + from pcapkit.vendor.tls import * from pcapkit.vendor.vlan import * # Crawlers only. This list is also the default target list of @@ -180,6 +181,8 @@ def __getattr__(name: 'str') -> 'typing.NoReturn': 'SCTP_PayloadProtocolIdentifier', # TCP 'TCP_Checksum', 'TCP_Option', 'TCP_MPTCPOption', 'TCP_Flags', + # TLS + 'TLS_ContentType', 'TLS_HandshakeType', # VLAN 'VLAN_PriorityLevel', # PCAPNG diff --git a/pcapkit/vendor/tls/__init__.py b/pcapkit/vendor/tls/__init__.py new file mode 100644 index 0000000000..a7f5d0be97 --- /dev/null +++ b/pcapkit/vendor/tls/__init__.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# pylint: disable=unused-import +""":mod:`pcapkit.const.tls` Vendor Crawlers +============================================= + +.. module:: pcapkit.vendor.tls + +This module contains all vendor crawlers of the TLS/DTLS record-layer +constants. Available crawlers include: + +.. list-table:: + + * - :class:`TLS_ContentType ` + - TLS/DTLS Content Types [*]_ + * - :class:`TLS_HandshakeType ` + - TLS/DTLS HandshakeType [*]_ + +.. [*] https://www.iana.org/assignments/tls-parameters/tls-parameters.xhtml#tls-parameters-5 +.. [*] https://www.iana.org/assignments/tls-parameters/tls-parameters.xhtml#tls-parameters-7 + +""" + +from pcapkit.vendor.tls.content_type import ContentType as TLS_ContentType +from pcapkit.vendor.tls.handshake_type import HandshakeType as TLS_HandshakeType + +__all__ = ['TLS_ContentType', 'TLS_HandshakeType'] diff --git a/pcapkit/vendor/tls/content_type.py b/pcapkit/vendor/tls/content_type.py new file mode 100644 index 0000000000..3ed6f40d96 --- /dev/null +++ b/pcapkit/vendor/tls/content_type.py @@ -0,0 +1,142 @@ +# -*- coding: utf-8 -*- +"""TLS/DTLS Content Types +============================ + +.. module:: pcapkit.vendor.tls.content_type + +This module contains the vendor crawler for **TLS/DTLS Content Types**, +which is automatically generating :class:`pcapkit.const.tls.content_type.ContentType`. + +The registry is IANA's ``TLS ContentType`` table, a five-column CSV -- +``Value, Description, DTLS-OK, Reference, Comment`` -- rather than the +three-column ``Value, Description, Reference`` shape +:meth:`~pcapkit.vendor.default.Vendor.process` assumes, so this crawler reads +the ``Reference`` column at index ``3`` instead of ``2`` and otherwise follows +the same rendering the base class uses elsewhere. + +The record-layer header -- :rfc:`9846#section-5.1` for TLS, :rfc:`9147#section-4`'s +``DTLSPlaintext`` for DTLS -- opens with this type in both. Every value IANA +assigns is named; everything else in the 0-255 range the registry lists as +Unassigned (0-19 and 28-31, plus 64-255, which :rfc:`9443` reserves for +coordination with other protocols multiplexed on the same port) or as +Reserved (32-63, :rfc:`9147`), and the generated ``_missing_`` renders each +such band from the registry's own range rows rather than naming a single +catch-all member. + +""" + +import collections +import csv +import re +import sys +from typing import TYPE_CHECKING + +from pcapkit.vendor.default import Vendor + +if TYPE_CHECKING: + from collections import Counter + +__all__ = ['ContentType'] + + +class ContentType(Vendor): + """TLS/DTLS Content Types""" + + #: Value limit checker. + FLAG = 'isinstance(value, int) and 0 <= value <= 255' + #: Link to registry. + LINK = 'https://www.iana.org/assignments/tls-parameters/tls-parameters-5.csv' + + def clean_name(self, raw: 'str') -> 'str': + """Turn an IANA ``Description`` cell into a member identifier. + + The registry's ``Description`` values are already the lower-case, + underscore-joined wire names TLS/DTLS use (``change_cipher_spec``, + ``heartbeat``, ...), except for ``ACK``, which IANA spells upper-case + even though the record-layer type it names is lower-case on the wire + like every sibling -- :rfc:`9147#section-4.1` itself writes it as + ``ack``. Lower-casing the whole cell, rather than special-casing that + one row, is what makes this crawler keep working if IANA adds another + upper-case entry later. + + Args: + raw: Verbatim ``Description`` cell. + + Returns: + The identifier :meth:`~pcapkit.vendor.default.Vendor.rename` + should render, before duplicate-suffixing. + + """ + return raw.split(' (', maxsplit=1)[0].lower() + + def count(self, data: 'list[str]') -> 'Counter[str]': + """Count field records. + + Args: + data: CSV data. + + Returns: + Field recordings. + + """ + reader = csv.reader(data) + next(reader) # header + return collections.Counter( + self.safe_name(self.clean_name(item[1])) + for item in reader if len(item[0].split('-')) != 2 + ) + + def process(self, data: 'list[str]') -> 'tuple[list[str], list[str]]': + """Process CSV data. + + Args: + data: CSV data. + + Returns: + Enumeration fields and missing fields. + + """ + reader = csv.reader(data) + next(reader) # header + + enum = [] # type: list[str] + miss = [] # type: list[str] + for item in reader: + name = item[1] + rfcs = item[3] + comment = item[4].strip() + + temp = [] # type: list[str] + for rfc in self.split_references(rfcs): + if 'RFC' in rfc and re.match(r'\d+', rfc[3:]): + temp_split = rfc[3:].split(', ', maxsplit=1) + if len(temp_split) > 1: + temp.append(f'[:rfc:`{temp_split[0]}#{temp_split[1].lower()}`]'.replace(' ', '-')) + else: + temp.append(f'[:rfc:`{temp_split[0]}`]') + else: + temp.append(f'[{rfc}]'.replace('_', ' ')) + body = '%s %s' % (name, ''.join(temp) if rfcs else '') # pylint: disable=consider-using-f-string + if comment: + body = '%s -- %s' % (body.strip(), comment) # pylint: disable=consider-using-f-string + desc = self.wrap_comment(re.sub(r'\r*\n\s*', ' ', body, flags=re.MULTILINE)) + + try: + code, _ = item[0], int(item[0]) + renm = self.rename(self.clean_name(name), code) + + pres = f'{renm} = {code}' + sufs = f'#: {desc}' + + enum.append(f'{sufs}\n {pres}') + except ValueError: + start, stop = item[0].split('-') + + miss.append(f'if {start} <= value <= {stop}:') + miss.append(f' #: {desc}') + miss.append(f" return cls._unregistered_member(value, '{self.safe_name(name)}')") + return enum, miss + + +if __name__ == '__main__': + sys.exit(ContentType()) # type: ignore[arg-type] diff --git a/pcapkit/vendor/tls/handshake_type.py b/pcapkit/vendor/tls/handshake_type.py new file mode 100644 index 0000000000..fb8b59ef84 --- /dev/null +++ b/pcapkit/vendor/tls/handshake_type.py @@ -0,0 +1,157 @@ +# -*- coding: utf-8 -*- +"""TLS/DTLS HandshakeType +============================ + +.. module:: pcapkit.vendor.tls.handshake_type + +This module contains the vendor crawler for **TLS/DTLS HandshakeType**, +which is automatically generating +:class:`pcapkit.const.tls.handshake_type.HandshakeType`. + +The registry is IANA's ``TLS HandshakeType`` table, a five-column CSV -- +``Value, Description, DTLS-OK, Reference, Comment`` -- rather than the +three-column ``Value, Description, Reference`` shape +:meth:`~pcapkit.vendor.default.Vendor.process` assumes, so this crawler reads +the ``Reference`` column at index ``3`` instead of ``2`` and otherwise follows +the same rendering the base class uses elsewhere. + +IANA suffixes ``_RESERVED`` onto the handful of code points TLS 1.3 retired -- +it keeps the code point reserved rather than reassigning it -- which is a +statement about the registry, not a different message name, so +:meth:`HandshakeType.clean_name` strips the suffix before it reaches the +generated member name. + +The registry also carries ``request_connection_id`` and ``new_connection_id`` +(:rfc:`9147#section-5.2`), the DTLS-only connection-ID messages that same +section adds to this table rather than a separate one, so they need no +special handling here -- they are rows like any other. + +""" + +import collections +import csv +import re +import sys +from typing import TYPE_CHECKING + +from pcapkit.vendor.default import Vendor + +if TYPE_CHECKING: + from collections import Counter + +__all__ = ['HandshakeType'] + +#: Suffix IANA appends to a ``Description`` cell for a code point TLS 1.3 +#: retired, kept reserved rather than reassigned. +RESERVED_SUFFIX = '_reserved' + + +class HandshakeType(Vendor): + """TLS/DTLS HandshakeType""" + + #: Value limit checker. + FLAG = 'isinstance(value, int) and 0 <= value <= 255' + #: Link to registry. + LINK = 'https://www.iana.org/assignments/tls-parameters/tls-parameters-7.csv' + + def clean_name(self, raw: 'str') -> 'str': + """Turn an IANA ``Description`` cell into a member identifier. + + Strips the ``(renamed from "...")`` parenthetical + :meth:`~pcapkit.vendor.default.Vendor.safe_name` would otherwise + carry into the identifier (e.g. ``new_session_ticket``, value + ``4``), lower-cases the cell, and drops the trailing + :data:`RESERVED_SUFFIX` the module docstring describes. + + Args: + raw: Verbatim ``Description`` cell. + + Returns: + The identifier :meth:`~pcapkit.vendor.default.Vendor.rename` + should render, before duplicate-suffixing. + + """ + name = raw.split(' (', maxsplit=1)[0].lower() + if name.endswith(RESERVED_SUFFIX): + name = name[:-len(RESERVED_SUFFIX)] + return name + + def count(self, data: 'list[str]') -> 'Counter[str]': + """Count field records. + + Args: + data: CSV data. + + Returns: + Field recordings. + + """ + reader = csv.reader(data) + next(reader) # header + return collections.Counter( + self.safe_name(self.clean_name(item[1])) + for item in reader if len(item[0].split('-')) != 2 + ) + + def process(self, data: 'list[str]') -> 'tuple[list[str], list[str]]': + """Process CSV data. + + Args: + data: CSV data. + + Returns: + Enumeration fields and missing fields. + + """ + reader = csv.reader(data) + next(reader) # header + + enum = [] # type: list[str] + miss = [] # type: list[str] + for item in reader: + name = item[1] + rfcs = item[3] + comment = item[4].strip() + + temp = [] # type: list[str] + for rfc in self.split_references(rfcs): + if 'RFC' in rfc and re.match(r'\d+', rfc[3:]): + temp_split = rfc[3:].split(', ', maxsplit=1) + if len(temp_split) > 1: + temp.append(f'[:rfc:`{temp_split[0]}#{temp_split[1].lower()}`]'.replace(' ', '-')) + else: + temp.append(f'[:rfc:`{temp_split[0]}`]') + else: + temp.append(f'[{rfc}]'.replace('_', ' ')) + # NOTE: the comment text, unlike the identifier, keeps the + # registry's original case -- only the ``_RESERVED`` suffix + # :data:`RESERVED_SUFFIX` describes is dropped, since it is a + # statement about the registry rather than part of the message + # name (see the module docstring). + display = name.split(' (', maxsplit=1)[0] + if display.upper().endswith(RESERVED_SUFFIX.upper()): + display = display[:-len(RESERVED_SUFFIX)] + body = '%s %s' % (display, ''.join(temp) if rfcs else '') # pylint: disable=consider-using-f-string + if comment: + body = '%s -- %s' % (body.strip(), comment) # pylint: disable=consider-using-f-string + desc = self.wrap_comment(re.sub(r'\r*\n\s*', ' ', body, flags=re.MULTILINE)) + + try: + code, _ = item[0], int(item[0]) + renm = self.rename(self.clean_name(name), code) + + pres = f'{renm} = {code}' + sufs = f'#: {desc}' + + enum.append(f'{sufs}\n {pres}') + except ValueError: + start, stop = item[0].split('-') + + miss.append(f'if {start} <= value <= {stop}:') + miss.append(f' #: {desc}') + miss.append(f" return cls._unregistered_member(value, '{self.safe_name(name)}')") + return enum, miss + + +if __name__ == '__main__': + sys.exit(HandshakeType()) # type: ignore[arg-type] diff --git a/tests/const/test_const_tls_vendor_pairing_2495_unit.py b/tests/const/test_const_tls_vendor_pairing_2495_unit.py new file mode 100644 index 0000000000..0328a4af45 --- /dev/null +++ b/tests/const/test_const_tls_vendor_pairing_2495_unit.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- +"""``ContentType``/``HandshakeType`` after pairing with their crawler. + +GitHub issue #2495. Regenerating both modules from the new +:mod:`pcapkit.vendor.tls.content_type` and +:mod:`pcapkit.vendor.tls.handshake_type` crawlers (:pr:`2503`) fixed a real +behaviour bug the hand-maintained modules had drifted into: ``ContentType`` +resolved values 32-63 to ``Unassigned`` via a flat ``_missing_`` fallback, +where IANA's registry -- and the crawler -- call that band ``Reserved`` +(:rfc:`9147`). It also dropped ``ContentType.invalid``, which had no IANA +counterpart and no reference anywhere in the codebase, and it means +``HandshakeType`` now names its two single-row ``Unassigned`` entries +(values 7 and 255) as explicit members instead of leaving them to +``_missing_``. + +""" +import unittest + + +class ContentTypeReservedBandTests(unittest.TestCase): + """Values 32-63 resolve to ``Reserved``, not ``Unassigned`` (:rfc:`9147`).""" + + def test_32_resolves_to_reserved(self) -> 'None': + from pcapkit.const.tls.content_type import ContentType + + self.assertEqual(ContentType(32).name, 'Reserved') + + def test_63_resolves_to_reserved(self) -> 'None': + from pcapkit.const.tls.content_type import ContentType + + self.assertEqual(ContentType(63).name, 'Reserved') + + def test_bands_either_side_of_32_63_still_resolve_to_unassigned(self) -> 'None': + from pcapkit.const.tls.content_type import ContentType + + for value in (19, 28, 31, 64, 255): + with self.subTest(value=value): + self.assertEqual(ContentType(value).name, 'Unassigned') + + def test_invalid_member_is_gone(self) -> 'None': + from pcapkit.const.tls.content_type import ContentType + + self.assertFalse(hasattr(ContentType, 'invalid')) + self.assertNotIn('invalid', ContentType.__members__) + # Value 0 still resolves -- to the 0-19 Unassigned band, by + # _missing_, rather than to a dedicated sentinel member. + self.assertEqual(ContentType(0).name, 'Unassigned') + + +class HandshakeTypeUnassignedMembersTests(unittest.TestCase): + """IANA's single-row ``Unassigned`` entries are named members.""" + + def test_value_7_is_a_named_unassigned_member(self) -> 'None': + from pcapkit.const.tls.handshake_type import HandshakeType + + self.assertIn('unassigned_7', HandshakeType.__members__) + self.assertEqual(HandshakeType.unassigned_7, 7) + self.assertEqual(HandshakeType(7).name, 'unassigned_7') + + def test_value_255_is_a_named_unassigned_member(self) -> 'None': + from pcapkit.const.tls.handshake_type import HandshakeType + + self.assertIn('unassigned_255', HandshakeType.__members__) + self.assertEqual(HandshakeType.unassigned_255, 255) + self.assertEqual(HandshakeType(255).name, 'unassigned_255') + + def test_other_unassigned_values_still_fall_through_to_missing(self) -> 'None': + from pcapkit.const.tls.handshake_type import HandshakeType + + for value in (18, 19, 27, 253): + with self.subTest(value=value): + self.assertEqual(HandshakeType(value).name, 'Unassigned') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/protocols/application/test_ssl_tls_dtls_unit.py b/tests/protocols/application/test_ssl_tls_dtls_unit.py index 3614adfbe7..fac8ebc004 100644 --- a/tests/protocols/application/test_ssl_tls_dtls_unit.py +++ b/tests/protocols/application/test_ssl_tls_dtls_unit.py @@ -202,9 +202,10 @@ class RecordValidationTests(unittest.TestCase): The content-type check is an allowlist of the assigned 20-27, not a test of whether :class:`~pcapkit.const.tls.content_type.ContentType` *names* - the value: ``invalid(0)`` is named -- a sentinel :rfc:`9846#section-5.1` - defines so a record carrying it can be rejected -- so a name-based check - accepts ``00 03 03 00 04 deadbeef`` on TCP:443 and decodes its fragment. + the value: 0 resolves to an ``Unassigned`` member via ``_missing_`` -- + the registry never assigns it, and :rfc:`9846#section-5.1` rejects a + record carrying it outright -- so a name-based check would accept + ``00 03 03 00 04 deadbeef`` on TCP:443 and decode its fragment. """