fix(dumpkit): drop redundant escape/date workarounds dictdumper 0.8.5 fixed

Pipfile

@@ -6,7 +6,7 @@ verify_ssl = true
 [packages]
 aenum = "*"
 chardet = "*"
-dictdumper = "~=0.8.4"
+dictdumper = ">=0.8.5"
 emoji = "*"
 tbtrim = "~=0.3.1"
 

pyproject.toml

@@ -74,7 +74,7 @@ classifiers = [
     "Typing :: Typed",
 ]
 dependencies = [
-    "dictdumper~=0.8.0",  # for formatted output
+    "dictdumper>=0.8.5",  # for formatted output -- the floor fixing #772/#1448 upstream
     "chardet",            # for bytes decode
     "aenum",              # for const types
     "tbtrim>=0.2.1",      # for refined exceptions

tests/dumpkit/test_common_unit.py

@@ -77,7 +77,6 @@ def __init__(self) -> None:
 #: because neither is special inside an XML text node.
 BYTES_KEY = bytes(range(0x20, 0x40))
 RAW_BYTES_KEY = """b' !"#$%&\\'()*+,-./0123456789:;<=>?'"""
-ESCAPED_BYTES_KEY = """b' !"#$%&amp;\\'()*+,-./0123456789:;&lt;=&gt;?'"""
 
 
 def plist_like_dumper() -> 'object':
@@ -226,123 +225,85 @@ def test_make_dumper_object_hook_conversions_and_fallbacks(self) -> None:
         dumper._append_fallback(123, file)
         self.assertEqual(file.getvalue(), "{'base': '123'}")
 
-    def test_make_dumper_escapes_strings_only_for_plist_like_output(self) -> None:
-        """GitHub issue #772: no entity escaping anywhere in :mod:`dictdumper`.
+    def test_make_dumper_object_hook_leaves_strings_and_enum_renderings_unescaped(self) -> None:
+        """GitHub issue #772: escaping is now the writer's own job, not this hook's.
 
         ``dictdumper.plist.PLIST`` (which is also what ``'xml'`` maps to --
-        see ``Extractor.__output__``) interpolates every ``<string>``/``<key>``
-        value straight into its markup, so a value containing ``&``, ``<`` or
-        ``>`` breaks the document it is embedded in. Filed upstream as
-        JarryShaw/DictDumper#125. ``json``/``tree``/``text`` have no such
-        defect and all three characters are legal there, so the escaping in
-        :func:`~pcapkit.dumpkit.common.make_dumper` must fire for a
-        PLIST-rooted ``output`` and stay off for anything else -- a plain
-        :class:`str`, an enum rendered by :func:`~pcapkit.dumpkit.common.render_enum`,
-        and a :class:`~pcapkit.corekit.multidict.MultiDict` key built from one,
-        since that key is written by the PLIST writer directly and never
-        passed back through this hook the way a value is.
+        see ``Extractor.__output__``) used to interpolate every
+        ``<string>``/``<key>`` value straight into its markup with no entity
+        escaping at all, so a value containing ``&``, ``<`` or ``>`` broke the
+        document it was embedded in. Filed upstream as JarryShaw/DictDumper#125
+        and fixed there in ``dictdumper`` 0.8.5 (DictDumper#123):
+        :func:`dictdumper.plist.encode_xml` now escapes a value itself. So
+        :meth:`~pcapkit.dumpkit.common.make_dumper.DictDumper.object_hook` no
+        longer escapes anything on the way out, for *any* ``output`` -- a plain
+        :class:`str`, an enum rendered by
+        :func:`~pcapkit.dumpkit.common.render_enum`, or a
+        :class:`~pcapkit.corekit.multidict.MultiDict` key built from one, since
+        that key is written by the writer directly and never passed back
+        through this hook the way a value is.
 
         """
         from pcapkit.corekit.multidict import MultiDict
         from pcapkit.dumpkit.common import make_dumper
 
-        plist_dumper = plist_like_dumper()
-        self.assertEqual(plist_dumper.object_hook('a & b <c>'), 'a &amp; b &lt;c&gt;')
-
         unknown = enum.IntEnum('<unknown>', {'<unassigned>': 1})
         member = getattr(unknown, '<unassigned>')
-        self.assertEqual(plist_dumper.object_hook(member),
-                          '&lt;unknown&gt;::&lt;unassigned&gt; [1]')
 
-        multidict = MultiDict()
-        multidict.add(member, 'value')
-        converted = plist_dumper.object_hook(multidict)
-        self.assertEqual(converted['&lt;unknown&gt;::&lt;unassigned&gt; [1]'], ['value'])
+        for name, dumper in (('plist-like', plist_like_dumper()), ('identity', make_dumper(IdentityDumper)())):
+            with self.subTest(dumper=name):
+                self.assertEqual(dumper.object_hook('a & b <c>'), 'a & b <c>')
+                self.assertEqual(dumper.object_hook(member), '<unknown>::<unassigned> [1]')
 
-        plain_dumper = make_dumper(IdentityDumper)()
-        self.assertEqual(plain_dumper.object_hook('a & b <c>'), 'a & b <c>')
-        self.assertEqual(plain_dumper.object_hook(member), '<unknown>::<unassigned> [1]')
+                multidict = MultiDict()
+                multidict.add(member, 'value')
+                converted = dumper.object_hook(multidict)
+                self.assertEqual(converted['<unknown>::<unassigned> [1]'], ['value'])
 
-    def test_make_dumper_escapes_mapping_keys_for_plist_like_output(self) -> None:
+    def test_make_dumper_never_rebuilds_or_escapes_a_mapping(self) -> None:
         """GitHub issue #772, the half the hook above cannot reach.
 
         ``PLIST._append_dict`` in :mod:`dictdumper.plist` writes a key as
-        ``'<key>{item}</key>'.format(item=item)`` and then calls ``_encode_value``
-        on the *value*, never on the key -- so :meth:`object_hook`
-        is handed every value the writer interpolates and no key at all, and the
-        escaping proven above cannot reach one. Both branches that build a
-        mapping therefore escape their own keys:
-        :class:`~pcapkit.corekit.multidict.MultiDict`, where without the fix
-        only an enum-derived key is escaped, and a plain :class:`dict`, where
-        nothing is.
-
-        The :class:`bytes` key is :file:`examples/captures/test.pcapng`'s own,
-        and it is the reason that fixture's ``plist`` report did not parse:
-        ``&``, ``<`` and ``>`` all inside one key. It is rendered with
-        :func:`format`, i.e. the conversion the writer's interpolation already
-        applies to it, so the ``<key>`` text is unchanged apart from the
-        escaping -- which is what the :data:`RAW_BYTES_KEY` assertion pins,
-        since an expected escaping is only as good as the rendering it is
-        derived from.
+        ``'<key>{item}</key>'.format(item=item)`` and
+        ``JSON._append_object`` into ``'"{item}": '``, then call
+        ``_encode_value`` on the *value*, never on the key -- so
+        :meth:`object_hook` is handed every value the writer interpolates and
+        no key at all. Since ``dictdumper`` 0.8.5 (DictDumper#123) both writers
+        escape a key themselves, exactly as they escape the value beside it, so
+        neither a plain :class:`dict` nor the single-key mappings a
+        :class:`~pcapkit.corekit.multidict.MultiDict` or
+        :class:`~pcapkit.corekit.multidict.OrderedMultiDict` becomes is rebuilt
+        here any more: the writer is handed the caller's own mapping, with the
+        caller's own key objects in it, exactly as given -- for *any*
+        ``output``, not only a non-PLIST one.
+
+        The :class:`bytes` key is :file:`examples/captures/test.pcapng`'s own
+        -- it used to be the reason that fixture's ``plist`` report did not
+        parse, since its repr carries ``&``, ``<`` and ``>``.
 
         """
         from pcapkit.corekit.multidict import MultiDict, OrderedMultiDict
+        from pcapkit.dumpkit.common import make_dumper
 
-        plist_dumper = plist_like_dumper()
-
-        # A plain dict: str keys, and the bytes key of the fixture.
-        self.assertEqual(format(BYTES_KEY, ''), RAW_BYTES_KEY)
-        converted = plist_dumper.object_hook({'a & b <c>': 'value', BYTES_KEY: 'secret'})
-        self.assertEqual(list(converted), ['a &amp; b &lt;c&gt;', ESCAPED_BYTES_KEY])
-        self.assertEqual(converted['a &amp; b &lt;c&gt;'], 'value')
-        self.assertEqual(converted[ESCAPED_BYTES_KEY], 'secret')
-
-        # An OrderedMultiDict: the bytes key of the fixture arrives through here
-        # -- its TLS key log entries are an OrderedMultiDict. Each entry becomes
-        # a single-key mapping, whose key is escaped when the writer hands that
-        # mapping back to the hook.
-        multidict = OrderedMultiDict()
-        multidict.add('a & b <c>', 'value')
-        multidict.add(BYTES_KEY, 'secret')
-        converted = [plist_dumper.object_hook(entry) for entry in plist_dumper.object_hook(multidict)]
-        self.assertEqual(converted, [{'a &amp; b &lt;c&gt;': 'value'}, {ESCAPED_BYTES_KEY: 'secret'}])
-
-        # An enum-derived key is escaped exactly once: this branch renders it
-        # with ``render_enum`` and escapes the result through the one shared
-        # ``escape_key`` helper, so escaping it twice is the mistake to rule out.
         unknown = enum.IntEnum('<unknown>', {'<unassigned>': 1})
         member = getattr(unknown, '<unassigned>')
-        multidict = MultiDict()
-        multidict.add(member, 'value')
-        converted = plist_dumper.object_hook(multidict)
-        self.assertEqual(list(converted), ['&lt;unknown&gt;::&lt;unassigned&gt; [1]'])
-        self.assertNotIn('&amp;', ''.join(converted))
-
-    def test_make_dumper_hands_other_output_the_mapping_it_was_given(self) -> None:
-        """``json``, ``tree`` and ``text`` take all three characters literally.
-
-        So no key is escaped for them, and -- the stronger statement, and the
-        one that keeps this branch free of any risk of double-escaping -- a
-        plain :class:`dict` is not even rebuilt: the writer is handed the
-        caller's own mapping, with the caller's own key objects in it, exactly
-        as given. A :class:`bytes` key stays :class:`bytes` there,
-        which is what ``dictdumper``'s ``json`` writer then breaks on, for
-        reasons of its own.
-
-        """
-        from pcapkit.corekit.multidict import OrderedMultiDict
-        from pcapkit.dumpkit.common import make_dumper
-
-        plain_dumper = make_dumper(IdentityDumper)()
-
-        mapping = {'a & b <c>': 'value', BYTES_KEY: 'secret'}
-        self.assertIs(plain_dumper.object_hook(mapping), mapping)
 
-        multidict = OrderedMultiDict()
-        multidict.add('a & b <c>', 'value')
-        multidict.add(BYTES_KEY, 'secret')
-        converted = plain_dumper.object_hook(multidict)
-        self.assertEqual(converted, [{'a & b <c>': 'value'}, {BYTES_KEY: 'secret'}])
+        for name, dumper in (('plist-like', plist_like_dumper()), ('identity', make_dumper(IdentityDumper)())):
+            with self.subTest(dumper=name):
+                mapping = {'a & b <c>': 'value', BYTES_KEY: 'secret'}
+                self.assertIs(dumper.object_hook(mapping), mapping)
+                self.assertEqual(format(BYTES_KEY, ''), RAW_BYTES_KEY)
+
+                multidict = OrderedMultiDict()
+                multidict.add('a & b <c>', 'value')
+                multidict.add(BYTES_KEY, 'secret')
+                converted = dumper.object_hook(multidict)
+                self.assertEqual(converted, [{'a & b <c>': 'value'}, {BYTES_KEY: 'secret'}])
+
+                key_multidict = MultiDict()
+                key_multidict.add(member, 'value')
+                converted_keys = dumper.object_hook(key_multidict)
+                self.assertEqual(list(converted_keys), ['<unknown>::<unassigned> [1]'])
 
     def test_an_unassigned_port_dumps_its_addon_keys_in_the_declared_order(self) -> None:
         """GitHub issue #575's fallback must not reorder what it renders.

tests/dumpkit/test_dumper_roundtrip_unit.py

@@ -173,6 +173,19 @@ def test_json_writes_astral_characters_as_surrogate_pairs(self) -> None:
         self.assertIn(b'"a\\u007f\\u000b\\u00e9\\"\\\\\\n"', data)
         self.assertEqual(json.loads(data)['x'], value)
 
+    def test_json_writes_astral_characters_in_keys_as_surrogate_pairs(self) -> None:
+        """#1261: a mapping KEY above ``U+FFFF`` round-trips, not just a value.
+
+        ``dictdumper.json.JSON._append_object`` escapes a key through the
+        module-level :func:`dictdumper.json.encode_string` directly, bypassing
+        the overridable :meth:`_append_string` the previous test covers, so the
+        key side needed its own, separate fix.
+        """
+        value = {'astral\U0001F600key': 'v', 'plain': 1}
+        data = self.dump('JSON', value)
+        self.assertIn(b'"astral\\ud83d\\ude00key"', data)
+        self.assertEqual(json.loads(data)['x'], value)
+
     def test_plist_writes_strings_xml_cannot_spell_as_data(self) -> None:
         """#1262: a control character does not make the document unparseable."""
         value = {'c': 'a\x01b\x1bc', 'ok': 'tab\there & <there>'}

tests/integration/test_output_formats.py

@@ -117,10 +117,11 @@ class PlistRoundTripTests(EndToEndTestCase):
     def test_plist_report_round_trips_through_plistlib(self) -> None:
         """A ``plist`` report is readable by :func:`plistlib.load` (GitHub issue #1448).
 
-        ``dictdumper`` writes a ``<date>`` with fractional seconds, which
-        :mod:`plistlib` rejects; :mod:`pcapkit.dumpkit` writes it to whole
-        seconds instead, and the exact value stays in ``time_epoch``. The date
-        is compared as naive UTC, since ``aware_datetime`` is new in Python 3.13.
+        ``dictdumper`` used to write a ``<date>`` with fractional seconds,
+        which :mod:`plistlib` rejects; fixed upstream in ``dictdumper`` 0.8.5
+        (DictDumper#124), which now writes whole seconds itself. The exact
+        value stays in ``time_epoch`` regardless. The date is compared as
+        naive UTC, since ``aware_datetime`` is new in Python 3.13.
 
         """
         import datetime

tests/integration/test_pcapng_end_to_end.py

@@ -171,9 +171,12 @@ class PcapngUnescapedKeyTests(EndToEndTestCase):
     """The ``json`` report of :file:`test.pcapng`, whose mapping keys need escaping.
 
     The fixture's decryption secrets block keys its TLS key log entries by a raw
-    :class:`bytes` client random, and :mod:`dictdumper` writes a mapping key
-    into a ``json`` report without escaping it. The report parses only because
-    :mod:`pcapkit.dumpkit` escapes the keys itself (GitHub issue #1152).
+    :class:`bytes` client random. ``dictdumper.json.JSON._append_object`` used
+    to write a mapping key into a ``json`` report without escaping it (GitHub
+    issue #1152), and the report parsed only because :mod:`pcapkit.dumpkit`
+    escaped the key itself. Fixed upstream in ``dictdumper`` 0.8.5
+    (DictDumper#123), so the report now parses on the writer's own escaping;
+    this test pins that the behaviour survives either way.
 
     """
 
@@ -184,12 +187,9 @@ def test_json_report_of_a_decryption_secrets_block_parses(self) -> None:
         <pcapkit.protocols.schema.misc.pcapng.TLSKeyLog.post_process>` keys the
         entries by ``bytes.fromhex(random)``, so the key the writer formats is a
         :class:`bytes` repr carrying ``"`` and ``\\``:
-        ``b' !"#$%&\\'()*+,-./0123456789:;<=>?'``.
-        :meth:`dictdumper.json.JSON._append_object` writes a key into
-        ``'"{item}": '`` raw, so that ``"`` would end the JSON string early --
-        still open upstream as JarryShaw/DictDumper#121. The ``escape_key``
-        helper of :func:`~pcapkit.dumpkit.common.make_dumper` escapes it with
-        :func:`json.dumps` first; without that, this test fails.
+        ``b' !"#$%&\\'()*+,-./0123456789:;<=>?'``, which a key written into
+        ``'"{item}": '`` raw -- as ``dictdumper`` 0.8.5 no longer does -- would
+        close the JSON string early.
 
         """
         extractor = self.extract(fin=sample_path('test.pcapng'), fout=self.out('report'),

tests/project/test_rfc_constants_one_home_1739_unit.py

@@ -336,6 +336,8 @@
         'the IP version an IPv6-in-IP tunnel carries, not the Next Header offset',
     ('pcapkit/dumpkit/common.py', 'ord(char) > 65535'):
         'the last code point of the Basic Multilingual Plane, not an IP length',
+    ('pcapkit/dumpkit/common.py', 'codepoint > 65535'):
+        'above the Basic Multilingual Plane, needs a UTF-16 surrogate pair, not an IP length',
 }
 
 #: Sweep hits of a homed value in a module whose own protocol owns no constant