fix(protocols): convert capture timestamps without the platform's fromtimestamp limits (#2099)

docs/source/pcapkit/utilities/index.rst

@@ -6,8 +6,8 @@ Module Utilities
 
 :mod:`pcapkit.utilities` holds the cross-cutting services :mod:`pcapkit` is
 built with: user-refined exceptions and warnings, logging, the tunable limits
-of :mod:`~pcapkit.utilities.config`, a compatibility layer, and wrappers around
-third-party libraries. It imports nothing from
+of :mod:`~pcapkit.utilities.config`, a compatibility layer, timestamp
+conversion, and wrappers around third-party libraries. It imports nothing from
 :mod:`pcapkit` outside itself; see :ref:`utilities-or-corekit`.
 
 .. toctree::
@@ -18,6 +18,7 @@ third-party libraries. It imports nothing from
    logging
    config
    chardet
+   timestamp
 
 Version Compatibility
 =====================

docs/source/pcapkit/utilities/timestamp.rst

@@ -0,0 +1,26 @@
+Timestamp Conversion
+====================
+
+.. module:: pcapkit.utilities.timestamp
+
+:mod:`pcapkit.utilities.timestamp` converts a UNIX timestamp to an aware
+:class:`~datetime.datetime` without the C library's range (:issue:`2099`). It is
+shared by the PCAP and PCAP-NG readers and the MH timestamp options, which is
+why it lives here rather than beside any one of them.
+
+.. function:: fromtimestamp(timestamp, tz)
+
+   ``datetime.datetime.fromtimestamp(timestamp, tz)`` as 64-bit Linux computes
+   it, on every platform: whole seconds and microseconds split from a float as
+   CPython splits them, rounding half to even, then added to the UNIX epoch and
+   rendered in ``tz``. The standard library breaks the instant down with the C
+   library's ``gmtime()``, whose range on Windows is 1970 to 3000, so there it
+   raises :exc:`OSError` for instants :class:`~datetime.datetime` holds.
+
+   :raises ValueError: if ``timestamp`` is NaN or names a year outside 1 to 9999
+   :raises OverflowError: if ``timestamp`` is past a 64-bit ``time_t``, or ``tz``
+      moves the result out of range
+
+   .. versionadded:: 1.5.0
+
+      Every capture and MH timestamp is converted with it (:issue:`2099`).

pcapkit/protocols/internet/mh.py

@@ -497,6 +497,7 @@
 from pcapkit.protocols.schema.internet.mh import VendorSpecificOption as Schema_VendorSpecificOption
 from pcapkit.protocols.schema.schema import Schema
 from pcapkit.utilities.exceptions import EnumValueError, ProtocolError, UnsupportedCall
+from pcapkit.utilities.timestamp import fromtimestamp
 from pcapkit.utilities.warnings import ProtocolWarning, RegistryWarning, warn
 
 if TYPE_CHECKING:
@@ -4088,8 +4089,9 @@ def _read_opt_timestamp(self, schema: 'Schema_TimestampOption', *,
         data = Data_TimestampOption(
             type=schema.type,
             length=self._mh_option_length(schema.length),
-            timestamp=datetime.datetime.fromtimestamp(
-                seconds + fraction / 65536, tz=datetime.timezone.utc),
+            # not the stdlib's ``fromtimestamp``, which takes no time after 3000
+            # on Windows (#2099)
+            timestamp=fromtimestamp(seconds + fraction / 65536, datetime.timezone.utc),
             pmip_timestamp=Data_PMIPv6Timestamp(seconds=seconds, fraction=fraction),
         )
         return data

pcapkit/protocols/misc/pcap/frame.py

@@ -49,6 +49,7 @@
 from pcapkit.protocols.schema.misc.pcap.header import RESOLUTION
 from pcapkit.utilities.compat import localcontext
 from pcapkit.utilities.exceptions import StreamEOFError, UnsupportedCall, stacklevel
+from pcapkit.utilities.timestamp import fromtimestamp
 from pcapkit.utilities.warnings import ProtocolWarning, RegistryWarning, warn
 
 if TYPE_CHECKING:
@@ -287,8 +288,9 @@ def read(self, length: 'Optional[int]' = None, *, _read: 'bool' = True,
             # aware UTC datetime too, and so does the PCAP-NG reader, which fills
             # the same
             # :attr:`Data_Frame.time <pcapkit.protocols.data.misc.pcap.frame.Frame.time>`
-            # field through :func:`pcapkit.toolkit.pcapng.block2frame`.
-            _time = datetime.datetime.fromtimestamp(_irat[0] / _irat[1], datetime.timezone.utc)
+            # field through :func:`pcapkit.toolkit.pcapng.block2frame`. Converted
+            # without the C library, as the PCAP-NG reader does (#2099).
+            _time = fromtimestamp(_irat[0] / _irat[1], datetime.timezone.utc)
         except ValueError:
             warn(f'PCAP: invalid timestamp: {_epch}', ProtocolWarning, stacklevel=stacklevel())
             _time = datetime.datetime.fromtimestamp(0, datetime.timezone.utc)

pcapkit/protocols/misc/pcapng.py

@@ -212,6 +212,7 @@
 from pcapkit.utilities.compat import StrEnum, localcontext
 from pcapkit.utilities.exceptions import (BaseError, EnumValueError, FormatError, ProtocolError,
                                           StreamEOFError, UnsupportedCall, stacklevel)
+from pcapkit.utilities.timestamp import fromtimestamp
 from pcapkit.utilities.warnings import (AttributeWarning, DeprecatedFormatWarning, ProtocolWarning,
                                         RegistryWarning, warn)
 
@@ -1877,10 +1878,12 @@ def _read_timestamp(self, timestamp_high: 'int', timestamp_low: 'int', *,
         try:
             # NOTE: ``tzone`` survives only here, as the zone the instant is
             # *rendered* in -- which is all a ``tzinfo`` can affect on an aware
-            # datetime. It never moves the instant itself.
-            ts_datetime = datetime.datetime.fromtimestamp(ts_ratio[0] / ts_ratio[1], tzone)
-        # an epoch past what ``datetime`` or the platform's ``time_t`` holds,
-        # e.g. from an extreme ``if_tsoffset``, raises one of these
+            # datetime. It never moves the instant itself. Not the stdlib's
+            # ``fromtimestamp``, whose range is the C library's: Windows' takes
+            # nothing before 1970 or after 3000 (#2099).
+            ts_datetime = fromtimestamp(ts_ratio[0] / ts_ratio[1], tzone)
+        # an epoch past what ``datetime`` or a 64-bit ``time_t`` holds, e.g.
+        # from an extreme ``if_tsoffset``, raises one of these
         except (ValueError, OverflowError, OSError):
             warn(f'PCAP-NG: [Block {self._type}] invalid timestamp: {timestamp_epoch}',
                  ProtocolWarning, stacklevel=stacklevel())

pcapkit/protocols/schema/internet/mh.py

@@ -62,6 +62,7 @@
 from pcapkit.corekit.fields.strings import BitField, BytesField, PaddingField, StringField
 from pcapkit.protocols.schema.schema import EnumSchema, Schema, schema_final
 from pcapkit.utilities.logging import SPHINX_TYPE_CHECKING
+from pcapkit.utilities.timestamp import fromtimestamp
 
 __all__ = [
     'MH',
@@ -848,7 +849,9 @@ def post_process(self, packet: 'dict[str, Any]') -> 'MesgIDOption':
         ts_sec = self.seconds - 2_208_988_800  # 70 years
         ts_frac = self.fraction / 2**32  # fraction of a second
 
-        self.timestamp = datetime.datetime.fromtimestamp(ts_sec + ts_frac, tz=datetime.timezone.utc)
+        # not the stdlib's ``fromtimestamp``, which takes no time before 1970,
+        # i.e. no NTP time before 2_208_988_800, on Windows (#2099)
+        self.timestamp = fromtimestamp(ts_sec + ts_frac, datetime.timezone.utc)
 
         return self
 

pcapkit/utilities/timestamp.py

@@ -0,0 +1,80 @@
+# -*- coding: utf-8 -*-
+"""Timestamp Conversion
+======================
+
+.. module:: pcapkit.utilities.timestamp
+
+:mod:`pcapkit.utilities.timestamp` converts a UNIX timestamp to an aware
+:class:`~datetime.datetime` without the C library's range (:issue:`2099`).
+
+"""
+
+import datetime
+import math
+import operator
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from datetime import tzinfo
+
+__all__ = ['fromtimestamp']
+
+# A UNIX timestamp as an aware datetime, without the C library (#2099).
+# :meth:`datetime.datetime.fromtimestamp` breaks the instant down with the
+# platform's ``gmtime()`` when given a ``tz``, so its range is the C library's
+# rather than ``datetime``'s: the Windows CRT's ``gmtime_s`` takes no time before
+# 1970 and none after 3000, and CPython raises :exc:`OSError` there for instants
+# ``datetime`` holds. This splits a float into whole seconds and microseconds
+# exactly as CPython does -- :func:`math.modf`, the fraction times ``1e6``
+# rounded half to even, then carried into ``[0, 1e6)`` -- and adds them to the
+# epoch, so every instant gets the result glibc's ``gmtime_r`` gives. It is not
+# a version gate: the same arithmetic runs on every interpreter.
+#
+# The errors are CPython's on 64-bit Linux, by class: :exc:`ValueError` for NaN
+# or a year outside 1 to 9999, and :exc:`OverflowError` for whole seconds past
+# a 64-bit ``time_t`` or a ``tz`` offset that pushes the result out of range.
+# The one exception is a year past what a C ``int`` holds, from about 6.7e16
+# seconds, where glibc's own ``gmtime_r`` fails first and CPython raises
+# :exc:`OSError` instead; this raises :exc:`ValueError` there too.
+_EPOCH = datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc)
+#: First and last whole second since the UNIX epoch in years 1 to 9999.
+_EPOCH_MIN = -62_135_596_800
+_EPOCH_MAX = 253_402_300_799
+#: Microseconds per second, the resolution of :class:`~datetime.datetime`.
+_MICROSECONDS = datetime.timedelta(seconds=1) // datetime.timedelta(microseconds=1)
+#: A 64-bit ``time_t`` holds ``[-_TIME_T, _TIME_T)``.
+_TIME_T = 2**63
+
+
+def fromtimestamp(timestamp: 'float | int', tz: 'tzinfo') -> 'datetime.datetime':
+    """:meth:`datetime.datetime.fromtimestamp` with a ``tz``, minus the C library's range.
+
+    The result equals ``datetime.datetime.fromtimestamp(timestamp, tz)`` on
+    64-bit Linux for every ``timestamp`` that call accepts, on every platform.
+
+    Args:
+        timestamp: Seconds since the UNIX epoch.
+        tz: Timezone the result is rendered in.
+
+    Raises:
+        ValueError: If ``timestamp`` is NaN, or names a year outside 1 to 9999.
+        OverflowError: If ``timestamp`` is past a 64-bit ``time_t``, or ``tz``
+            moves the result out of the range :class:`~datetime.datetime` holds.
+
+    """
+    if isinstance(timestamp, float):
+        if math.isnan(timestamp):
+            raise ValueError('Invalid value NaN (not a number)')
+        frac, whole = math.modf(timestamp)
+        # ``round`` of a float is half to even; the carry is -1, 0 or 1
+        carry, micro = divmod(round(frac * _MICROSECONDS), _MICROSECONDS)
+        whole += carry
+    else:
+        whole, micro = operator.index(timestamp), 0
+    if not -_TIME_T <= whole < _TIME_T:
+        raise OverflowError('timestamp out of range for platform time_t')
+    seconds = int(whole)
+    if not _EPOCH_MIN <= seconds <= _EPOCH_MAX:
+        raise ValueError(f'year is out of range for timestamp {timestamp!r}')
+    utc = _EPOCH + datetime.timedelta(seconds=seconds, microseconds=micro)
+    return tz.fromutc(utc.replace(tzinfo=tz))

tests/protocols/misc/pcap/test_header_frame_unit.py

@@ -288,31 +288,17 @@ class Other(Raw):
             self.assertEqual(Frame.read(no_read, _read=False).number, 5)
             self.assertEqual(no_read._file.tell(), 0)
 
-        real_datetime = datetime.datetime
-
-        class FakeDateTime:
-            calls = 0
-
-            @classmethod
-            def fromtimestamp(cls, *args):
-                if cls.calls == 0:
-                    cls.calls += 1
-                    raise ValueError
-                return real_datetime.fromtimestamp(*args)
-
         bad_time = object.__new__(Frame)
         bad_time.__header__ = Schema_Frame(ts_sec=1, ts_usec=0,
                                            incl_len=0, orig_len=0, packet=b'')
         bad_time._ghdr = frame._ghdr
         bad_time._nsec = False
         bad_time._fnum = 6
         bad_time._file = io.BytesIO(b'')
-        # Faked through the module's name in frame.py, not through
-        # ``datetime.datetime`` itself: PyPy's datetime is pure Python and
-        # reads its own ``datetime`` global, so faking that fails the real call
-        # the fake hands on to (#1958).
-        fake_module = SimpleNamespace(**dict(vars(datetime), datetime=FakeDateTime))
-        with mock.patch('pcapkit.protocols.misc.pcap.frame.datetime', fake_module), \
+        # Faked through frame.py's own binding of
+        # :func:`pcapkit.utilities.timestamp.fromtimestamp`, which it calls
+        # instead of :meth:`datetime.datetime.fromtimestamp` since #2099.
+        with mock.patch('pcapkit.protocols.misc.pcap.frame.fromtimestamp', side_effect=ValueError), \
                 mock.patch.object(Frame, '_decode_next_layer',
                                   lambda self, info, proto=None, length=None, packet=None: info), \
                 mock.patch('pcapkit.protocols.misc.pcap.frame.warn') as warn: