fix(utilities): encode pre-1970 datetimes on GraalPy without its lost second (#2134)

docs/source/pcapkit/utilities/datetime.rst

@@ -0,0 +1,15 @@
+Timestamp Conversion
+====================
+
+.. module:: pcapkit.utilities.datetime
+
+:mod:`pcapkit.utilities.datetime` converts a UNIX timestamp to an aware
+:class:`~datetime.datetime` without the C library's range (:issue:`2099`), and
+a :class:`~datetime.datetime` back to a UNIX timestamp without GraalPy's lost
+second before 1970 (:issue:`2134`). It is shared by the PCAP and PCAP-NG readers
+and writers and the MH timestamp options, which is why it lives here rather than
+beside any one of them.
+
+.. autofunction:: pcapkit.utilities.datetime.fromtimestamp
+
+.. autofunction:: pcapkit.utilities.datetime.totimestamp

docs/source/pcapkit/utilities/index.rst

@@ -18,7 +18,7 @@ conversion, and wrappers around third-party libraries. It imports nothing from
    logging
    config
    chardet
-   timestamp
+   datetime
 
 Version Compatibility
 =====================

pcapkit/protocols/internet/mh.py

@@ -496,8 +496,8 @@
     UpdateNotificationMessage as Schema_UpdateNotificationMessage
 from pcapkit.protocols.schema.internet.mh import VendorSpecificOption as Schema_VendorSpecificOption
 from pcapkit.protocols.schema.schema import Schema
+from pcapkit.utilities.datetime import fromtimestamp, totimestamp
 from pcapkit.utilities.exceptions import EnumValueError, ProtocolError, UnsupportedCall
-from pcapkit.utilities.timestamp import fromtimestamp
 from pcapkit.utilities.warnings import ProtocolWarning, RegistryWarning, warn
 
 if TYPE_CHECKING:
@@ -8383,7 +8383,7 @@ def _make_opt_mesg_id(self, type: 'Enum_Option', option: 'Optional[Data_MesgIDOp
         if timestamp is None:
             interval = interval or datetime.datetime.now(datetime.timezone.utc)
 
-            int_ts = interval.timestamp()
+            int_ts = totimestamp(interval)
             ts_sec = math.floor(int_ts)
             ts_frc = min(round((int_ts - ts_sec) * 2**32), 0xFFFF_FFFF)  # 1/2**32 seconds per unit
 

pcapkit/protocols/misc/pcap/frame.py

@@ -48,8 +48,8 @@
 from pcapkit.protocols.schema.misc.pcap.frame import Frame as Schema_Frame
 from pcapkit.protocols.schema.misc.pcap.header import RESOLUTION
 from pcapkit.utilities.compat import localcontext
+from pcapkit.utilities.datetime import fromtimestamp, totimestamp
 from pcapkit.utilities.exceptions import StreamEOFError, UnsupportedCall, stacklevel
-from pcapkit.utilities.timestamp import fromtimestamp
 from pcapkit.utilities.warnings import ProtocolWarning, RegistryWarning, warn
 
 if TYPE_CHECKING:
@@ -605,7 +605,7 @@ def _make_timestamp(self, timestamp: 'Optional[float | Decimal | dt_type | int]'
                     timestamp = decimal.Decimal(time.time())
             else:
                 if isinstance(timestamp, datetime.datetime):
-                    timestamp = timestamp.timestamp()
+                    timestamp = totimestamp(timestamp)
                 timestamp = decimal.Decimal(timestamp)
 
         if ts_sec is None:

pcapkit/protocols/misc/pcapng.py

@@ -210,9 +210,9 @@
 from pcapkit.protocols.schema.misc.pcapng import zigbee_key_held
 from pcapkit.protocols.schema.schema import Schema
 from pcapkit.utilities.compat import StrEnum, localcontext
+from pcapkit.utilities.datetime import fromtimestamp, totimestamp
 from pcapkit.utilities.exceptions import (BaseError, EnumValueError, FormatError, ProtocolError,
                                           StreamEOFError, UnsupportedCall, stacklevel)
-from pcapkit.utilities.timestamp import fromtimestamp
 from pcapkit.utilities.warnings import (AttributeWarning, DeprecatedFormatWarning, ProtocolWarning,
                                         RegistryWarning, warn)
 
@@ -2044,7 +2044,7 @@ def _make_timestamp(self, timestamp: 'Optional[float | Decimal | dt_type | int]'
                     timestamp = decimal.Decimal(time.time())
             else:
                 if isinstance(timestamp, datetime.datetime):
-                    timestamp = timestamp.timestamp()
+                    timestamp = totimestamp(timestamp)
                 timestamp = decimal.Decimal(timestamp)
 
             ts_info = int(

pcapkit/protocols/schema/internet/mh.py

@@ -61,8 +61,8 @@
                                             UInt64Field)
 from pcapkit.corekit.fields.strings import BitField, BytesField, PaddingField, StringField
 from pcapkit.protocols.schema.schema import EnumSchema, Schema, schema_final
+from pcapkit.utilities.datetime import fromtimestamp
 from pcapkit.utilities.logging import SPHINX_TYPE_CHECKING
-from pcapkit.utilities.timestamp import fromtimestamp
 
 __all__ = [
     'MH',

pcapkit/utilities/timestamp.py → pcapkit/utilities/datetime.py

@@ -2,22 +2,25 @@
 """Timestamp Conversion
 ======================
 
-.. module:: pcapkit.utilities.timestamp
+.. module:: pcapkit.utilities.datetime
 
-:mod:`pcapkit.utilities.timestamp` converts a UNIX timestamp to an aware
-:class:`~datetime.datetime` without the C library's range (:issue:`2099`).
+:mod:`pcapkit.utilities.datetime` converts a UNIX timestamp to an aware
+:class:`~datetime.datetime` without the C library's range (:issue:`2099`), and
+a :class:`~datetime.datetime` back to a UNIX timestamp without GraalPy's lost
+second before 1970 (:issue:`2134`).
 
 """
 
 import datetime
 import math
 import operator
+import sys
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from datetime import tzinfo
 
-__all__ = ['fromtimestamp']
+__all__ = ['fromtimestamp', 'totimestamp']
 
 # A UNIX timestamp as an aware datetime, without the C library (#2099).
 # :meth:`datetime.datetime.fromtimestamp` breaks the instant down with the
@@ -35,7 +38,9 @@
 # a 64-bit ``time_t`` or a ``tz`` offset that pushes the result out of range.
 # The one exception is a year past what a C ``int`` holds, from about 6.7e16
 # seconds, where glibc's own ``gmtime_r`` fails first and CPython raises
-# :exc:`OSError` instead; this raises :exc:`ValueError` there too.
+# :exc:`OSError` instead; this raises :exc:`ValueError` there too. On GraalPy a
+# :class:`~datetime.timezone` whose offset pushes the result out of range raises
+# :exc:`ValueError` from its own ``fromutc``, which this does not change (#2134).
 _EPOCH = datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc)
 #: First and last whole second since the UNIX epoch in years 1 to 9999.
 _EPOCH_MIN = -62_135_596_800
@@ -45,22 +50,45 @@
 #: A 64-bit ``time_t`` holds ``[-_TIME_T, _TIME_T)``.
 _TIME_T = 2**63
 
+# GraalPy's :meth:`datetime.datetime.timestamp` of an aware datetime counts the
+# whole seconds since the epoch toward zero and then adds the fraction, so an
+# instant before 1970 that has one comes out a second late: 23:59:59.5 on
+# December 31, 1969, UTC, gives ``0.5``, not ``-0.5`` (#2134, measured on
+# GraalPy 25.4.4). ``TimestampNode`` takes ``ChronoUnit.SECONDS.between`` of the
+# epoch and the instant, which truncates (graalpython, DateTimeBuiltins.java, at
+# graal-25.4.4; no upstream issue is filed yet). CPython and PyPy compute
+# ``(dt - epoch).total_seconds()`` for an aware datetime, which is right, so
+# there :func:`totimestamp` is the stdlib call itself.
+_TIMESTAMP_TRUNCATES = sys.implementation.name == 'graalpy'
+
 
 def fromtimestamp(timestamp: 'float | int', tz: 'tzinfo') -> 'datetime.datetime':
     """:meth:`datetime.datetime.fromtimestamp` with a ``tz``, minus the C library's range.
 
     The result equals ``datetime.datetime.fromtimestamp(timestamp, tz)`` on
-    64-bit Linux for every ``timestamp`` that call accepts, on every platform.
+    64-bit Linux for every ``timestamp`` that call accepts, on every platform:
+    whole seconds and microseconds split from a float as CPython splits them,
+    rounding half to even, then added to the UNIX epoch and rendered in ``tz``.
+    The standard library breaks the instant down with the C library's
+    ``gmtime()``, whose range on Windows is 1970 to 3000, so there it raises
+    :exc:`OSError` for instants :class:`~datetime.datetime` holds.
 
     Args:
         timestamp: Seconds since the UNIX epoch.
         tz: Timezone the result is rendered in.
 
+    Returns:
+        The instant ``timestamp`` names, rendered in ``tz``.
+
     Raises:
         ValueError: If ``timestamp`` is NaN, or names a year outside 1 to 9999.
         OverflowError: If ``timestamp`` is past a 64-bit ``time_t``, or ``tz``
             moves the result out of the range :class:`~datetime.datetime` holds.
 
+    .. versionadded:: 1.5.0
+
+       Every capture and MH timestamp is converted with it (:issue:`2099`).
+
     """
     if isinstance(timestamp, float):
         if math.isnan(timestamp):
@@ -78,3 +106,29 @@ def fromtimestamp(timestamp: 'float | int', tz: 'tzinfo') -> 'datetime.datetime'
         raise ValueError(f'year is out of range for timestamp {timestamp!r}')
     utc = _EPOCH + datetime.timedelta(seconds=seconds, microseconds=micro)
     return tz.fromutc(utc.replace(tzinfo=tz))
+
+
+def totimestamp(dt: 'datetime.datetime') -> 'float':
+    """:meth:`datetime.datetime.timestamp`, as CPython computes it, on every interpreter.
+
+    For an aware ``dt`` that is ``(dt - epoch).total_seconds()``. GraalPy's own
+    counts the whole seconds toward zero, so there an instant before 1970 with a
+    fraction of a second comes out one second late, and this computes it as
+    CPython does. A naive ``dt`` is local time, and is ``dt.timestamp()``
+    everywhere.
+
+    Args:
+        dt: Moment to convert.
+
+    Returns:
+        Seconds since the UNIX epoch.
+
+    .. versionadded:: 1.5.0
+
+       The PCAP and PCAP-NG writers and the MH replay protection option convert
+       a :class:`~datetime.datetime` with it (:issue:`2134`).
+
+    """
+    if _TIMESTAMP_TRUNCATES and dt.tzinfo is not None:
+        return (dt - _EPOCH).total_seconds()
+    return dt.timestamp()

tests/_support.py

@@ -4,6 +4,7 @@
 import collections.abc
 import contextlib
 import functools
+import gc
 import hashlib
 import importlib.util
 import inspect
@@ -15,7 +16,7 @@
 import time
 import types
 import unittest
-from typing import Iterable, Iterator, Optional
+from typing import Callable, Iterable, Iterator, Optional
 
 from tests._tiers import (ROOT, SAMPLE_ROOT, REGENERATE_SAMPLES_CMD,
                           GeneratedFixtureInUnitTierError, check_unit_tier_read)
@@ -1168,3 +1169,39 @@ def close_extractor(extractor: object) -> None:
         _close_quietly(stream)
     finally:
         _close_quietly(engine)
+
+
+#: Seconds :func:`collect_until` keeps collecting for a finaliser after the first
+#: :func:`gc.collect`. CPython runs the finaliser before ``collect`` returns, so it
+#: gets no grace. GraalPy's ``collect`` forces a full Java GC, sleeps 15 ms "just
+#: trying to give the GC more time" and then runs only the weak reference
+#: callbacks queued by then (graalpython ``GcModuleBuiltins.javaCollect``, at
+#: graal-25.4.4), so on a loaded runner the finaliser can run after it returns
+#: (#1963). A finaliser that never runs still fails once the grace is spent.
+#: PyPy frees an extractor, and so runs the finaliser, one collection later
+#: than CPython: the class still defines ``__del__`` (#1958, measured on PyPy
+#: 8.0.1), so a short grace lets a second collection run there. Shared by
+#: every test that waits on a finaliser, a reader's buffer file among them
+#: (#2134).
+FINALISER_GRACE = {'graalpy': 10.0, 'pypy': 1.0}.get(sys.implementation.name, 0.0)
+
+
+def collect_until(done: 'Callable[[], bool]', grace: float = FINALISER_GRACE) -> bool:
+    """Collect garbage until ``done()`` holds or ``grace`` seconds have passed.
+
+    Args:
+        done: Zero-argument callable, checked after each collection.
+        grace: Seconds to keep collecting after the first collection.
+
+    Returns:
+        Whether ``done()`` held before the grace ran out.
+
+    """
+    deadline = time.monotonic() + grace
+    while True:
+        gc.collect()
+        if done():
+            return True
+        if time.monotonic() >= deadline:
+            return False
+        time.sleep(0.05)

tests/corekit/test_io_buffer_save_tempfile_2098_unit.py

@@ -12,7 +12,6 @@
 
 import builtins
 import errno
-import gc
 import importlib.util
 import io
 import os
@@ -22,7 +21,7 @@
 import warnings
 from unittest import mock
 
-from tests._support import bootstrap_core_modules, load_module, reimport_once_per_class
+from tests._support import bootstrap_core_modules, collect_until, load_module, reimport_once_per_class
 
 #: Test tier (#1958): exercises finalisation and stream semantics.
 __tier__ = 'ALL'
@@ -86,10 +85,12 @@ def test_collection_removes_the_temporary_file_of_an_unclosed_reader(self) -> No
         reader = self.SeekableReader(io.BytesIO(b'abcdefghij'), buffer_size=4, buffer_save=True)
         path = reader._buffer_path  # pylint: disable=protected-access
         reader.read()
+        # the reader's finaliser runs in the collection on CPython, and can run
+        # after it returns on GraalPy, so this waits as long as it may (#2134)
         with warnings.catch_warnings():
             warnings.simplefilter('ignore', ResourceWarning)
             del reader
-            gc.collect()
+            collect_until(lambda: not os.path.exists(path))
         self.assertFalse(os.path.exists(path))
 
     def test_a_buffer_path_the_caller_gave_is_kept(self) -> None:

tests/foundation/test_extraction_ownership.py

@@ -51,16 +51,11 @@
 import importlib.util
 import io
 import os
-import sys
 import time
 import unittest
-from typing import TYPE_CHECKING
 from unittest import mock
 
-from tests._support import sample_path
-
-if TYPE_CHECKING:
-    from typing import Callable
+from tests._support import collect_until, sample_path
 
 #: Test tier (#1958): depends on garbage collection, weak references or object
 #: finalisation.
@@ -73,39 +68,6 @@
 #: Linux; the :func:`builtins.open` measurement below covers the rest.
 HAS_PROC_FD = os.path.isdir('/proc/self/fd')
 
-#: Seconds :func:`collect_until` keeps collecting for a finaliser after the first
-#: :func:`gc.collect`. CPython runs the finaliser before ``collect`` returns, so it
-#: gets no grace. GraalPy's ``collect`` forces a full Java GC, sleeps 15 ms "just
-#: trying to give the GC more time" and then runs only the weak reference
-#: callbacks queued by then (graalpython ``GcModuleBuiltins.javaCollect``, at
-#: graal-25.4.4), so on a loaded runner the finaliser can run after it returns
-#: (#1963). A finaliser that never runs still fails once the grace is spent.
-#: PyPy frees the extractor, and so runs the finaliser, one collection later
-#: than CPython: the class still defines ``__del__`` (#1958, measured on PyPy
-#: 8.0.1), so a short grace lets a second collection run there.
-FINALISER_GRACE = {'graalpy': 10.0, 'pypy': 1.0}.get(sys.implementation.name, 0.0)
-
-
-def collect_until(done: 'Callable[[], bool]', grace: float = FINALISER_GRACE) -> bool:
-    """Collect garbage until ``done()`` holds or ``grace`` seconds have passed.
-
-    Args:
-        done: Zero-argument callable, checked after each collection.
-        grace: Seconds to keep collecting after the first collection.
-
-    Returns:
-        Whether ``done()`` held before the grace ran out.
-
-    """
-    deadline = time.monotonic() + grace
-    while True:
-        gc.collect()
-        if done():
-            return True
-        if time.monotonic() >= deadline:
-            return False
-        time.sleep(0.05)
-
 
 def fd_count(target: str) -> int:
     """How many of this process's open descriptors resolve to ``target``.

tests/protocols/misc/pcap/test_header_frame_unit.py

@@ -296,7 +296,7 @@ class Other(Raw):
         bad_time._fnum = 6
         bad_time._file = io.BytesIO(b'')
         # Faked through frame.py's own binding of
-        # :func:`pcapkit.utilities.timestamp.fromtimestamp`, which it calls
+        # :func:`pcapkit.utilities.datetime.fromtimestamp`, which it calls
         # instead of :meth:`datetime.datetime.fromtimestamp` since #2099.
         with mock.patch('pcapkit.protocols.misc.pcap.frame.fromtimestamp', side_effect=ValueError), \
                 mock.patch.object(Frame, '_decode_next_layer',