From d5c0e7c53420e5928c9555e0c8f735639c96d1b3 Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Fri, 23 Oct 2026 23:48:19 -0400 Subject: [PATCH 1/2] fix(utilities): encode pre-1970 datetimes on GraalPy without its lost second (#2134) GraalPy's aware datetime.timestamp() counts whole seconds toward zero, so an instant before 1970 with a fraction encoded a second late in the MH replay protection option, the PCAP frame header and PCAP-NG blocks. - pcapkit.utilities.timestamp.totimestamp computes (dt - epoch).total_seconds() for an aware dt on GraalPy, and is dt.timestamp() everywhere else; the three encoders call it. The module's rst page renders both functions with autofunction, their docstrings carrying the prose and versionadded notes. - test_fromtimestamp_windows_range_2099_unit: GraalPy's timezone.fromutc raises ValueError, not OverflowError, for a result before year 1; the test accepts it there. - test_io_buffer_save_tempfile_2098_unit: wait for the reader's finaliser with collect_until, since GraalPy can run it after gc.collect() returns; the helper moves from test_extraction_ownership to tests/_support. - New tests/utilities/test_totimestamp_graalpy_2134_unit emulates GraalPy's timestamp() on any interpreter. Target tests pass on CPython 3.14 and GraalPy 25.4.4; tier guard passes. --- docs/source/pcapkit/utilities/timestamp.rst | 25 +-- pcapkit/protocols/internet/mh.py | 4 +- pcapkit/protocols/misc/pcap/frame.py | 4 +- pcapkit/protocols/misc/pcapng.py | 4 +- pcapkit/utilities/timestamp.py | 62 +++++- tests/_support.py | 39 +++- .../test_io_buffer_save_tempfile_2098_unit.py | 7 +- tests/foundation/test_extraction_ownership.py | 40 +--- ...t_fromtimestamp_windows_range_2099_unit.py | 15 +- .../test_totimestamp_graalpy_2134_unit.py | 194 ++++++++++++++++++ 10 files changed, 321 insertions(+), 73 deletions(-) create mode 100644 tests/utilities/test_totimestamp_graalpy_2134_unit.py diff --git a/docs/source/pcapkit/utilities/timestamp.rst b/docs/source/pcapkit/utilities/timestamp.rst index bf2eb1790f..73b1569fb5 100644 --- a/docs/source/pcapkit/utilities/timestamp.rst +++ b/docs/source/pcapkit/utilities/timestamp.rst @@ -4,23 +4,12 @@ Timestamp Conversion .. module:: pcapkit.utilities.timestamp :mod:`pcapkit.utilities.timestamp` converts a UNIX timestamp to an aware -:class:`~datetime.datetime` without the C library's range (:issue:`2099`). It is -shared by the PCAP and PCAP-NG readers and the MH timestamp options, which is -why it lives here rather than beside any one of them. +:class:`~datetime.datetime` without the C library's range (:issue:`2099`), and +a :class:`~datetime.datetime` back to a UNIX timestamp without GraalPy's lost +second before 1970 (:issue:`2134`). It is shared by the PCAP and PCAP-NG readers +and writers and the MH timestamp options, which is why it lives here rather than +beside any one of them. -.. function:: fromtimestamp(timestamp, tz) +.. autofunction:: pcapkit.utilities.timestamp.fromtimestamp - ``datetime.datetime.fromtimestamp(timestamp, tz)`` as 64-bit Linux computes - it, on every platform: whole seconds and microseconds split from a float as - CPython splits them, rounding half to even, then added to the UNIX epoch and - rendered in ``tz``. The standard library breaks the instant down with the C - library's ``gmtime()``, whose range on Windows is 1970 to 3000, so there it - raises :exc:`OSError` for instants :class:`~datetime.datetime` holds. - - :raises ValueError: if ``timestamp`` is NaN or names a year outside 1 to 9999 - :raises OverflowError: if ``timestamp`` is past a 64-bit ``time_t``, or ``tz`` - moves the result out of range - - .. versionadded:: 1.5.0 - - Every capture and MH timestamp is converted with it (:issue:`2099`). +.. autofunction:: pcapkit.utilities.timestamp.totimestamp diff --git a/pcapkit/protocols/internet/mh.py b/pcapkit/protocols/internet/mh.py index 16d3a95d97..f7e593e192 100644 --- a/pcapkit/protocols/internet/mh.py +++ b/pcapkit/protocols/internet/mh.py @@ -497,7 +497,7 @@ from pcapkit.protocols.schema.internet.mh import VendorSpecificOption as Schema_VendorSpecificOption from pcapkit.protocols.schema.schema import Schema from pcapkit.utilities.exceptions import EnumValueError, ProtocolError, UnsupportedCall -from pcapkit.utilities.timestamp import fromtimestamp +from pcapkit.utilities.timestamp import fromtimestamp, totimestamp from pcapkit.utilities.warnings import ProtocolWarning, RegistryWarning, warn if TYPE_CHECKING: @@ -8383,7 +8383,7 @@ def _make_opt_mesg_id(self, type: 'Enum_Option', option: 'Optional[Data_MesgIDOp if timestamp is None: interval = interval or datetime.datetime.now(datetime.timezone.utc) - int_ts = interval.timestamp() + int_ts = totimestamp(interval) ts_sec = math.floor(int_ts) ts_frc = min(round((int_ts - ts_sec) * 2**32), 0xFFFF_FFFF) # 1/2**32 seconds per unit diff --git a/pcapkit/protocols/misc/pcap/frame.py b/pcapkit/protocols/misc/pcap/frame.py index 81deb9d44f..6381f1ccb2 100644 --- a/pcapkit/protocols/misc/pcap/frame.py +++ b/pcapkit/protocols/misc/pcap/frame.py @@ -49,7 +49,7 @@ from pcapkit.protocols.schema.misc.pcap.header import RESOLUTION from pcapkit.utilities.compat import localcontext from pcapkit.utilities.exceptions import StreamEOFError, UnsupportedCall, stacklevel -from pcapkit.utilities.timestamp import fromtimestamp +from pcapkit.utilities.timestamp import fromtimestamp, totimestamp from pcapkit.utilities.warnings import ProtocolWarning, RegistryWarning, warn if TYPE_CHECKING: @@ -605,7 +605,7 @@ def _make_timestamp(self, timestamp: 'Optional[float | Decimal | dt_type | int]' timestamp = decimal.Decimal(time.time()) else: if isinstance(timestamp, datetime.datetime): - timestamp = timestamp.timestamp() + timestamp = totimestamp(timestamp) timestamp = decimal.Decimal(timestamp) if ts_sec is None: diff --git a/pcapkit/protocols/misc/pcapng.py b/pcapkit/protocols/misc/pcapng.py index 84ced594ad..e2ccd33025 100644 --- a/pcapkit/protocols/misc/pcapng.py +++ b/pcapkit/protocols/misc/pcapng.py @@ -212,7 +212,7 @@ from pcapkit.utilities.compat import StrEnum, localcontext from pcapkit.utilities.exceptions import (BaseError, EnumValueError, FormatError, ProtocolError, StreamEOFError, UnsupportedCall, stacklevel) -from pcapkit.utilities.timestamp import fromtimestamp +from pcapkit.utilities.timestamp import fromtimestamp, totimestamp from pcapkit.utilities.warnings import (AttributeWarning, DeprecatedFormatWarning, ProtocolWarning, RegistryWarning, warn) @@ -2044,7 +2044,7 @@ def _make_timestamp(self, timestamp: 'Optional[float | Decimal | dt_type | int]' timestamp = decimal.Decimal(time.time()) else: if isinstance(timestamp, datetime.datetime): - timestamp = timestamp.timestamp() + timestamp = totimestamp(timestamp) timestamp = decimal.Decimal(timestamp) ts_info = int( diff --git a/pcapkit/utilities/timestamp.py b/pcapkit/utilities/timestamp.py index 53949e751c..256f8b8d55 100644 --- a/pcapkit/utilities/timestamp.py +++ b/pcapkit/utilities/timestamp.py @@ -5,19 +5,22 @@ .. module:: pcapkit.utilities.timestamp :mod:`pcapkit.utilities.timestamp` converts a UNIX timestamp to an aware -:class:`~datetime.datetime` without the C library's range (:issue:`2099`). +:class:`~datetime.datetime` without the C library's range (:issue:`2099`), and +a :class:`~datetime.datetime` back to a UNIX timestamp without GraalPy's lost +second before 1970 (:issue:`2134`). """ import datetime import math import operator +import sys from typing import TYPE_CHECKING if TYPE_CHECKING: from datetime import tzinfo -__all__ = ['fromtimestamp'] +__all__ = ['fromtimestamp', 'totimestamp'] # A UNIX timestamp as an aware datetime, without the C library (#2099). # :meth:`datetime.datetime.fromtimestamp` breaks the instant down with the @@ -35,7 +38,9 @@ # a 64-bit ``time_t`` or a ``tz`` offset that pushes the result out of range. # The one exception is a year past what a C ``int`` holds, from about 6.7e16 # seconds, where glibc's own ``gmtime_r`` fails first and CPython raises -# :exc:`OSError` instead; this raises :exc:`ValueError` there too. +# :exc:`OSError` instead; this raises :exc:`ValueError` there too. On GraalPy a +# :class:`~datetime.timezone` whose offset pushes the result out of range raises +# :exc:`ValueError` from its own ``fromutc``, which this does not change (#2134). _EPOCH = datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc) #: First and last whole second since the UNIX epoch in years 1 to 9999. _EPOCH_MIN = -62_135_596_800 @@ -45,22 +50,45 @@ #: A 64-bit ``time_t`` holds ``[-_TIME_T, _TIME_T)``. _TIME_T = 2**63 +# GraalPy's :meth:`datetime.datetime.timestamp` of an aware datetime counts the +# whole seconds since the epoch toward zero and then adds the fraction, so an +# instant before 1970 that has one comes out a second late: 23:59:59.5 on +# December 31, 1969, UTC, gives ``0.5``, not ``-0.5`` (#2134, measured on +# GraalPy 25.4.4). ``TimestampNode`` takes ``ChronoUnit.SECONDS.between`` of the +# epoch and the instant, which truncates (graalpython, DateTimeBuiltins.java, at +# graal-25.4.4; no upstream issue is filed yet). CPython and PyPy compute +# ``(dt - epoch).total_seconds()`` for an aware datetime, which is right, so +# there :func:`totimestamp` is the stdlib call itself. +_TIMESTAMP_TRUNCATES = sys.implementation.name == 'graalpy' + def fromtimestamp(timestamp: 'float | int', tz: 'tzinfo') -> 'datetime.datetime': """:meth:`datetime.datetime.fromtimestamp` with a ``tz``, minus the C library's range. The result equals ``datetime.datetime.fromtimestamp(timestamp, tz)`` on - 64-bit Linux for every ``timestamp`` that call accepts, on every platform. + 64-bit Linux for every ``timestamp`` that call accepts, on every platform: + whole seconds and microseconds split from a float as CPython splits them, + rounding half to even, then added to the UNIX epoch and rendered in ``tz``. + The standard library breaks the instant down with the C library's + ``gmtime()``, whose range on Windows is 1970 to 3000, so there it raises + :exc:`OSError` for instants :class:`~datetime.datetime` holds. Args: timestamp: Seconds since the UNIX epoch. tz: Timezone the result is rendered in. + Returns: + The instant ``timestamp`` names, rendered in ``tz``. + Raises: ValueError: If ``timestamp`` is NaN, or names a year outside 1 to 9999. OverflowError: If ``timestamp`` is past a 64-bit ``time_t``, or ``tz`` moves the result out of the range :class:`~datetime.datetime` holds. + .. versionadded:: 1.5.0 + + Every capture and MH timestamp is converted with it (:issue:`2099`). + """ if isinstance(timestamp, float): if math.isnan(timestamp): @@ -78,3 +106,29 @@ def fromtimestamp(timestamp: 'float | int', tz: 'tzinfo') -> 'datetime.datetime' raise ValueError(f'year is out of range for timestamp {timestamp!r}') utc = _EPOCH + datetime.timedelta(seconds=seconds, microseconds=micro) return tz.fromutc(utc.replace(tzinfo=tz)) + + +def totimestamp(dt: 'datetime.datetime') -> 'float': + """:meth:`datetime.datetime.timestamp`, as CPython computes it, on every interpreter. + + For an aware ``dt`` that is ``(dt - epoch).total_seconds()``. GraalPy's own + counts the whole seconds toward zero, so there an instant before 1970 with a + fraction of a second comes out one second late, and this computes it as + CPython does. A naive ``dt`` is local time, and is ``dt.timestamp()`` + everywhere. + + Args: + dt: Moment to convert. + + Returns: + Seconds since the UNIX epoch. + + .. versionadded:: 1.5.0 + + The PCAP and PCAP-NG writers and the MH replay protection option convert + a :class:`~datetime.datetime` with it (:issue:`2134`). + + """ + if _TIMESTAMP_TRUNCATES and dt.tzinfo is not None: + return (dt - _EPOCH).total_seconds() + return dt.timestamp() diff --git a/tests/_support.py b/tests/_support.py index 408aac49f7..2a20b1b45e 100644 --- a/tests/_support.py +++ b/tests/_support.py @@ -4,6 +4,7 @@ import collections.abc import contextlib import functools +import gc import hashlib import importlib.util import inspect @@ -15,7 +16,7 @@ import time import types import unittest -from typing import Iterable, Iterator, Optional +from typing import Callable, Iterable, Iterator, Optional from tests._tiers import (ROOT, SAMPLE_ROOT, REGENERATE_SAMPLES_CMD, GeneratedFixtureInUnitTierError, check_unit_tier_read) @@ -1168,3 +1169,39 @@ def close_extractor(extractor: object) -> None: _close_quietly(stream) finally: _close_quietly(engine) + + +#: Seconds :func:`collect_until` keeps collecting for a finaliser after the first +#: :func:`gc.collect`. CPython runs the finaliser before ``collect`` returns, so it +#: gets no grace. GraalPy's ``collect`` forces a full Java GC, sleeps 15 ms "just +#: trying to give the GC more time" and then runs only the weak reference +#: callbacks queued by then (graalpython ``GcModuleBuiltins.javaCollect``, at +#: graal-25.4.4), so on a loaded runner the finaliser can run after it returns +#: (#1963). A finaliser that never runs still fails once the grace is spent. +#: PyPy frees an extractor, and so runs the finaliser, one collection later +#: than CPython: the class still defines ``__del__`` (#1958, measured on PyPy +#: 8.0.1), so a short grace lets a second collection run there. Shared by +#: every test that waits on a finaliser, a reader's buffer file among them +#: (#2134). +FINALISER_GRACE = {'graalpy': 10.0, 'pypy': 1.0}.get(sys.implementation.name, 0.0) + + +def collect_until(done: 'Callable[[], bool]', grace: float = FINALISER_GRACE) -> bool: + """Collect garbage until ``done()`` holds or ``grace`` seconds have passed. + + Args: + done: Zero-argument callable, checked after each collection. + grace: Seconds to keep collecting after the first collection. + + Returns: + Whether ``done()`` held before the grace ran out. + + """ + deadline = time.monotonic() + grace + while True: + gc.collect() + if done(): + return True + if time.monotonic() >= deadline: + return False + time.sleep(0.05) diff --git a/tests/corekit/test_io_buffer_save_tempfile_2098_unit.py b/tests/corekit/test_io_buffer_save_tempfile_2098_unit.py index 4ee839f749..80ed24c299 100644 --- a/tests/corekit/test_io_buffer_save_tempfile_2098_unit.py +++ b/tests/corekit/test_io_buffer_save_tempfile_2098_unit.py @@ -12,7 +12,6 @@ import builtins import errno -import gc import importlib.util import io import os @@ -22,7 +21,7 @@ import warnings from unittest import mock -from tests._support import bootstrap_core_modules, load_module, reimport_once_per_class +from tests._support import bootstrap_core_modules, collect_until, load_module, reimport_once_per_class #: Test tier (#1958): exercises finalisation and stream semantics. __tier__ = 'ALL' @@ -86,10 +85,12 @@ def test_collection_removes_the_temporary_file_of_an_unclosed_reader(self) -> No reader = self.SeekableReader(io.BytesIO(b'abcdefghij'), buffer_size=4, buffer_save=True) path = reader._buffer_path # pylint: disable=protected-access reader.read() + # the reader's finaliser runs in the collection on CPython, and can run + # after it returns on GraalPy, so this waits as long as it may (#2134) with warnings.catch_warnings(): warnings.simplefilter('ignore', ResourceWarning) del reader - gc.collect() + collect_until(lambda: not os.path.exists(path)) self.assertFalse(os.path.exists(path)) def test_a_buffer_path_the_caller_gave_is_kept(self) -> None: diff --git a/tests/foundation/test_extraction_ownership.py b/tests/foundation/test_extraction_ownership.py index f612ba76bb..9614ecae0a 100644 --- a/tests/foundation/test_extraction_ownership.py +++ b/tests/foundation/test_extraction_ownership.py @@ -51,16 +51,11 @@ import importlib.util import io import os -import sys import time import unittest -from typing import TYPE_CHECKING from unittest import mock -from tests._support import sample_path - -if TYPE_CHECKING: - from typing import Callable +from tests._support import collect_until, sample_path #: Test tier (#1958): depends on garbage collection, weak references or object #: finalisation. @@ -73,39 +68,6 @@ #: Linux; the :func:`builtins.open` measurement below covers the rest. HAS_PROC_FD = os.path.isdir('/proc/self/fd') -#: Seconds :func:`collect_until` keeps collecting for a finaliser after the first -#: :func:`gc.collect`. CPython runs the finaliser before ``collect`` returns, so it -#: gets no grace. GraalPy's ``collect`` forces a full Java GC, sleeps 15 ms "just -#: trying to give the GC more time" and then runs only the weak reference -#: callbacks queued by then (graalpython ``GcModuleBuiltins.javaCollect``, at -#: graal-25.4.4), so on a loaded runner the finaliser can run after it returns -#: (#1963). A finaliser that never runs still fails once the grace is spent. -#: PyPy frees the extractor, and so runs the finaliser, one collection later -#: than CPython: the class still defines ``__del__`` (#1958, measured on PyPy -#: 8.0.1), so a short grace lets a second collection run there. -FINALISER_GRACE = {'graalpy': 10.0, 'pypy': 1.0}.get(sys.implementation.name, 0.0) - - -def collect_until(done: 'Callable[[], bool]', grace: float = FINALISER_GRACE) -> bool: - """Collect garbage until ``done()`` holds or ``grace`` seconds have passed. - - Args: - done: Zero-argument callable, checked after each collection. - grace: Seconds to keep collecting after the first collection. - - Returns: - Whether ``done()`` held before the grace ran out. - - """ - deadline = time.monotonic() + grace - while True: - gc.collect() - if done(): - return True - if time.monotonic() >= deadline: - return False - time.sleep(0.05) - def fd_count(target: str) -> int: """How many of this process's open descriptors resolve to ``target``. diff --git a/tests/protocols/misc/test_fromtimestamp_windows_range_2099_unit.py b/tests/protocols/misc/test_fromtimestamp_windows_range_2099_unit.py index 362dde6422..711f4af343 100644 --- a/tests/protocols/misc/test_fromtimestamp_windows_range_2099_unit.py +++ b/tests/protocols/misc/test_fromtimestamp_windows_range_2099_unit.py @@ -34,7 +34,8 @@ from typing import Any, Iterator #: Test tier (#1958): splits floats into seconds and microseconds, which PyPy's -#: and GraalPy's pure-Python datetime do in code of their own. +#: and GraalPy's pure-Python datetime do in code of their own, and gates one +#: error class on GraalPy. __tier__ = 'ALL' UTC = datetime.timezone.utc @@ -48,6 +49,16 @@ _REAL_DATETIME = datetime.datetime +#: What :meth:`datetime.timezone.fromutc` raises for a result before year 1. +#: CPython's C and pure-Python ``datetime`` both raise :exc:`OverflowError`, as +#: ``datetime + timedelta`` does. GraalPy's ``FromUtcNode`` adds the offset to a +#: Java ``LocalDateTime``, which has no year-1 floor, and its constructor then +#: raises ``ValueError: year 0 is out of range`` (#2134, measured on GraalPy +#: 25.4.4; graalpython ``TimeZoneBuiltins.java`` and ``DatetimeModuleBuiltins +#: .addOffsetToDateTime`` at graal-25.4.4, no upstream issue filed yet). Its +#: ``datetime + timedelta`` still raises :exc:`OverflowError`. +ZONE_OVERFLOW = ValueError if sys.implementation.name == 'graalpy' else OverflowError + class WindowsDateTime(_REAL_DATETIME): """A ``datetime`` whose ``fromtimestamp`` has the Windows CRT's range.""" @@ -130,7 +141,7 @@ def test_errors_are_cpythons(self) -> None: with self.subTest(timestamp=value), self.assertRaises(error): fromtimestamp(value, UTC) # the instant is in range and the zone pushes it out - with self.assertRaises(OverflowError): + with self.assertRaises(ZONE_OVERFLOW): fromtimestamp(-62135596800.0, datetime.timezone(datetime.timedelta(hours=-5))) diff --git a/tests/utilities/test_totimestamp_graalpy_2134_unit.py b/tests/utilities/test_totimestamp_graalpy_2134_unit.py new file mode 100644 index 0000000000..92d1e7a5a7 --- /dev/null +++ b/tests/utilities/test_totimestamp_graalpy_2134_unit.py @@ -0,0 +1,194 @@ +# -*- coding: utf-8 -*- +"""GitHub issue #2134: an aware datetime before 1970 encodes to the instant it names on GraalPy. + +GraalPy's :meth:`datetime.datetime.timestamp` counts the whole seconds of an +aware datetime toward zero and then adds the fraction, so 23:59:59.5 on December +31, 1969, UTC, gives ``0.5`` where CPython gives ``-0.5``. The MH replay +protection option, the PCAP frame header and the PCAP-NG packet blocks built +from such a datetime were therefore one second late there, and decoding gave +back another instant than the one encoded. + +:class:`GraalPyDateTime` applies GraalPy's rule on any interpreter, and the +encoders are run under :func:`graalpy_timestamps`, which turns on +:func:`pcapkit.utilities.timestamp.totimestamp`'s GraalPy branch. + +:mod:`pcapkit` is imported inside each test, after +:func:`~tests._support.reimport_once_per_class`. + +""" + +import contextlib +import datetime +import decimal +import sys +import unittest +from typing import TYPE_CHECKING +from unittest import mock + +if TYPE_CHECKING: + from typing import Iterator + +#: Test tier (#1958): works around a GraalPy datetime defect, behind an +#: implementation gate. +__tier__ = 'ALL' + +UTC = datetime.timezone.utc +EPOCH = datetime.datetime(1970, 1, 1, tzinfo=UTC) + +#: Aware instants before, at and after the epoch, with and without a fraction. +INSTANTS = ( + datetime.datetime(1, 1, 1, 0, 0, 0, 1, tzinfo=UTC), + datetime.datetime(1900, 1, 1, tzinfo=UTC), + datetime.datetime(1950, 6, 1, 12, 0, 0, 500_000, tzinfo=UTC), + datetime.datetime(1969, 12, 31, 23, 59, 59, 500_000, tzinfo=UTC), + datetime.datetime(1969, 12, 31, 18, 59, 58, 250_000, tzinfo=datetime.timezone(datetime.timedelta(hours=-5))), + datetime.datetime(1970, 1, 1, tzinfo=UTC), + datetime.datetime(1970, 1, 1, 0, 0, 0, 500_000, tzinfo=UTC), + datetime.datetime(2026, 10, 23, 12, 34, 56, 789_012, tzinfo=datetime.timezone(datetime.timedelta(hours=8))), + datetime.datetime(9999, 12, 31, 23, 59, 59, 999_999, tzinfo=UTC), +) + + +class GraalPyDateTime(datetime.datetime): + """A ``datetime`` whose aware :meth:`timestamp` truncates its whole seconds, as GraalPy's does.""" + + def timestamp(self) -> float: + if self.tzinfo is None: + return super().timestamp() + delta = self - EPOCH + whole = delta.days * 86400 + delta.seconds + if whole < 0 and delta.microseconds: + whole += 1 # ``ChronoUnit.SECONDS.between`` rounds toward zero + return whole + delta.microseconds / 1_000_000 + + +def graal(instant: 'datetime.datetime') -> 'GraalPyDateTime': + """``instant`` as a :class:`GraalPyDateTime`.""" + return GraalPyDateTime(instant.year, instant.month, instant.day, instant.hour, instant.minute, + instant.second, instant.microsecond, tzinfo=instant.tzinfo) + + +@contextlib.contextmanager +def graalpy_timestamps() -> 'Iterator[None]': + """Run :func:`~pcapkit.utilities.timestamp.totimestamp` as it runs on GraalPy.""" + with mock.patch('pcapkit.utilities.timestamp._TIMESTAMP_TRUNCATES', True): + yield + + +class TestGraalPyEmulation(unittest.TestCase): + """The stand-in loses the second GraalPy loses, or every test below proves nothing.""" + + def test_measured_values(self) -> None: + # measured on GraalPy 25.4.4 for the same instants (#2134) + cases = ( + (datetime.datetime(1950, 6, 1, 12, 0, 0, 500_000, tzinfo=UTC), -618062398.5), + (datetime.datetime(1969, 12, 31, 23, 59, 59, 500_000, tzinfo=UTC), 0.5), + (datetime.datetime(1969, 12, 31, 23, 59, 58, 250_000, tzinfo=UTC), -0.75), + (datetime.datetime(1970, 1, 1, 0, 0, 0, 500_000, tzinfo=UTC), 0.5), + ) + for instant, expected in cases: + with self.subTest(instant=instant): + self.assertEqual(graal(instant).timestamp(), expected) + + +class TestToTimestamp(unittest.TestCase): + """:func:`pcapkit.utilities.timestamp.totimestamp` is CPython's :meth:`~datetime.datetime.timestamp`.""" + + def setUp(self) -> None: + from tests._support import reimport_once_per_class + + reimport_once_per_class(self) + + def test_aware_is_the_seconds_since_the_epoch(self) -> None: + from pcapkit.utilities.timestamp import totimestamp + + for instant in INSTANTS: + with self.subTest(instant=instant): + self.assertEqual(totimestamp(instant), (instant - EPOCH).total_seconds()) + + @unittest.skipIf(sys.implementation.name == 'graalpy', "GraalPy's own timestamp() is the defect") + def test_is_the_stdlib_call_off_graalpy(self) -> None: + from pcapkit.utilities.timestamp import totimestamp + + naive = datetime.datetime(1950, 6, 1, 12, 0, 0, 500_000) + for instant in (*INSTANTS, naive): + with self.subTest(instant=instant): + self.assertEqual(totimestamp(instant).hex(), instant.timestamp().hex()) + + def test_graalpys_lost_second_is_restored(self) -> None: + from pcapkit.utilities.timestamp import totimestamp + + with graalpy_timestamps(): + for instant in INSTANTS: + with self.subTest(instant=instant): + self.assertEqual(totimestamp(graal(instant)), (instant - EPOCH).total_seconds()) + + def test_naive_is_local_time_everywhere(self) -> None: + from pcapkit.utilities.timestamp import totimestamp + + naive = datetime.datetime(1950, 6, 1, 12, 0, 0, 500_000) + with graalpy_timestamps(): + self.assertEqual(totimestamp(naive), naive.timestamp()) + + def test_an_offsetless_zone_fails_as_on_cpython(self) -> None: + from pcapkit.utilities.timestamp import totimestamp + + class NoOffset(datetime.tzinfo): + def utcoffset(self, dt: 'object') -> None: + return None + + with graalpy_timestamps(), self.assertRaises(TypeError): + totimestamp(datetime.datetime(1950, 6, 1, tzinfo=NoOffset())) + + +class TestEncodersOnGraalPy(unittest.TestCase): + """Every encoder that takes a datetime encodes the instant it names under GraalPy's rule.""" + + def setUp(self) -> None: + from tests._support import reimport_once_per_class + + reimport_once_per_class(self) + + def test_mh_mesg_id(self) -> None: + from pcapkit.const.mh.option import Option + from pcapkit.protocols.internet.mh import MH + from pcapkit.protocols.schema.internet.mh import MesgIDOption + + proto = object.__new__(MH) + for instant in INSTANTS[1:5]: + with self.subTest(instant=instant): + expected = proto._make_opt_mesg_id(Option.MESG_ID_OPTION_TYPE, interval=instant).pack() + with graalpy_timestamps(): + made = proto._make_opt_mesg_id(Option.MESG_ID_OPTION_TYPE, interval=graal(instant)) + self.assertEqual(made.pack().hex(), expected.hex()) + self.assertEqual(MesgIDOption.unpack(made.pack()).timestamp, instant) + + def test_pcap_frame(self) -> None: + from pcapkit.protocols.misc.pcap.frame import Frame + + frame = object.__new__(Frame) + instant = datetime.datetime(1969, 12, 31, 23, 59, 58, 250_000, tzinfo=UTC) + for nanosecond in (False, True): + with self.subTest(nanosecond=nanosecond): + expected = frame._make_timestamp(timestamp=instant, nanosecond=nanosecond) + with graalpy_timestamps(): + made = frame._make_timestamp(timestamp=graal(instant), nanosecond=nanosecond) + self.assertEqual(made, expected) + + def test_pcapng_block(self) -> None: + from pcapkit.protocols.misc.pcapng import PCAPNG + + pcapng = object.__new__(PCAPNG) + instant = datetime.datetime(1950, 6, 1, 12, 0, 0, 500_000, tzinfo=UTC) + offset = -700_000_000 # if_tsoffset, so the scaled value is positive + with mock.patch.object(PCAPNG, '_get_offset', lambda self, interface_id: decimal.Decimal(offset)), \ + mock.patch.object(PCAPNG, '_get_resolution', lambda self, interface_id: 1_000_000): + expected = pcapng._make_timestamp(instant) + with graalpy_timestamps(): + made = pcapng._make_timestamp(graal(instant)) + self.assertEqual(made, expected) + self.assertEqual(((made[0] << 32) | made[1]) / 1_000_000 + offset, (instant - EPOCH).total_seconds()) + + +if __name__ == '__main__': + unittest.main() From 586b819db7506aaea5d2e86d6d8d80368df39e65 Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Sat, 24 Oct 2026 09:09:19 -0400 Subject: [PATCH 2/2] refactor(utilities): rename the timestamp module to datetime (#2134) Per the owner's ruling on #2136, pcapkit.utilities.timestamp becomes pcapkit.utilities.datetime, reusing the stdlib name as utilities.logging and utilities.warnings do. The old path was added by #2111 and is in no release, so it is deleted with no re-export. - move the module and its docs page, and update the toctree - update the MH, PCAP frame and PCAP-NG importers and the tests - test that the new module exposes fromtimestamp and totimestamp, imports the stdlib datetime, and that the old path no longer imports --- .../utilities/{timestamp.rst => datetime.rst} | 8 ++-- docs/source/pcapkit/utilities/index.rst | 2 +- pcapkit/protocols/internet/mh.py | 2 +- pcapkit/protocols/misc/pcap/frame.py | 2 +- pcapkit/protocols/misc/pcapng.py | 2 +- pcapkit/protocols/schema/internet/mh.py | 2 +- .../utilities/{timestamp.py => datetime.py} | 4 +- .../misc/pcap/test_header_frame_unit.py | 2 +- ...t_fromtimestamp_windows_range_2099_unit.py | 6 +-- tests/protocols/misc/test_pcapng_unit.py | 2 +- .../test_totimestamp_graalpy_2134_unit.py | 18 ++++----- .../test_utilities_datetime_module_unit.py | 40 +++++++++++++++++++ 12 files changed, 65 insertions(+), 25 deletions(-) rename docs/source/pcapkit/utilities/{timestamp.rst => datetime.rst} (62%) rename pcapkit/utilities/{timestamp.py => datetime.py} (98%) create mode 100644 tests/utilities/test_utilities_datetime_module_unit.py diff --git a/docs/source/pcapkit/utilities/timestamp.rst b/docs/source/pcapkit/utilities/datetime.rst similarity index 62% rename from docs/source/pcapkit/utilities/timestamp.rst rename to docs/source/pcapkit/utilities/datetime.rst index 73b1569fb5..62cb3027d6 100644 --- a/docs/source/pcapkit/utilities/timestamp.rst +++ b/docs/source/pcapkit/utilities/datetime.rst @@ -1,15 +1,15 @@ Timestamp Conversion ==================== -.. module:: pcapkit.utilities.timestamp +.. module:: pcapkit.utilities.datetime -:mod:`pcapkit.utilities.timestamp` converts a UNIX timestamp to an aware +:mod:`pcapkit.utilities.datetime` converts a UNIX timestamp to an aware :class:`~datetime.datetime` without the C library's range (:issue:`2099`), and a :class:`~datetime.datetime` back to a UNIX timestamp without GraalPy's lost second before 1970 (:issue:`2134`). It is shared by the PCAP and PCAP-NG readers and writers and the MH timestamp options, which is why it lives here rather than beside any one of them. -.. autofunction:: pcapkit.utilities.timestamp.fromtimestamp +.. autofunction:: pcapkit.utilities.datetime.fromtimestamp -.. autofunction:: pcapkit.utilities.timestamp.totimestamp +.. autofunction:: pcapkit.utilities.datetime.totimestamp diff --git a/docs/source/pcapkit/utilities/index.rst b/docs/source/pcapkit/utilities/index.rst index 7b3058c061..085ed3c8de 100644 --- a/docs/source/pcapkit/utilities/index.rst +++ b/docs/source/pcapkit/utilities/index.rst @@ -18,7 +18,7 @@ conversion, and wrappers around third-party libraries. It imports nothing from logging config chardet - timestamp + datetime Version Compatibility ===================== diff --git a/pcapkit/protocols/internet/mh.py b/pcapkit/protocols/internet/mh.py index f7e593e192..c2bc6dbb56 100644 --- a/pcapkit/protocols/internet/mh.py +++ b/pcapkit/protocols/internet/mh.py @@ -496,8 +496,8 @@ UpdateNotificationMessage as Schema_UpdateNotificationMessage from pcapkit.protocols.schema.internet.mh import VendorSpecificOption as Schema_VendorSpecificOption from pcapkit.protocols.schema.schema import Schema +from pcapkit.utilities.datetime import fromtimestamp, totimestamp from pcapkit.utilities.exceptions import EnumValueError, ProtocolError, UnsupportedCall -from pcapkit.utilities.timestamp import fromtimestamp, totimestamp from pcapkit.utilities.warnings import ProtocolWarning, RegistryWarning, warn if TYPE_CHECKING: diff --git a/pcapkit/protocols/misc/pcap/frame.py b/pcapkit/protocols/misc/pcap/frame.py index 6381f1ccb2..7279ce58e5 100644 --- a/pcapkit/protocols/misc/pcap/frame.py +++ b/pcapkit/protocols/misc/pcap/frame.py @@ -48,8 +48,8 @@ from pcapkit.protocols.schema.misc.pcap.frame import Frame as Schema_Frame from pcapkit.protocols.schema.misc.pcap.header import RESOLUTION from pcapkit.utilities.compat import localcontext +from pcapkit.utilities.datetime import fromtimestamp, totimestamp from pcapkit.utilities.exceptions import StreamEOFError, UnsupportedCall, stacklevel -from pcapkit.utilities.timestamp import fromtimestamp, totimestamp from pcapkit.utilities.warnings import ProtocolWarning, RegistryWarning, warn if TYPE_CHECKING: diff --git a/pcapkit/protocols/misc/pcapng.py b/pcapkit/protocols/misc/pcapng.py index e2ccd33025..d0cee5873a 100644 --- a/pcapkit/protocols/misc/pcapng.py +++ b/pcapkit/protocols/misc/pcapng.py @@ -210,9 +210,9 @@ from pcapkit.protocols.schema.misc.pcapng import zigbee_key_held from pcapkit.protocols.schema.schema import Schema from pcapkit.utilities.compat import StrEnum, localcontext +from pcapkit.utilities.datetime import fromtimestamp, totimestamp from pcapkit.utilities.exceptions import (BaseError, EnumValueError, FormatError, ProtocolError, StreamEOFError, UnsupportedCall, stacklevel) -from pcapkit.utilities.timestamp import fromtimestamp, totimestamp from pcapkit.utilities.warnings import (AttributeWarning, DeprecatedFormatWarning, ProtocolWarning, RegistryWarning, warn) diff --git a/pcapkit/protocols/schema/internet/mh.py b/pcapkit/protocols/schema/internet/mh.py index 51467a279d..726556ccbe 100644 --- a/pcapkit/protocols/schema/internet/mh.py +++ b/pcapkit/protocols/schema/internet/mh.py @@ -61,8 +61,8 @@ UInt64Field) from pcapkit.corekit.fields.strings import BitField, BytesField, PaddingField, StringField from pcapkit.protocols.schema.schema import EnumSchema, Schema, schema_final +from pcapkit.utilities.datetime import fromtimestamp from pcapkit.utilities.logging import SPHINX_TYPE_CHECKING -from pcapkit.utilities.timestamp import fromtimestamp __all__ = [ 'MH', diff --git a/pcapkit/utilities/timestamp.py b/pcapkit/utilities/datetime.py similarity index 98% rename from pcapkit/utilities/timestamp.py rename to pcapkit/utilities/datetime.py index 256f8b8d55..7f44c93c63 100644 --- a/pcapkit/utilities/timestamp.py +++ b/pcapkit/utilities/datetime.py @@ -2,9 +2,9 @@ """Timestamp Conversion ====================== -.. module:: pcapkit.utilities.timestamp +.. module:: pcapkit.utilities.datetime -:mod:`pcapkit.utilities.timestamp` converts a UNIX timestamp to an aware +:mod:`pcapkit.utilities.datetime` converts a UNIX timestamp to an aware :class:`~datetime.datetime` without the C library's range (:issue:`2099`), and a :class:`~datetime.datetime` back to a UNIX timestamp without GraalPy's lost second before 1970 (:issue:`2134`). diff --git a/tests/protocols/misc/pcap/test_header_frame_unit.py b/tests/protocols/misc/pcap/test_header_frame_unit.py index 84c180e6da..451e5e986d 100644 --- a/tests/protocols/misc/pcap/test_header_frame_unit.py +++ b/tests/protocols/misc/pcap/test_header_frame_unit.py @@ -296,7 +296,7 @@ class Other(Raw): bad_time._fnum = 6 bad_time._file = io.BytesIO(b'') # Faked through frame.py's own binding of - # :func:`pcapkit.utilities.timestamp.fromtimestamp`, which it calls + # :func:`pcapkit.utilities.datetime.fromtimestamp`, which it calls # instead of :meth:`datetime.datetime.fromtimestamp` since #2099. with mock.patch('pcapkit.protocols.misc.pcap.frame.fromtimestamp', side_effect=ValueError), \ mock.patch.object(Frame, '_decode_next_layer', diff --git a/tests/protocols/misc/test_fromtimestamp_windows_range_2099_unit.py b/tests/protocols/misc/test_fromtimestamp_windows_range_2099_unit.py index 711f4af343..8c6e4f7733 100644 --- a/tests/protocols/misc/test_fromtimestamp_windows_range_2099_unit.py +++ b/tests/protocols/misc/test_fromtimestamp_windows_range_2099_unit.py @@ -110,10 +110,10 @@ def test_stdlib_fromtimestamp_fails_outside_the_crt_range(self) -> None: class TestFromTimestamp(unittest.TestCase): - """:func:`pcapkit.utilities.timestamp.fromtimestamp` against the stdlib inside the CRT range.""" + """:func:`pcapkit.utilities.datetime.fromtimestamp` against the stdlib inside the CRT range.""" def test_matches_the_stdlib_where_both_can_answer(self) -> None: - from pcapkit.utilities.timestamp import fromtimestamp + from pcapkit.utilities.datetime import fromtimestamp zones = (UTC, datetime.timezone(datetime.timedelta(hours=-5)), datetime.timezone(datetime.timedelta(seconds=86399))) @@ -131,7 +131,7 @@ def test_matches_the_stdlib_where_both_can_answer(self) -> None: self.assertIs(type(actual), _REAL_DATETIME) def test_errors_are_cpythons(self) -> None: - from pcapkit.utilities.timestamp import fromtimestamp + from pcapkit.utilities.datetime import fromtimestamp cases = ( (float('nan'), ValueError), (float('inf'), OverflowError), (2.0**63, OverflowError), diff --git a/tests/protocols/misc/test_pcapng_unit.py b/tests/protocols/misc/test_pcapng_unit.py index 3c2ab11d9f..ea496cd415 100644 --- a/tests/protocols/misc/test_pcapng_unit.py +++ b/tests/protocols/misc/test_pcapng_unit.py @@ -559,7 +559,7 @@ def now(cls, tz): section=types.SimpleNamespace(byteorder='little')) pcapng._type = BlockType.Enhanced_Packet_Block # Faked through pcapng.py's own binding of - # :func:`pcapkit.utilities.timestamp.fromtimestamp`, which it calls + # :func:`pcapkit.utilities.datetime.fromtimestamp`, which it calls # instead of :meth:`datetime.datetime.fromtimestamp` since #2099. with mock.patch('pcapkit.protocols.misc.pcapng.fromtimestamp', side_effect=ValueError), \ mock.patch('pcapkit.protocols.misc.pcapng.warn') as warn: diff --git a/tests/utilities/test_totimestamp_graalpy_2134_unit.py b/tests/utilities/test_totimestamp_graalpy_2134_unit.py index 92d1e7a5a7..94a7081490 100644 --- a/tests/utilities/test_totimestamp_graalpy_2134_unit.py +++ b/tests/utilities/test_totimestamp_graalpy_2134_unit.py @@ -10,7 +10,7 @@ :class:`GraalPyDateTime` applies GraalPy's rule on any interpreter, and the encoders are run under :func:`graalpy_timestamps`, which turns on -:func:`pcapkit.utilities.timestamp.totimestamp`'s GraalPy branch. +:func:`pcapkit.utilities.datetime.totimestamp`'s GraalPy branch. :mod:`pcapkit` is imported inside each test, after :func:`~tests._support.reimport_once_per_class`. @@ -70,8 +70,8 @@ def graal(instant: 'datetime.datetime') -> 'GraalPyDateTime': @contextlib.contextmanager def graalpy_timestamps() -> 'Iterator[None]': - """Run :func:`~pcapkit.utilities.timestamp.totimestamp` as it runs on GraalPy.""" - with mock.patch('pcapkit.utilities.timestamp._TIMESTAMP_TRUNCATES', True): + """Run :func:`~pcapkit.utilities.datetime.totimestamp` as it runs on GraalPy.""" + with mock.patch('pcapkit.utilities.datetime._TIMESTAMP_TRUNCATES', True): yield @@ -92,7 +92,7 @@ def test_measured_values(self) -> None: class TestToTimestamp(unittest.TestCase): - """:func:`pcapkit.utilities.timestamp.totimestamp` is CPython's :meth:`~datetime.datetime.timestamp`.""" + """:func:`pcapkit.utilities.datetime.totimestamp` is CPython's :meth:`~datetime.datetime.timestamp`.""" def setUp(self) -> None: from tests._support import reimport_once_per_class @@ -100,7 +100,7 @@ def setUp(self) -> None: reimport_once_per_class(self) def test_aware_is_the_seconds_since_the_epoch(self) -> None: - from pcapkit.utilities.timestamp import totimestamp + from pcapkit.utilities.datetime import totimestamp for instant in INSTANTS: with self.subTest(instant=instant): @@ -108,7 +108,7 @@ def test_aware_is_the_seconds_since_the_epoch(self) -> None: @unittest.skipIf(sys.implementation.name == 'graalpy', "GraalPy's own timestamp() is the defect") def test_is_the_stdlib_call_off_graalpy(self) -> None: - from pcapkit.utilities.timestamp import totimestamp + from pcapkit.utilities.datetime import totimestamp naive = datetime.datetime(1950, 6, 1, 12, 0, 0, 500_000) for instant in (*INSTANTS, naive): @@ -116,7 +116,7 @@ def test_is_the_stdlib_call_off_graalpy(self) -> None: self.assertEqual(totimestamp(instant).hex(), instant.timestamp().hex()) def test_graalpys_lost_second_is_restored(self) -> None: - from pcapkit.utilities.timestamp import totimestamp + from pcapkit.utilities.datetime import totimestamp with graalpy_timestamps(): for instant in INSTANTS: @@ -124,14 +124,14 @@ def test_graalpys_lost_second_is_restored(self) -> None: self.assertEqual(totimestamp(graal(instant)), (instant - EPOCH).total_seconds()) def test_naive_is_local_time_everywhere(self) -> None: - from pcapkit.utilities.timestamp import totimestamp + from pcapkit.utilities.datetime import totimestamp naive = datetime.datetime(1950, 6, 1, 12, 0, 0, 500_000) with graalpy_timestamps(): self.assertEqual(totimestamp(naive), naive.timestamp()) def test_an_offsetless_zone_fails_as_on_cpython(self) -> None: - from pcapkit.utilities.timestamp import totimestamp + from pcapkit.utilities.datetime import totimestamp class NoOffset(datetime.tzinfo): def utcoffset(self, dt: 'object') -> None: diff --git a/tests/utilities/test_utilities_datetime_module_unit.py b/tests/utilities/test_utilities_datetime_module_unit.py new file mode 100644 index 0000000000..1f5e044e40 --- /dev/null +++ b/tests/utilities/test_utilities_datetime_module_unit.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +"""#2134: the timestamp conversions live in :mod:`pcapkit.utilities.datetime`. + +The owner ruled on #2136 that the module reuses the stdlib name, as +:mod:`pcapkit.utilities.logging` and :mod:`pcapkit.utilities.warnings` do. The +old ``pcapkit.utilities.timestamp`` path was added by #2111 and is in no +release, so it is deleted rather than re-exported (#1715). + +""" + +import datetime +import importlib +import sys +import unittest + +#: Test tier (#1958): a module-layout check, independent of the interpreter. +__tier__ = 'LATEST' + + +class TestDatetimeModule(unittest.TestCase): + """:mod:`pcapkit.utilities.datetime` holds the conversions, and the old path is gone.""" + + def test_new_module_exposes_the_conversions(self) -> None: + module = importlib.import_module('pcapkit.utilities.datetime') + self.assertEqual(sorted(module.__all__), ['fromtimestamp', 'totimestamp']) + self.assertTrue(callable(module.fromtimestamp)) + self.assertTrue(callable(module.totimestamp)) + + def test_new_module_imports_the_stdlib_datetime(self) -> None: + module = importlib.import_module('pcapkit.utilities.datetime') + self.assertIs(module.datetime, datetime) + self.assertIs(sys.modules['datetime'], datetime) + + def test_old_module_path_is_gone(self) -> None: + with self.assertRaises(ModuleNotFoundError): + importlib.import_module('pcapkit.utilities.timestamp') + + +if __name__ == '__main__': + unittest.main()