From c33a80295b1d79dfb25b9dbf320ed783f15fd310 Mon Sep 17 00:00:00 2001 From: yuxin00j Date: Fri, 2 Oct 2026 05:28:46 +0000 Subject: [PATCH 1/7] fix(crc32c): accept any contiguous bytes-like object in value() and extend() `google_crc32c.value()`, `extend()` and `Checksum.update()` parsed their chunk with `PyArg_ParseTuple("y#")`, which only accepts objects without a `releasebuffer` slot, so `memoryview`, `bytearray` and `array.array` were rejected with "argument must be read-only bytes-like object". Callers that hold a view into a larger buffer had to copy it into `bytes` first just to checksum it. Parse the chunk as a `Py_buffer` ("y*") instead, which accepts any C-contiguous object supporting the buffer protocol, and release it once the checksum is computed. The GIL is still released for chunks of at least 1 MiB, now for any read-only buffer (`bytes`, or a read-only view) rather than only `bytes`; mutable buffers such as `bytearray` keep the GIL as before. Non-contiguous views raise `BufferError`, as with `hashlib`. --- .../google-crc32c/src/google_crc32c/_crc32c.c | 61 ++++++++++--------- .../google-crc32c/src/google_crc32c/cext.py | 8 ++- packages/google-crc32c/tests/test___init__.py | 49 +++++++++++++++ 3 files changed, 87 insertions(+), 31 deletions(-) diff --git a/packages/google-crc32c/src/google_crc32c/_crc32c.c b/packages/google-crc32c/src/google_crc32c/_crc32c.c index b20b94d5542d..7667a1c1dab3 100644 --- a/packages/google-crc32c/src/google_crc32c/_crc32c.c +++ b/packages/google-crc32c/src/google_crc32c/_crc32c.c @@ -6,35 +6,48 @@ static const Py_ssize_t gil_threshold = 1024 * 1024; static int -_should_release_gil(Py_ssize_t length, PyObject *chunk_obj) +_should_release_gil(const Py_buffer *chunk) { - /* Checks if the chunk is immutable (bytes) to prevent concurrent modification, - * and large enough to benefit from releasing the GIL. */ - return (length >= gil_threshold && PyBytes_Check(chunk_obj)); + /* Checks if the chunk is read-only (bytes, or a read-only view such as a + * memoryview over bytes) to prevent concurrent modification, and large + * enough to benefit from releasing the GIL. The buffer stays exported + * while the checksum is computed, so it cannot be resized or freed. */ + return (chunk->len >= gil_threshold && chunk->readonly); } -static PyObject * -_crc32c_extend(PyObject *self, PyObject *args) +static uint32_t +_extend_buffer(uint32_t crc, const Py_buffer *chunk) { - unsigned long crc_input; - uint32_t crc; - const char *chunk; - Py_ssize_t length; PyThreadState *save = NULL; - if (!PyArg_ParseTuple(args, "ky#", &crc_input, &chunk, &length)) - return NULL; - - if (_should_release_gil(length, PyTuple_GET_ITEM(args, 1))) { + if (_should_release_gil(chunk)) { save = PyEval_SaveThread(); } - crc = crc32c_extend((uint32_t)crc_input, (const uint8_t*)chunk, length); + crc = crc32c_extend(crc, (const uint8_t*)chunk->buf, (size_t)chunk->len); if (save) { PyEval_RestoreThread(save); } + return crc; +} + +static PyObject * +_crc32c_extend(PyObject *self, PyObject *args) +{ + unsigned long crc_input; + uint32_t crc; + Py_buffer chunk; + + /* "y*" accepts any C-contiguous object supporting the buffer protocol + * (bytes, bytearray, memoryview, array.array, mmap, ...). */ + if (!PyArg_ParseTuple(args, "ky*", &crc_input, &chunk)) + return NULL; + + crc = _extend_buffer((uint32_t)crc_input, &chunk); + PyBuffer_Release(&chunk); + return PyLong_FromUnsignedLong(crc); } @@ -43,22 +56,14 @@ static PyObject * _crc32c_value(PyObject *self, PyObject *args) { uint32_t crc; - const char *chunk; - Py_ssize_t length; - PyThreadState *save = NULL; + Py_buffer chunk; - if (!PyArg_ParseTuple(args, "y#", &chunk, &length)) + if (!PyArg_ParseTuple(args, "y*", &chunk)) return NULL; - if (_should_release_gil(length, PyTuple_GET_ITEM(args, 0))) { - save = PyEval_SaveThread(); - } - - crc = crc32c_value((const uint8_t*)chunk, length); - - if (save) { - PyEval_RestoreThread(save); - } + /* crc32c_value(data, n) is crc32c_extend(0, data, n). */ + crc = _extend_buffer(0, &chunk); + PyBuffer_Release(&chunk); return PyLong_FromUnsignedLong(crc); } diff --git a/packages/google-crc32c/src/google_crc32c/cext.py b/packages/google-crc32c/src/google_crc32c/cext.py index ec22abec5c1c..fde116c84cbc 100644 --- a/packages/google-crc32c/src/google_crc32c/cext.py +++ b/packages/google-crc32c/src/google_crc32c/cext.py @@ -26,8 +26,10 @@ class Checksum(CommonChecksum): """Hashlib-alike helper for CRC32C operations. Args: - initial_value (Optional[bytes]): the initial chunk of data from - which the CRC32C checksum is computed. Defaults to b''. + initial_value (Optional[bytes-like]): the initial chunk of data from + which the CRC32C checksum is computed. Any C-contiguous object + supporting the buffer protocol (``bytes``, ``bytearray``, + ``memoryview``, ...) is accepted. Defaults to b''. """ __slots__ = ("_crc",) @@ -39,7 +41,7 @@ def update(self, chunk): """Update the checksum with a new chunk of data. Args: - chunk (Optional[bytes]): a chunk of data used to extend + chunk (Optional[bytes-like]): a chunk of data used to extend the CRC32C checksum. """ self._crc = extend(self._crc, chunk) diff --git a/packages/google-crc32c/tests/test___init__.py b/packages/google-crc32c/tests/test___init__.py index 8d108ac91657..9f2fcf8bac9c 100644 --- a/packages/google-crc32c/tests/test___init__.py +++ b/packages/google-crc32c/tests/test___init__.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import array import functools import itertools from unittest import mock @@ -194,6 +195,37 @@ def test_value(chunk, expected): assert google_crc32c.value(bytes(chunk)) == expected +_BYTES_LIKE = [ + pytest.param(bytes, id="bytes"), + pytest.param(bytearray, id="bytearray"), + pytest.param(memoryview, id="memoryview"), + pytest.param(lambda data: array.array("B", data), id="array"), +] + + +@pytest.mark.parametrize("factory", _BYTES_LIKE) +def test_value_w_bytes_like(_crc32c, factory): + assert _crc32c.value(factory(ISCSI_BYTES)) == ISCSI_CRC + + +@pytest.mark.parametrize("factory", _BYTES_LIKE) +def test_extend_w_bytes_like(_crc32c, factory): + chunks = [factory(bytes(chunk)) for chunk in iscsi_chunks(7)] + assert functools.reduce(_crc32c.extend, chunks, 0) == ISCSI_CRC + + +def test_value_w_large_readonly_memoryview(_cext): + # Larger than the 1 MiB threshold above which the C extension releases + # the GIL while hashing read-only buffers. + data = ISCSI_BYTES * (2 * 1024 * 1024 // ISCSI_LENGTH + 1) + assert _cext.value(memoryview(data)) == _cext.value(data) + + +def test_value_w_non_contiguous_memoryview(_cext): + with pytest.raises(BufferError): + _cext.value(memoryview(ISCSI_BYTES)[::2]) + + def pytest_generate_tests(metafunc): if "_crc32c" in metafunc.fixturenames: metafunc.parametrize("_crc32c", ["python", "cext"], indirect=True) @@ -216,6 +248,16 @@ def _crc32c(request): raise ValueError("invalid internal test config") +@pytest.fixture +def _cext(): + try: + from google_crc32c import cext + except ImportError: # pragma: NO COVER + pytest.skip("C extension not compiled") # pragma: NO COVER + + return cext # pragma: NO COVER + + class TestChecksum(object): @staticmethod def test_ctor_defaults(_crc32c): @@ -246,6 +288,13 @@ def test_update(_crc32c): helper.update(chunk) assert helper._crc == google_crc32c.value(chunk) + @staticmethod + @pytest.mark.parametrize("factory", _BYTES_LIKE) + def test_update_w_bytes_like(_crc32c, factory): + helper = _crc32c.Checksum() + helper.update(factory(ISCSI_BYTES)) + assert helper._crc == ISCSI_CRC + @staticmethod def test_update_w_multiple_chunks(_crc32c): helper = google_crc32c.Checksum() From fdc45c5d4a1473a7af5d168dca001767e1e250d2 Mon Sep 17 00:00:00 2001 From: yuxin00j Date: Fri, 2 Oct 2026 10:21:33 +0000 Subject: [PATCH 2/7] test(crc32c): cover sliced and writable views in the bytes-like tests Add a non-zero-offset memoryview slice and a writable bytearray-backed view to the bytes-like parametrization so the extension is shown to hash the exported slice rather than the exporter's whole buffer, test the Checksum constructor with every bytes-like type, check that extend() rejects non-contiguous views like value() does, and align the _checksum.py / python.py docstrings with cext.py. --- .../src/google_crc32c/_checksum.py | 4 ++-- .../google-crc32c/src/google_crc32c/python.py | 4 ++-- packages/google-crc32c/tests/test___init__.py | 18 ++++++++++++++++++ 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/packages/google-crc32c/src/google_crc32c/_checksum.py b/packages/google-crc32c/src/google_crc32c/_checksum.py index 0aaf0854ce34..458ff24c4672 100644 --- a/packages/google-crc32c/src/google_crc32c/_checksum.py +++ b/packages/google-crc32c/src/google_crc32c/_checksum.py @@ -21,7 +21,7 @@ class CommonChecksum(object): This class should not be used directly and requires an update implementation. Args: - initial_value (Optional[bytes]): the initial chunk of data from + initial_value (Optional[bytes-like]): the initial chunk of data from which the CRC32C checksum is computed. Defaults to b''. """ @@ -36,7 +36,7 @@ def update(self, data): """Update the checksum with a new chunk of data. Args: - chunk (Optional[bytes]): a chunk of data used to extend + chunk (Optional[bytes-like]): a chunk of data used to extend the CRC32C checksum. """ raise NotImplementedError diff --git a/packages/google-crc32c/src/google_crc32c/python.py b/packages/google-crc32c/src/google_crc32c/python.py index 04cedabffe26..ce1de4547271 100644 --- a/packages/google-crc32c/src/google_crc32c/python.py +++ b/packages/google-crc32c/src/google_crc32c/python.py @@ -54,7 +54,7 @@ class Checksum(CommonChecksum): """Hashlib-alike helper for CRC32C operations. Args: - initial_value (Optional[bytes]): the initial chunk of data from + initial_value (Optional[bytes-like]): the initial chunk of data from which the CRC32C checksum is computed. Defaults to b''. """ @@ -67,7 +67,7 @@ def update(self, data): """Update the checksum with a new chunk of data. Args: - chunk (Optional[bytes]): a chunk of data used to extend + chunk (Optional[bytes-like]): a chunk of data used to extend the CRC32C checksum. """ if not isinstance(data, array.array) or data.itemsize != 1: diff --git a/packages/google-crc32c/tests/test___init__.py b/packages/google-crc32c/tests/test___init__.py index 9f2fcf8bac9c..f0c205b9bac9 100644 --- a/packages/google-crc32c/tests/test___init__.py +++ b/packages/google-crc32c/tests/test___init__.py @@ -199,6 +199,13 @@ def test_value(chunk, expected): pytest.param(bytes, id="bytes"), pytest.param(bytearray, id="bytearray"), pytest.param(memoryview, id="memoryview"), + pytest.param( + # A view into the middle of a larger buffer: the slice, not the + # exporter's whole buffer, must be hashed. + lambda data: memoryview(b"\x00" * 3 + data + b"\xff" * 5)[3:-5], + id="memoryview-slice", + ), + pytest.param(lambda data: memoryview(bytearray(data)), id="memoryview-writable"), pytest.param(lambda data: array.array("B", data), id="array"), ] @@ -226,6 +233,11 @@ def test_value_w_non_contiguous_memoryview(_cext): _cext.value(memoryview(ISCSI_BYTES)[::2]) +def test_extend_w_non_contiguous_memoryview(_cext): + with pytest.raises(BufferError): + _cext.extend(0, memoryview(ISCSI_BYTES)[::2]) + + def pytest_generate_tests(metafunc): if "_crc32c" in metafunc.fixturenames: metafunc.parametrize("_crc32c", ["python", "cext"], indirect=True) @@ -270,6 +282,12 @@ def test_ctor_explicit(_crc32c): helper = google_crc32c.Checksum(chunk) assert helper._crc == google_crc32c.value(chunk) + @staticmethod + @pytest.mark.parametrize("factory", _BYTES_LIKE) + def test_ctor_w_bytes_like(_crc32c, factory): + helper = _crc32c.Checksum(factory(ISCSI_BYTES)) + assert helper._crc == ISCSI_CRC + @staticmethod def test_update_array(): import array From ceccba0a44efca0bb6abcc05722fa18265070af3 Mon Sep 17 00:00:00 2001 From: yuxin00j Date: Sat, 3 Oct 2026 14:29:28 +0000 Subject: [PATCH 3/7] fix(crc32c): hash raw buffer bytes in the pure-Python implementation Checksum.update() converted its input with array.array("B", data), which iterates items rather than bytes. Multi-byte buffers such as array("H") or memoryview.cast("I") therefore hashed differently from the C extension (or raised OverflowError), and mmap, 'c'-format and N-D buffers raised, although the C extension hashes their raw bytes. Hash memoryview(data).cast("B") for buffer objects, raise BufferError for views that are not C-contiguous as the C extension does, and keep array.array("B", data) only for iterables of ints. --- .../google-crc32c/src/google_crc32c/python.py | 36 ++++++---- packages/google-crc32c/tests/test___init__.py | 71 +++++++++++++++++-- 2 files changed, 89 insertions(+), 18 deletions(-) diff --git a/packages/google-crc32c/src/google_crc32c/python.py b/packages/google-crc32c/src/google_crc32c/python.py index ce1de4547271..e412ccbc83f4 100644 --- a/packages/google-crc32c/src/google_crc32c/python.py +++ b/packages/google-crc32c/src/google_crc32c/python.py @@ -22,8 +22,9 @@ def extend(crc, chunk): Args crc (int): An existing CRC check sum. - chunk (Union[bytes, List[int], Tuple[int]]): A new chunk of data. - Intended to be a byte string or similar. + chunk (Union[bytes-like, Iterable[int]]): A new chunk of data. + Bytes-like objects are hashed as raw bytes; an iterable of + ints in ``range(256)`` is also accepted. Returns int: New CRC checksum computed by extending existing CRC @@ -39,8 +40,9 @@ def value(chunk): """Compute a CRC checksum for a chunk of data. Args - chunk (Union[bytes, List[int], Tuple[int]]): A new chunk of data. - Intended to be a byte string or similar. + chunk (Union[bytes-like, Iterable[int]]): A new chunk of data. + Bytes-like objects are hashed as raw bytes; an iterable of + ints in ``range(256)`` is also accepted. Returns int: New CRC checksum computed for ``chunk``. @@ -70,15 +72,23 @@ def update(self, data): chunk (Optional[bytes-like]): a chunk of data used to extend the CRC32C checksum. """ - if not isinstance(data, array.array) or data.itemsize != 1: - buffer = array.array("B", data) - else: - buffer = data - self._crc = self._crc ^ 0xFFFFFFFF - for b in buffer: - table_poly = _TABLE[(b ^ self._crc) & 0xFF] - self._crc = table_poly ^ ((self._crc >> 8) & 0xFFFFFFFF) - self._crc = self._crc ^ 0xFFFFFFFF + try: + view = memoryview(data) + except TypeError: + # Not a buffer: an iterable of ints in range(256). + view = memoryview(array.array("B", data)) + with view: + # Hash the raw bytes, like the C extension. + if not view.c_contiguous: + raise BufferError("memoryview: underlying buffer is not C-contiguous") + if not view.nbytes: + return # Nothing to hash; cast() rejects empty N-D views. + with view.cast("B") as buffer: + self._crc = self._crc ^ 0xFFFFFFFF + for b in buffer: + table_poly = _TABLE[(b ^ self._crc) & 0xFF] + self._crc = table_poly ^ ((self._crc >> 8) & 0xFFFFFFFF) + self._crc = self._crc ^ 0xFFFFFFFF # fmt:off diff --git a/packages/google-crc32c/tests/test___init__.py b/packages/google-crc32c/tests/test___init__.py index f0c205b9bac9..707e607c8392 100644 --- a/packages/google-crc32c/tests/test___init__.py +++ b/packages/google-crc32c/tests/test___init__.py @@ -12,8 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. import array +import ctypes import functools import itertools +import mmap from unittest import mock import pytest # type: ignore @@ -207,6 +209,19 @@ def test_value(chunk, expected): ), pytest.param(lambda data: memoryview(bytearray(data)), id="memoryview-writable"), pytest.param(lambda data: array.array("B", data), id="array"), + # Multi-byte, 'c' and N-D exporters: their raw bytes are hashed, not items. + pytest.param(lambda data: array.array("H", data), id="array-H"), + pytest.param(lambda data: array.array("I", data), id="array-I"), + pytest.param(lambda data: memoryview(data).cast("I"), id="memoryview-cast-I"), + pytest.param(lambda data: memoryview(data).cast("c"), id="memoryview-cast-c"), + pytest.param( + lambda data: memoryview(data).cast("B", [2, len(data) // 2]), + id="memoryview-2d", + ), + pytest.param( + lambda data: (ctypes.c_uint16 * (len(data) // 2)).from_buffer_copy(data), + id="ctypes-uint16", + ), ] @@ -217,10 +232,43 @@ def test_value_w_bytes_like(_crc32c, factory): @pytest.mark.parametrize("factory", _BYTES_LIKE) def test_extend_w_bytes_like(_crc32c, factory): - chunks = [factory(bytes(chunk)) for chunk in iscsi_chunks(7)] + # 8-byte chunks hold a whole number of items for every factory. + chunks = [factory(bytes(chunk)) for chunk in iscsi_chunks(8)] assert functools.reduce(_crc32c.extend, chunks, 0) == ISCSI_CRC +def test_value_w_mmap(_crc32c): + # Closing the map raises BufferError if an export was not released. + with mmap.mmap(-1, ISCSI_LENGTH) as buffer: + buffer.write(ISCSI_BYTES) + assert _crc32c.value(buffer) == ISCSI_CRC + assert _crc32c.extend(0, buffer) == ISCSI_CRC + + +@pytest.mark.parametrize( + "empty", + [ + pytest.param(array.array("I"), id="array-I"), + pytest.param((ctypes.c_uint8 * 0 * 3)(), id="ctypes-2d"), + ], +) +def test_value_w_empty_bytes_like(_crc32c, empty): + assert _crc32c.value(empty) == EMPTY_CRC + assert _crc32c.extend(123, empty) == 123 + + +def test_value_w_iterable_of_ints(): + from google_crc32c import python + + assert python.value(ISCSI_SCSI_READ_10_COMMAND_PDU) == ISCSI_CRC + assert python.value(iter(ISCSI_SCSI_READ_10_COMMAND_PDU)) == ISCSI_CRC + + +def test_value_w_str(_crc32c): + with pytest.raises(TypeError): + _crc32c.value("DEADBEEF") + + def test_value_w_large_readonly_memoryview(_cext): # Larger than the 1 MiB threshold above which the C extension releases # the GIL while hashing read-only buffers. @@ -228,14 +276,27 @@ def test_value_w_large_readonly_memoryview(_cext): assert _cext.value(memoryview(data)) == _cext.value(data) -def test_value_w_non_contiguous_memoryview(_cext): +def _fortran_order(data): + np = pytest.importorskip("numpy") + return memoryview(np.frombuffer(data, np.uint8).reshape(2, -1, order="F")) + + +_NON_C_CONTIGUOUS = [ + pytest.param(lambda data: memoryview(data)[::2], id="strided"), + pytest.param(_fortran_order, id="fortran-order"), +] + + +@pytest.mark.parametrize("factory", _NON_C_CONTIGUOUS) +def test_value_w_non_contiguous_memoryview(_crc32c, factory): with pytest.raises(BufferError): - _cext.value(memoryview(ISCSI_BYTES)[::2]) + _crc32c.value(factory(ISCSI_BYTES)) -def test_extend_w_non_contiguous_memoryview(_cext): +@pytest.mark.parametrize("factory", _NON_C_CONTIGUOUS) +def test_extend_w_non_contiguous_memoryview(_crc32c, factory): with pytest.raises(BufferError): - _cext.extend(0, memoryview(ISCSI_BYTES)[::2]) + _crc32c.extend(0, factory(ISCSI_BYTES)) def pytest_generate_tests(metafunc): From 91d0ee438f7c7e6ad1951e142d3cf9fcb0d155bb Mon Sep 17 00:00:00 2001 From: yuxin00j Date: Sat, 3 Oct 2026 14:42:17 +0000 Subject: [PATCH 4/7] fix(crc32c): release the GIL only for chunks backed by bytes The extension released the GIL for any read-only chunk of at least 1 MiB, but a read-only buffer is not necessarily immutable: the owner of memoryview(bytearray).toreadonly(), or of a numpy array with writeable=False, can still write to it from another thread, so the checksum could cover a mix of old and new contents. Release the GIL only when the memory belongs to a bytes object, passed directly or through a memoryview (including slices) of it, so slices of large bytes payloads are still hashed without the GIL. All other buffers are hashed with the GIL held, as bytearray already was. --- .../google-crc32c/src/google_crc32c/_crc32c.c | 21 ++++-- packages/google-crc32c/tests/test___init__.py | 66 +++++++++++++++++-- 2 files changed, 77 insertions(+), 10 deletions(-) diff --git a/packages/google-crc32c/src/google_crc32c/_crc32c.c b/packages/google-crc32c/src/google_crc32c/_crc32c.c index 7667a1c1dab3..c5d5e4c7a1cd 100644 --- a/packages/google-crc32c/src/google_crc32c/_crc32c.c +++ b/packages/google-crc32c/src/google_crc32c/_crc32c.c @@ -8,11 +8,22 @@ static const Py_ssize_t gil_threshold = 1024 * 1024; static int _should_release_gil(const Py_buffer *chunk) { - /* Checks if the chunk is read-only (bytes, or a read-only view such as a - * memoryview over bytes) to prevent concurrent modification, and large - * enough to benefit from releasing the GIL. The buffer stays exported - * while the checksum is computed, so it cannot be resized or freed. */ - return (chunk->len >= gil_threshold && chunk->readonly); + /* Release the GIL only for chunks large enough to benefit and whose memory + * is immutable: a bytes object, or a memoryview (incl. slices) of one. + * Read-only views of mutable memory (memoryview(bytearray).toreadonly(), + * numpy arrays with writeable=False) can still be written through their + * owner, so the GIL stays held for them as for any other exporter. The + * buffer stays exported while the checksum is computed, so it cannot be + * resized or freed. */ + PyObject *owner = chunk->obj; + + if (chunk->len < gil_threshold || owner == NULL) { + return 0; + } + if (PyMemoryView_Check(owner)) { + owner = PyMemoryView_GET_BASE(owner); + } + return owner != NULL && PyBytes_Check(owner); } static uint32_t diff --git a/packages/google-crc32c/tests/test___init__.py b/packages/google-crc32c/tests/test___init__.py index 707e607c8392..35f8adca837f 100644 --- a/packages/google-crc32c/tests/test___init__.py +++ b/packages/google-crc32c/tests/test___init__.py @@ -16,6 +16,8 @@ import functools import itertools import mmap +import sys +import threading from unittest import mock import pytest # type: ignore @@ -208,6 +210,10 @@ def test_value(chunk, expected): id="memoryview-slice", ), pytest.param(lambda data: memoryview(bytearray(data)), id="memoryview-writable"), + pytest.param( + lambda data: memoryview(bytearray(data)).toreadonly(), + id="memoryview-readonly", + ), pytest.param(lambda data: array.array("B", data), id="array"), # Multi-byte, 'c' and N-D exporters: their raw bytes are hashed, not items. pytest.param(lambda data: array.array("H", data), id="array-H"), @@ -269,11 +275,61 @@ def test_value_w_str(_crc32c): _crc32c.value("DEADBEEF") -def test_value_w_large_readonly_memoryview(_cext): - # Larger than the 1 MiB threshold above which the C extension releases - # the GIL while hashing read-only buffers. - data = ISCSI_BYTES * (2 * 1024 * 1024 // ISCSI_LENGTH + 1) - assert _cext.value(memoryview(data)) == _cext.value(data) +# Larger than the 1 MiB threshold above which the C extension releases the +# GIL for chunks backed by bytes. CRC from the pure-Python implementation. +LARGE = ISCSI_BYTES * (2 * 1024 * 1024 // ISCSI_LENGTH + 1) +LARGE_CRC = 0x29CF5F4B + + +@pytest.mark.parametrize("factory", _BYTES_LIKE) +def test_value_w_large_bytes_like(_cext, factory): + assert _cext.value(factory(LARGE)) == LARGE_CRC + assert _cext.extend(0, factory(LARGE)) == LARGE_CRC + + +def test_value_w_large_readonly_view_of_bytearray_releases_buffer(_cext): + buffer = bytearray(LARGE) + view = memoryview(buffer).toreadonly() + assert _cext.value(view) == LARGE_CRC + assert _cext.extend(0, view) == LARGE_CRC + view.release() # BufferError if the extension kept its export. + buffer.append(0) + + +@pytest.mark.parametrize( + "factory", + [ + pytest.param(lambda buffer: buffer, id="bytearray"), + pytest.param(lambda buffer: memoryview(buffer).toreadonly(), id="readonly"), + ], +) +def test_value_holds_gil_for_mutable_memory(_cext, factory): + # Read-only views of mutable memory can still be written through their + # owner, so the GIL must stay held while they are hashed. With a 10 s + # switch interval this thread keeps the GIL until value() releases it or + # join() blocks, so the probe sees in_call=True only if value() did. + data = factory(bytearray(64 * 1024 * 1024)) + go = threading.Event() + in_call = False + seen = [] + + def probe(): + go.wait() + seen.append(in_call) + + interval = sys.getswitchinterval() + sys.setswitchinterval(10) + try: + thread = threading.Thread(target=probe, daemon=True) + thread.start() + go.set() + in_call = True + _cext.value(data) + in_call = False + thread.join() + finally: + sys.setswitchinterval(interval) + assert seen == [False] def _fortran_order(data): From 9c01c94c3924561f00adf6260da840c6afbf8116 Mon Sep 17 00:00:00 2001 From: yuxin00j Date: Sun, 11 Oct 2026 10:16:45 +0000 Subject: [PATCH 5/7] test(crc32c): assert the GIL is released for bytes-backed input The tests proved only the negative half of the GIL rule: value() keeps the GIL for a bytearray and for read-only views of one. The property the storage SDK's off-loop checksum depends on, that hashing a bytes object or a memoryview into one of 1 MiB or more runs with the GIL released, had no test. The same probe thread now checks that it runs while value() is in progress for bytes, a memoryview of bytes and a slice of that view. The numpy-only line of the Fortran-order helper is excluded from coverage; the unit session does not install numpy, so it skips before that line and the cover session would otherwise stop at 99%. Test plan: with the extension built from this worktree (google_crc32c.implementation == "c") and with CRC32C_PURE_PYTHON=1: python -m pytest packages/google-crc32c/tests --cov=google_crc32c --cov=tests \ --cov-config=.coveragerc --cov-report=term-missing 180 pass, 4 skipped (numpy), coverage 100%; the three new cases pass on three consecutive runs. ruff check/format (0.14.14) and flake8 are clean. Authored with an AI assistant. --- packages/google-crc32c/tests/test___init__.py | 41 ++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/packages/google-crc32c/tests/test___init__.py b/packages/google-crc32c/tests/test___init__.py index 35f8adca837f..1c7af52b9d4f 100644 --- a/packages/google-crc32c/tests/test___init__.py +++ b/packages/google-crc32c/tests/test___init__.py @@ -332,9 +332,48 @@ def probe(): assert seen == [False] +@pytest.mark.parametrize( + "factory", + [ + pytest.param(lambda data: data, id="bytes"), + pytest.param(lambda data: memoryview(data), id="memoryview"), + pytest.param(lambda data: memoryview(data)[1:-1], id="slice"), + ], +) +def test_value_releases_gil_for_immutable_memory(_cext, factory): + # The counterpart of the test above: nothing can write to a bytes object, + # so hashing it (or a view into it) runs with the GIL released and the + # probe thread gets to run while value() is in progress. + data = factory(bytes(64 * 1024 * 1024)) + go = threading.Event() + in_call = False + seen = [] + + def probe(): + go.wait() + seen.append(in_call) + + interval = sys.getswitchinterval() + sys.setswitchinterval(10) + try: + thread = threading.Thread(target=probe, daemon=True) + thread.start() + go.set() + in_call = True + _cext.value(data) + in_call = False + thread.join() + finally: + sys.setswitchinterval(interval) + assert seen == [True] + + def _fortran_order(data): np = pytest.importorskip("numpy") - return memoryview(np.frombuffer(data, np.uint8).reshape(2, -1, order="F")) + # Only reached when numpy is installed, which the unit session does not do. + return memoryview( # pragma: NO COVER + np.frombuffer(data, np.uint8).reshape(2, -1, order="F") + ) _NON_C_CONTIGUOUS = [ From d385ddc8d2a6c3a28b23218b443fecb88997d04e Mon Sep 17 00:00:00 2001 From: yuxin00j Date: Tue, 20 Oct 2026 09:03:55 +0000 Subject: [PATCH 6/7] feat(crc32c): export GIL_RELEASE_THRESHOLD and address review comments --- .../src/google_crc32c/__init__.py | 9 ++++++++- .../google-crc32c/src/google_crc32c/_crc32c.c | 19 +++++++++++-------- .../google-crc32c/src/google_crc32c/cext.py | 4 ++++ .../google-crc32c/src/google_crc32c/python.py | 4 +++- packages/google-crc32c/tests/test___init__.py | 5 +++++ 5 files changed, 31 insertions(+), 10 deletions(-) diff --git a/packages/google-crc32c/src/google_crc32c/__init__.py b/packages/google-crc32c/src/google_crc32c/__init__.py index 0acf5476ba2f..f2452c7a3f6b 100644 --- a/packages/google-crc32c/src/google_crc32c/__init__.py +++ b/packages/google-crc32c/src/google_crc32c/__init__.py @@ -34,7 +34,14 @@ extend = impl.extend value = impl.value +GIL_RELEASE_THRESHOLD = impl.GIL_RELEASE_THRESHOLD Checksum = impl.Checksum -__all__ = ["extend", "value", "Checksum", "implementation"] +__all__ = [ + "extend", + "value", + "Checksum", + "GIL_RELEASE_THRESHOLD", + "implementation", +] diff --git a/packages/google-crc32c/src/google_crc32c/_crc32c.c b/packages/google-crc32c/src/google_crc32c/_crc32c.c index c5d5e4c7a1cd..b6eee928651f 100644 --- a/packages/google-crc32c/src/google_crc32c/_crc32c.c +++ b/packages/google-crc32c/src/google_crc32c/_crc32c.c @@ -8,13 +8,8 @@ static const Py_ssize_t gil_threshold = 1024 * 1024; static int _should_release_gil(const Py_buffer *chunk) { - /* Release the GIL only for chunks large enough to benefit and whose memory - * is immutable: a bytes object, or a memoryview (incl. slices) of one. - * Read-only views of mutable memory (memoryview(bytearray).toreadonly(), - * numpy arrays with writeable=False) can still be written through their - * owner, so the GIL stays held for them as for any other exporter. The - * buffer stays exported while the checksum is computed, so it cannot be - * resized or freed. */ + /* Release the GIL only for chunks of at least gil_threshold bytes backed + * by an immutable bytes object (a bytes instance or a memoryview of one). */ PyObject *owner = chunk->obj; if (chunk->len < gil_threshold || owner == NULL) { @@ -102,5 +97,13 @@ static struct PyModuleDef crc32cmodule = { PyMODINIT_FUNC PyInit__crc32c(void) { - return PyModule_Create(&crc32cmodule); + PyObject *m = PyModule_Create(&crc32cmodule); + if (m == NULL) { + return NULL; + } + if (PyModule_AddIntConstant(m, "GIL_RELEASE_THRESHOLD", (long)gil_threshold) < 0) { + Py_DECREF(m); + return NULL; + } + return m; } diff --git a/packages/google-crc32c/src/google_crc32c/cext.py b/packages/google-crc32c/src/google_crc32c/cext.py index fde116c84cbc..95ebe49bf788 100644 --- a/packages/google-crc32c/src/google_crc32c/cext.py +++ b/packages/google-crc32c/src/google_crc32c/cext.py @@ -17,6 +17,7 @@ import google_crc32c.__config__ # noqa: F401 from google_crc32c._checksum import CommonChecksum from google_crc32c._crc32c import ( # type: ignore + GIL_RELEASE_THRESHOLD, extend, value, ) @@ -45,3 +46,6 @@ def update(self, chunk): the CRC32C checksum. """ self._crc = extend(self._crc, chunk) + + +__all__ = ["extend", "value", "Checksum", "GIL_RELEASE_THRESHOLD"] diff --git a/packages/google-crc32c/src/google_crc32c/python.py b/packages/google-crc32c/src/google_crc32c/python.py index e412ccbc83f4..c90c8a01d219 100644 --- a/packages/google-crc32c/src/google_crc32c/python.py +++ b/packages/google-crc32c/src/google_crc32c/python.py @@ -16,6 +16,8 @@ from google_crc32c._checksum import CommonChecksum +GIL_RELEASE_THRESHOLD = 1024 * 1024 + def extend(crc, chunk): """Update an existing CRC checksum with new chunk of data. @@ -69,7 +71,7 @@ def update(self, data): """Update the checksum with a new chunk of data. Args: - chunk (Optional[bytes-like]): a chunk of data used to extend + data (Optional[bytes-like]): a chunk of data used to extend the CRC32C checksum. """ try: diff --git a/packages/google-crc32c/tests/test___init__.py b/packages/google-crc32c/tests/test___init__.py index 1c7af52b9d4f..e31ec37c2640 100644 --- a/packages/google-crc32c/tests/test___init__.py +++ b/packages/google-crc32c/tests/test___init__.py @@ -275,6 +275,11 @@ def test_value_w_str(_crc32c): _crc32c.value("DEADBEEF") +def test_gil_release_threshold(_crc32c): + assert _crc32c.GIL_RELEASE_THRESHOLD == 1024 * 1024 + assert google_crc32c.GIL_RELEASE_THRESHOLD == 1024 * 1024 + + # Larger than the 1 MiB threshold above which the C extension releases the # GIL for chunks backed by bytes. CRC from the pure-Python implementation. LARGE = ISCSI_BYTES * (2 * 1024 * 1024 // ISCSI_LENGTH + 1) From 26d4369c5ea9ad49a42fcde6f01ef11b371f0e83 Mon Sep 17 00:00:00 2001 From: yuxin00j Date: Thu, 29 Oct 2026 07:01:34 +0000 Subject: [PATCH 7/7] fix(crc32c): call Checksum.update unconditionally and stabilize GIL test --- .../google-crc32c/src/google_crc32c/python.py | 3 +-- packages/google-crc32c/tests/test___init__.py | 23 ++++++++++++++++--- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/packages/google-crc32c/src/google_crc32c/python.py b/packages/google-crc32c/src/google_crc32c/python.py index c90c8a01d219..1d98ac4885ee 100644 --- a/packages/google-crc32c/src/google_crc32c/python.py +++ b/packages/google-crc32c/src/google_crc32c/python.py @@ -64,8 +64,7 @@ class Checksum(CommonChecksum): def __init__(self, initial_value=b""): self._crc = 0 - if initial_value != b"": - self.update(initial_value) + self.update(initial_value) def update(self, data): """Update the checksum with a new chunk of data. diff --git a/packages/google-crc32c/tests/test___init__.py b/packages/google-crc32c/tests/test___init__.py index e31ec37c2640..90bcc6e3cb9e 100644 --- a/packages/google-crc32c/tests/test___init__.py +++ b/packages/google-crc32c/tests/test___init__.py @@ -18,6 +18,7 @@ import mmap import sys import threading +import time from unittest import mock import pytest # type: ignore @@ -349,7 +350,7 @@ def test_value_releases_gil_for_immutable_memory(_cext, factory): # The counterpart of the test above: nothing can write to a bytes object, # so hashing it (or a view into it) runs with the GIL released and the # probe thread gets to run while value() is in progress. - data = factory(bytes(64 * 1024 * 1024)) + data = factory(bytes(2 * _cext.GIL_RELEASE_THRESHOLD)) go = threading.Event() in_call = False seen = [] @@ -363,9 +364,14 @@ def probe(): try: thread = threading.Thread(target=probe, daemon=True) thread.start() - go.set() in_call = True - _cext.value(data) + go.set() + # On a busy machine one call can finish before the probe is scheduled, + # so hash until it has run. The deadline stays below the switch + # interval: only a released GIL lets the probe in during the loop. + deadline = time.monotonic() + 5 + while not seen and time.monotonic() < deadline: + _cext.value(data) in_call = False thread.join() finally: @@ -449,6 +455,17 @@ def test_ctor_w_bytes_like(_crc32c, factory): helper = _crc32c.Checksum(factory(ISCSI_BYTES)) assert helper._crc == ISCSI_CRC + @staticmethod + def test_ctor_w_exporter_whose_ne_raises(_crc32c): + # Like a numpy array, whose ``!=`` compares element-wise and has no + # truth value, so the constructor must not compare initial_value. + class NoNotEqual(bytes): + def __ne__(self, other): + raise TypeError("compared") # pragma: NO COVER + + helper = _crc32c.Checksum(NoNotEqual(ISCSI_BYTES)) + assert helper._crc == ISCSI_CRC + @staticmethod def test_update_array(): import array