Modernize cChardet for Python 3.11–3.14 and stable 2.2.0

.github/workflows/build.yaml

@@ -1,59 +1,90 @@
-name: Build Wheels
+name: Build wheels
+
 on:
   push:
     branches: [master]
-  create:
+    tags: ["v*"]
+  pull_request:
   workflow_dispatch:
 
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
-  build_wheels:
+  build-sdist:
+    name: Source distribution
+    if: startsWith(github.ref, 'refs/tags/v')
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          submodules: recursive
+
+      - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
+        with:
+          version: latest-known
+          python-version: "3.14"
+
+      - run: uv sync --locked
+      - run: uv build --sdist
+      - run: uv run twine check dist/*
+      - name: Verify tag matches package version
+        if: startsWith(github.ref, 'refs/tags/v')
+        run: test -f "dist/cchardet-${GITHUB_REF_NAME#v}.tar.gz"
+
+      - uses: actions/upload-artifact@v7
+        with:
+          name: sdist
+          path: dist/*.tar.gz
+          if-no-files-found: error
+
+  build-wheels:
+    name: Wheels / ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: false
       matrix:
-        os: ["ubuntu-latest", "windows-latest", "macos-latest"]
-    runs-on: ${{ matrix.os }}
-    name: Build wheels on ${{ matrix.os }}
+        os: [ubuntu-latest, windows-latest, macos-latest]
 
     steps:
       - name: Set up QEMU
         if: runner.os == 'Linux'
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@v4
         with:
-          platforms: all
+          platforms: arm64
 
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v7
         with:
-          submodules: "recursive"
+          submodules: recursive
 
-      - uses: actions/setup-python@v5
+      - uses: pypa/cibuildwheel@v4.2.0
+
+      - uses: actions/upload-artifact@v7
         with:
-          python-version: "3.x"
-          cache: "pip"
-
-      - name: Install dependencies
-        shell: bash
-        run: |
-          python -VV
-          pip install -r requirements-dev.txt
-
-      - name: Cythonize
-        shell: bash
-        run: |
-          python -VV
-          make cython
-
-      - name: Build
-        uses: pypa/cibuildwheel@v2.18.1
-        env:
-          # NOTICE: ローカルでは想定通りにテストは通るが､ cibuildwheel では 特定のファイルの文字コード検知がうまくいかないので一時的に無効化する
-          # CIBW_TEST_REQUIRES: "pytest"
-          # CIBW_TEST_COMMAND: "pytest {project}/tests"
-
-          CIBW_ARCHS_LINUX: auto aarch64
-          CIBW_ARCHS_MACOS: x86_64 universal2 arm64
-          CIBW_SKIP: pp* cp36-* cp37-* cp38-*
-
-      - name: Upload Wheels to artifact
-        uses: actions/upload-artifact@v4
+          name: wheels-${{ runner.os }}
+          path: wheelhouse/*.whl
+          if-no-files-found: error
+
+  publish:
+    name: Publish to PyPI
+    if: startsWith(github.ref, 'refs/tags/v')
+    needs: [build-sdist, build-wheels]
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/project/cchardet/
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v7
         with:
-          name: wheels-${{ matrix.os }}
-          path: wheelhouse
+          pattern: "*"
+          path: dist
+          merge-multiple: true
+
+      - name: Publish distributions with attestations
+        uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2

.github/workflows/test.yml

@@ -1,53 +1,85 @@
-name: Run tests
+name: Test
+
 on:
   push:
+    branches: [master]
   pull_request:
-  create:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 jobs:
+  native-safety:
+    name: Native ASan / UBSan
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          submodules: recursive
+
+      - name: Configure sanitizers
+        run: >-
+          cmake -S src/ext/uchardet -B build/uchardet-sanitize
+          -DBUILD_BINARY=OFF -DCMAKE_BUILD_TYPE=Debug
+          -DCMAKE_C_FLAGS="-fsanitize=address,undefined -fno-omit-frame-pointer"
+          -DCMAKE_CXX_FLAGS="-fsanitize=address,undefined -fno-omit-frame-pointer -Werror=maybe-uninitialized"
+          -DCMAKE_EXE_LINKER_FLAGS="-fsanitize=address,undefined"
+          -DCMAKE_SHARED_LINKER_FLAGS="-fsanitize=address,undefined"
+
+      - run: cmake --build build/uchardet-sanitize --parallel 2
+      - run: ctest --test-dir build/uchardet-sanitize --output-on-failure
+
   test:
+    name: Python ${{ matrix.python-version }} / ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: false
       matrix:
-        os: ["ubuntu-latest", "windows-latest", "macos-latest"]
-        py: ["3.13-dev", "3.12", "3.11", "3.10", "3.9"]
-
-    runs-on: ${{ matrix.os }}
-    name: Run test with Python ${{ matrix.py }} on ${{ matrix.os }}
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        python-version: ["3.11", "3.12", "3.13", "3.14"]
 
     steps:
-      - name: Checkout
-        uses: actions/checkout@v4
+      - uses: actions/checkout@v7
         with:
-          submodules: "recursive"
+          submodules: recursive
 
-      - name: Set up Python
-        uses: actions/setup-python@v5
+      - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
         with:
-          python-version: ${{ matrix.py }}
-          allow-prereleases: true
-          cache: "pip"
-
-      - name: Install dependencies
-        shell: bash
-        run: |
-          python -VV
-          pip install -r requirements-dev.txt
-
-      - name: Build
-        shell: bash
-        run: |
-          python -VV
-          make cython
-          pip install .
+          version: latest-known
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install locked dependencies
+        run: uv sync --locked
 
       - name: Lint
-        shell: bash
-        run: |
-          python -VV
-          ruff check
+        if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.14'
+        run: uv run ruff check
+
+      - name: Type check
+        if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.14'
+        run: uv run mypy
 
       - name: Test
-        shell: bash
-        run: |
-          python -VV
-          pytest -vs tests
+        run: uv run pytest
+
+  package:
+    name: Verify distributions
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          submodules: recursive
+
+      - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
+        with:
+          version: latest-known
+          python-version: "3.14"
+
+      - run: uv sync --locked
+      - run: uv build
+      - run: uv run twine check dist/*

.gitignore

@@ -41,7 +41,6 @@ MANIFEST
 None
 *.tar.gz
 .directory
-.python-version
 .tox/
 
 .DS_Store

.gitmodules

@@ -1,3 +1,4 @@
 [submodule "src/ext/uchardet"]
 	path = src/ext/uchardet
 	url = https://github.com/PyYoshi/uchardet.git
+	branch = cchardet

.python-version

@@ -0,0 +1 @@
+3.14

CHANGES.md

@@ -1,6 +1,31 @@
 # CHANGES
 
-## 2.x.x
+## 2.2.0 (Unreleased)
+
+- Support Python 3.11, 3.12, 3.13, and 3.14, and drop support for Python 3.10
+  and earlier versions
+- Modernize development, testing, and packaging around `uv`, current Cython, and
+  current Python packaging standards
+- Update to the maintained PyYoshi/uchardet fork while preserving the existing
+  cChardet-compatible API
+- Improve UTF-8 validation and GB18030 candidate ranking in the bundled uchardet
+- Return language and confidence information from the detection APIs
+- Add `detect_all()` for inspecting ranked encoding and language candidates
+- Add optional language weights to the Python API and CLI without changing the
+  default candidate ranking
+- Accept `bytes`, `bytearray`, and `memoryview` inputs, and add `max_bytes` to
+  bound detection work without slicing the input
+- Stream input in `cchardetect`, with JSON output and configurable chunk and byte
+  limits
+- Expose uchardet's early-completion state for incremental detection and safely
+  chunk inputs larger than its internal 32-bit length
+- Ship PEP 561 type information for the public Python and Cython-extension APIs
+- Add reproducible performance and accuracy comparisons for cChardet, chardet,
+  and charset-normalizer, including their available native accelerators
+- Build and test distributions across supported Python versions on Linux, macOS,
+  and Windows, including ARM64 wheel coverage
+- Add native sanitizer, arbitrary-byte consistency, and installed-wheel smoke
+  tests
 
 ## 2.2.0-alpha.2 (2024-06-xx)
 

MANIFEST.in

@@ -1,5 +1,5 @@
-include *.txt
-include CHANGES.rst COPYING README.rst
-recursive-include src *.pyx *.pxd *.pxi *.py *.cpp
-recursive-include src/ext *
-recursive-include tests *
+include CHANGES.md COPYING README.md
+recursive-include benchmarks *.py
+recursive-include docs *.md
+recursive-include src/cchardet *.py *.pyi *.pyx py.typed
+recursive-include src/ext/uchardet/src *.cpp *.h *.tab

Makefile

@@ -1,4 +1,4 @@
-.PHONY: clean
+.PHONY: clean sync cython build test lint typecheck format check sdist
 clean:
 	$(RM) -r \
 		.pytest_cache \
@@ -15,28 +15,32 @@ clean:
 		tests/*.pyc \
 		wheelhouse
 
-.PHONY: cython
 cython:
-	cython --cplus src/cchardet/_cchardet.pyx
+	uv run cython --cplus src/cchardet/_cchardet.pyx
+
+sync:
+	uv sync --locked
 
-.PHONY: test
 test: clean cython
-	python setup.py build_ext -i -f
-	pytest tests
+	uv run python setup.py build_ext -i -f
+	uv run pytest tests
 
-.PHONY: lint
 lint:
-	ruff check
+	uv run ruff check
+
+typecheck:
+	uv run mypy
 
-.PHONY: format
 format:
-	ruff format
+	uv run ruff format
+
+check: lint typecheck test
 
-.PHONY: bench
 bench: clean cython
-	python setup.py build_ext -i -f
-	python tests/bench.py
+	uv run python setup.py build_ext -i -f
+	uv run python tests/bench.py
+
+build: clean
+	uv build
 
-.PHONY: sdist
-sdist: clean cython
-	python setup.py sdist
+sdist: build