feat: add the zarr-indexing package (TensorStore-style index transforms, ndsel wire format)

.github/workflows/check_changelogs.yml

@@ -29,3 +29,6 @@ jobs:
 
       - name: Check zarr-metadata changelog entries
         run: uv run --no-sync python ci/check_changelog_entries.py packages/zarr-metadata/changes
+
+      - name: Check zarr-indexing changelog entries
+        run: uv run --no-sync python ci/check_changelog_entries.py packages/zarr-indexing/changes

.github/workflows/zarr-indexing-release.yml

@@ -0,0 +1,117 @@
+name: zarr-indexing release
+
+on:
+  workflow_dispatch:
+  push:
+    tags:
+      - 'zarr_indexing-v*'
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  build:
+    name: Build wheel and sdist
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+        working-directory: packages/zarr-indexing
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+          fetch-depth: 0          # hatch-vcs needs full history + tags
+
+      - name: Install Hatch
+        uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc
+        with:
+          version: '1.16.5'
+
+      - name: Build
+        run: hatch build
+
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: zarr-indexing-dist
+          path: packages/zarr-indexing/dist
+
+  test_artifacts:
+    name: Test built artifacts
+    needs: [build]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: zarr-indexing-dist
+          path: dist
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
+        with:
+          enable-cache: false
+
+      - name: Set up Python
+        run: uv python install 3.12
+
+      - name: Install built wheel and run import smoke test
+        run: |
+          wheel=$(ls dist/*.whl)
+          uv run --with "${wheel}" --python 3.12 --no-project \
+            python -c "import zarr_indexing; print('zarr_indexing', zarr_indexing.__version__)"
+
+  upload_pypi:
+    name: Upload to PyPI
+    needs: [build, test_artifacts]
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/zarr_indexing-v')
+    runs-on: ubuntu-latest
+    environment:
+      name: zarr-indexing-releases
+      url: https://pypi.org/p/zarr-indexing
+    permissions:
+      id-token: write          # required for OIDC trusted publishing
+      attestations: write      # required for artifact attestations
+    steps:
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: zarr-indexing-dist
+          path: dist
+
+      - name: Generate artifact attestation
+        uses: actions/attest@f7c74d28b9d84cb8768d0b8ca14a4bac6ef463e6 # v4.2.0
+        with:
+          subject-path: dist/*
+
+      - name: Publish package to PyPI
+        uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0
+
+  upload_testpypi:
+    name: Upload to TestPyPI
+    needs: [build, test_artifacts]
+    if: github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    environment:
+      name: zarr-indexing-releases-test
+      url: https://test.pypi.org/p/zarr-indexing
+    permissions:
+      id-token: write
+      attestations: write
+    steps:
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: zarr-indexing-dist
+          path: dist
+
+      - name: Generate artifact attestation
+        uses: actions/attest@f7c74d28b9d84cb8768d0b8ca14a4bac6ef463e6 # v4.2.0
+        with:
+          subject-path: dist/*
+
+      - name: Publish package to TestPyPI
+        uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0
+        with:
+          repository-url: https://test.pypi.org/legacy/

.github/workflows/zarr-indexing.yml

@@ -0,0 +1,123 @@
+name: zarr-indexing
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'packages/zarr-indexing/**'
+      - '.github/workflows/zarr-indexing.yml'
+  pull_request:
+    paths:
+      - 'packages/zarr-indexing/**'
+      - '.github/workflows/zarr-indexing.yml'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: pytest py=${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.12', '3.13', '3.14']
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+      - name: Install uv
+        uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
+        with:
+          enable-cache: true
+      - name: Set up Python ${{ matrix.python-version }}
+        run: uv python install ${{ matrix.python-version }}
+      # The transform tests exercise chunk resolution against zarr's ChunkGrid,
+      # so they run from the repo root against the root environment (which
+      # provides `zarr`) with this package as an editable overlay rather than in
+      # package isolation.
+      - name: Sync test dependency group
+        run: uv sync --group test --python ${{ matrix.python-version }}
+      - name: Run pytest
+        run: uv run --no-sync --group test --with-editable ./packages/zarr-indexing python -m pytest packages/zarr-indexing/tests
+
+  ruff:
+    name: ruff
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+        working-directory: packages/zarr-indexing
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+      - name: Install uv
+        uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
+      - name: Run ruff
+        run: uvx ruff check .
+
+  pyright:
+    name: pyright
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+        working-directory: packages/zarr-indexing
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+      - name: Install uv
+        uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
+        with:
+          enable-cache: true
+      - name: Set up Python
+        run: uv python install 3.12
+      - name: Sync test dependency group
+        run: uv sync --group test --python 3.12
+      - name: Run pyright
+        run: uv run --group test --with pyright pyright src
+
+  docs:
+    name: docs
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+        working-directory: packages/zarr-indexing
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+      - name: Install uv
+        uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
+        with:
+          enable-cache: true
+      - name: Install just
+        uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4
+      - name: Build docs
+        # The strict mkdocs build lives in packages/zarr-indexing/justfile.
+        run: just docs-check
+
+  zarr-indexing-complete:
+    name: zarr-indexing complete
+    needs: [test, ruff, pyright, docs]
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check failure
+        if: |
+          contains(needs.*.result, 'failure') ||
+          contains(needs.*.result, 'cancelled')
+        run: exit 1
+      - name: Success
+        run: echo Success!

.readthedocs.yaml

@@ -6,15 +6,14 @@ build:
     python: "3.12"
   jobs:
     post_checkout:
-      # Cancel pull request builds whose changes are confined to the
-      # zarr-metadata package, which has its own Read the Docs project. Exit
-      # code 183 cancels the build and reports success to the Git provider.
-      # Scoped to PR builds ("external" versions) because origin/main is only
-      # a meaningful diff base there. Read the Docs strips shell quoting from
-      # commands, so the exclude pathspec must use the quote-free :! form,
-      # not ':(exclude)'.
+      # Cancel pull request builds whose changes are confined to the packages
+      # that have their own Read the Docs projects. Exit code 183 cancels the
+      # build and reports success to the Git provider. Scoped to PR builds
+      # ("external" versions) because origin/main is only a meaningful diff
+      # base there. Read the Docs strips shell quoting from commands, so the
+      # exclude pathspecs must use the quote-free :! form, not ':(exclude)'.
       - |
-        if [ "$READTHEDOCS_VERSION_TYPE" = "external" ] && git diff --quiet origin/main -- :!packages/zarr-metadata;
+        if [ "$READTHEDOCS_VERSION_TYPE" = "external" ] && git diff --quiet origin/main -- :!packages/zarr-metadata :!packages/zarr-indexing;
         then
           exit 183;
         fi

changes/4205.bugfix.md

@@ -0,0 +1,9 @@
+Fixed several small correctness issues from the codec-pipeline performance work: construction-time
+codec warnings (e.g. sharding's "disables partial reads" warning) no longer fire twice per array
+open — including for `FusedCodecPipeline`, which previously re-warned via its own codec-chain
+reconstruction and, on the async fallback path, on every decode/encode call; `concurrent_iter` now
+schedules its tasks eagerly, matching its documented contract; an invalid
+`codec_pipeline.max_workers` config/environment value now warns and falls back to the default
+instead of raising mid-read; and `FusedCodecPipeline`'s async fallback helpers now cancel
+already-spawned fetch/decode/write tasks instead of abandoning them in the background when one
+fails.

changes/4219.bugfix.md

@@ -0,0 +1,3 @@
+`DefaultChunkKeyEncoding.decode_chunk_key` now validates that a chunk key
+starts with the configured `c<separator>` prefix and raises `ValueError` for
+malformed keys, instead of silently decoding them incorrectly.

mkdocs.yml

@@ -93,6 +93,7 @@ nav:
       - '<code class="doc-symbol doc-symbol-toc doc-symbol-function"></code> <code>zarr.zeros</code>': api/zarr/functions/zeros.md
       - '<code class="doc-symbol doc-symbol-toc doc-symbol-function"></code> <code>zarr.zeros_like</code>': api/zarr/functions/zeros_like.md
       - 'zarr-metadata ↪': https://zarr-metadata.readthedocs.io/
+      - 'zarr-indexing ↪': https://zarr-indexing.readthedocs.io/
   - release-notes.md
   - contributing.md
 hooks:

packages/zarr-indexing/.readthedocs.yaml

@@ -0,0 +1,30 @@
+# Read the Docs configuration for the zarr-indexing docs site, separate from
+# the zarr-python site configured by the repo-root .readthedocs.yaml. The RTD
+# project for zarr-indexing must set its configuration-file path to
+# packages/zarr-indexing/.readthedocs.yaml.
+version: 2
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.12"
+  jobs:
+    post_checkout:
+      # Cancel pull request builds that do not touch this package. Exit code
+      # 183 cancels the build and reports success to the Git provider. Scoped
+      # to PR builds ("external" versions) because origin/main is only a
+      # meaningful diff base there.
+      - |
+        if [ "$READTHEDOCS_VERSION_TYPE" = "external" ] && git diff --quiet origin/main -- packages/zarr-indexing;
+        then
+          exit 183;
+        fi
+    install:
+      - pip install --upgrade pip
+      - pip install ./packages/zarr-indexing --group packages/zarr-indexing/pyproject.toml:docs
+    build:
+      html:
+        - mkdocs build --strict -f packages/zarr-indexing/mkdocs.yml --site-dir $READTHEDOCS_OUTPUT/html
+
+mkdocs:
+  configuration: packages/zarr-indexing/mkdocs.yml

packages/zarr-indexing/CHANGELOG.md

@@ -0,0 +1,3 @@
+# Release notes
+
+<!-- towncrier release notes start -->

packages/zarr-indexing/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015-2025 Zarr Developers <https://github.com/zarr-developers>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.