Port the pileup, cut-site tables and the memory planner — Closes #3

.github/workflows/test.yml

@@ -0,0 +1,50 @@
+name: Run tests
+
+on:
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      - main
+
+permissions: {}
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: Python ${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version:
+          - '3.11'
+          - '3.12'
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      # The pileup extra resolves giql from a git revision, so the
+      # checkout needs network here. See the comment on
+      # `[tool.uv.sources]` in pyproject.toml.
+      - name: Install
+        run: uv sync --all-extras --group dev
+
+      - name: Lint
+        run: |
+          uv run ruff format --check src tests
+          uv run ruff check src tests
+
+      - name: Test
+        run: uv run pytest -q

.gitignore

@@ -0,0 +1,39 @@
+# macOS
+.DS_Store
+
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+build/
+dist/
+sdist/
+wheels/
+*.egg-info/
+*.egg
+.eggs/
+
+# Environments
+.venv/
+venv/
+
+# Tooling
+.pytest_cache/
+.ruff_cache/
+.hypothesis/
+.coverage
+coverage.xml
+htmlcov/
+
+# Docs
+docs/_build/
+
+# Jupyter
+.ipynb_checkpoints/
+
+# SDLC working state: local review roles, guides and accumulated review
+# documents. Untracked here as it is in the sibling projects -- the
+# review history is versioned by its own repo under .sdlc, and the
+# guides are local working config rather than a project artifact.
+.sdlc/

MANIFEST.in

@@ -0,0 +1,7 @@
+# The whole test tree, so the tests run from an sdist exactly as they do
+# from the repository -- without conftest.py they would run against a
+# different cache directory than the one the suite assumes.
+graft tests
+
+global-exclude */.ipynb_checkpoints/*
+global-exclude *.py[cod]

README.md

@@ -0,0 +1,24 @@
+# quadrat
+
+Sequence-to-function training data, straight from BAM.
+
+A quadrat is the fixed-size frame a field ecologist drops on the ground to tally everything inside it, moved across a landscape by a sampling design chosen separately from the frame. This package is that: a fixed-length window, placed at loci by a sampling scheme, tallying the cut sites that fall inside.
+
+Everything between a set of alignments and a batch of `(sequence, signal)` pairs is independent of the model that consumes them. `quadrat` implements that span — the pileup, the window cache, the sampler and the loader — so any sequence-to-function architecture can be trained without reimplementing it.
+
+## Status
+
+Under construction. The data path is being extracted from [cherimoya](https://github.com/conradbzura/cherimoya); see [#1](https://github.com/abdenlab/quadrat/issues/1) for the specification and the slices it lands in.
+
+## Install
+
+```bash
+pip install quadrat            # stream windows over cut-site tables
+pip install "quadrat[pileup]"  # also build those tables from alignments
+```
+
+The pileup reads alignments through [polars-bio](https://github.com/biodatageeks/polars-bio) and transpiles its aggregate with [giql](https://github.com/abdenlab/giql). Both are imported only when a pileup runs, so the base install stays light for a caller who already has tables.
+
+## License
+
+MIT.

pyproject.toml

@@ -0,0 +1,114 @@
+[build-system]
+build-backend = "hatchling.build"
+requires = ["hatchling"]
+
+[project]
+authors = [
+    { name = "Nezar Abdennur", email = "nabdennur@gmail.com" },
+    { name = "Conrad Bzura", email = "conradbzura@gmail.com" },
+]
+classifiers = [
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Topic :: Scientific/Engineering :: Bio-Informatics",
+]
+description = "Sequence-to-function training data, straight from BAM."
+dynamic = ["version"]
+license = "MIT"
+maintainers = [
+    { name = "Nezar Abdennur", email = "nabdennur@gmail.com" },
+    { name = "Conrad Bzura", email = "conradbzura@gmail.com" },
+]
+name = "quadrat"
+readme = "README.md"
+requires-python = ">=3.11"
+
+# Floors, not tested ceilings. 3.11 rather than 3.10 because the pileup
+# engine -- polars-bio, polars, giql -- is gated there, and the pileup is
+# not an optional part of what this package is for.
+dependencies = [
+    "numpy>=1.14.2",
+    "pandas>=1.3.3",
+    "torch>=2.9.0",
+    "tqdm>=4.64.1",
+    "joblib>=1.0.0",
+    "tangermeme>=1.5.0",
+]
+
+# The pileup reads alignments through polars-bio's BAM provider and
+# transpiles its aggregate with giql. Every one of these is imported
+# lazily inside a function body, so a caller that only streams windows
+# over cut-site tables it already has never pays for them.
+#
+# `pyfaidx` belongs here rather than in a dev group: `chrom_lengths` and
+# `SequenceStore` import it at runtime.
+[project.optional-dependencies]
+pileup = [
+    "polars-bio>=0.35.1",
+    "polars>=1.30",
+    "giql",
+    "pyfaidx",
+]
+
+[dependency-groups]
+dev = [
+    "pytest>=9.1.1",
+    "pytest-mock>=3.14",
+    "hypothesis>=6.0.0",
+    "pysam",
+    "pybigtools",
+    "pyBigWig",
+    "ruff>=0.1.0",
+]
+docs = [
+    "sphinx>=7.0",
+    "sphinx-autobuild>=2024.0",
+    "sphinx-book-theme>=1.1",
+    "sphinx-design>=0.6",
+]
+
+# BRANCH-ONLY. The datafusion-bio GIQL target is not on PyPI and lives on
+# a proof-of-concept branch, so this pins the public commit at that
+# branch's tip. Its library code is identical to what cherimoya builds
+# against; the only delta is benchmark material outside `src`. Replace
+# with a version specifier once the target reaches `main` and ships.
+[tool.uv.sources]
+giql = { git = "https://github.com/abdenlab/giql", rev = "59c74276e07cc4680cb24303a9192c3a71c11d59" }
+
+[tool.hatch.version]
+path = "src/quadrat/__init__.py"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/quadrat"]
+
+[tool.ruff]
+line-length = 89
+
+# Stated explicitly rather than left to the default, which has widened
+# over ruff releases and would otherwise change what passes when the
+# pin moves. Pyflakes plus the pycodestyle errors that are errors:
+# undefined names, unused imports, broken syntax.
+#
+# Deliberately absent are the style-migration families -- UP, SIM, FURB,
+# RUF. The code ported from cherimoya uses `str.format` throughout and
+# `class X(object)` by convention. Flagging either would mean rewriting
+# thousands of lines in the same change that moves them, which is the
+# one thing that makes a port unreviewable.
+[tool.ruff.lint]
+select = ["E4", "E7", "E9", "F"]
+
+# A test module calls `pytest.importorskip` before importing quadrat, so
+# a machine without the pileup extra skips the module instead of failing
+# collection. That necessarily puts imports after statements.
+[tool.ruff.lint.per-file-ignores]
+"tests/*" = ["E402"]
+
+[tool.ruff.format]
+docstring-code-format = true
+
+[tool.pytest.ini_options]
+markers = [
+    "integration: exercises a real cross-boundary interaction rather than a unit",
+]
+filterwarnings = ["ignore::DeprecationWarning"]

src/quadrat/__init__.py

@@ -0,0 +1,53 @@
+"""Training data for sequence-to-function models, from alignments.
+
+A sequence-to-function model consumes pairs of a one-hot DNA sequence
+over an input window and a measured signal profile over a shorter,
+concentric output window. Everything between a set of alignments and a
+batch of such pairs is independent of the architecture downstream, and
+that is what this package implements.
+
+Four terms, used consistently throughout:
+
+``locus``
+    A genomic coordinate, typically a peak summit or a sampled
+    background position.
+
+``window``
+    A fixed-length interval centred on a locus, in two nested lengths --
+    an input window supplying sequence and a shorter output window
+    supplying signal.
+
+``channel``
+    One measured track.
+
+``cut-site table``
+    The intermediate representation: a per-contig, position-sorted
+    Parquet table of ``(position, value)`` pairs for one channel,
+    written once by the pileup and read many times by the loader.
+
+``draw``
+    One ``(locus, jitter offset, strand)`` triple the sampler emits, so
+    a locus yields many draws under jitter and reverse-complement
+    augmentation.
+
+The pileup reads alignments through polars-bio and transpiles its
+aggregate with giql, both of which are installed by the ``pileup``
+extra and imported only when a pileup actually runs. A caller that
+already has cut-site tables needs neither.
+"""
+
+__version__ = "0.1.0.dev0"
+
+from quadrat.pileup import pileup_alignments, pileup_many
+from quadrat.reference import chrom_lengths
+from quadrat.tables import CUT_SITE_SUFFIX, read_manifest, require_complete
+
+__all__ = [
+    "CUT_SITE_SUFFIX",
+    "__version__",
+    "chrom_lengths",
+    "pileup_alignments",
+    "pileup_many",
+    "read_manifest",
+    "require_complete",
+]