diff --git a/.cargo/config.toml b/.cargo/config.toml deleted file mode 100644 index ddff440..0000000 --- a/.cargo/config.toml +++ /dev/null @@ -1,2 +0,0 @@ -[build] -rustflags = ["-C", "target-cpu=native"] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d145442..615253d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,9 +16,14 @@ jobs: - name: Build run: cargo build --verbose - name: Run tests - run: cargo test --verbose + run: cargo test --verbose --all-features - name: Run tests (release) - run: cargo test --verbose --release + run: cargo test --verbose --release --all-features + - name: Check feature matrix + run: | + cargo check --no-default-features + cargo check --no-default-features --features paraseq + cargo check --no-default-features --features anyhow fmt_lint: runs-on: ubuntu-latest @@ -37,7 +42,7 @@ jobs: steps: - uses: actions/checkout@v7 - name: run example ${{ matrix.ext }} - run: cargo run --release --example grep -- ./data/subset.${{ matrix.ext }} "ACGTACGT" + run: cargo run --example grep -- ./data/subset.${{ matrix.ext }} "ACGTACGT" example_range: runs-on: ubuntu-latest @@ -47,7 +52,7 @@ jobs: steps: - uses: actions/checkout@v7 - name: run example ${{ matrix.ext }} - run: cargo run --release --example parallel_range -- ./data/subset.${{ matrix.ext }} 4 30 200 + run: cargo run --example parallel_range -- ./data/subset.${{ matrix.ext }} -T 4 --start 30 --end 200 example_write: runs-on: ubuntu-latest @@ -57,9 +62,9 @@ jobs: steps: - uses: actions/checkout@v7 - name: run example (single) ${{ matrix.ext }} - run: cargo run --release --example write -- ./data/subset_R1.fastq.gz -o ./output.${{ matrix.ext }} + run: cargo run --example auto-write -F paraseq -- ./data/subset_R1.fastq.gz -o ./output.${{ matrix.ext }} - name: run example (paired) ${{ matrix.ext }} - run: cargo run --release --example write -- ./data/subset_R1.fastq.gz ./data/subset_R2.fastq.gz -o ./output.${{ matrix.ext }} + run: cargo run --example auto-write -F paraseq -- ./data/subset_R1.fastq.gz ./data/subset_R2.fastq.gz -o ./output.${{ matrix.ext }} example_read: runs-on: ubuntu-latest @@ -69,4 +74,4 @@ jobs: steps: - uses: actions/checkout@v7 - name: run example ${{ matrix.ext }} - run: cargo run --release --example read -- ./data/subset.${{ matrix.ext }} + run: cargo run --example read -- ./data/subset.${{ matrix.ext }} diff --git a/.gitignore b/.gitignore index e147b57..703c3c8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ /target -Cargo.lock data/test* diff --git a/CHANGELOG.md b/CHANGELOG.md index ab0efee..4e72fdb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,59 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.10.0] - 2026-08-25 + +- Large cleanup of internal and public API. +- Runtime dependencies removed (`byteorder`, `num_cpus`, `auto_impl`, and `memchr`). +- Deprecated items removed from the public API. + +### Removed + +- **`bq::StreamWriter` / `bq::StreamWriterBuilder`** + - pure delegating wrappers; compose `bq::WriterBuilder` with `std::io::BufWriter` instead (see the `network_streaming` example). +- **Deprecated `write_record` / `write_paired_record`** on the `bq` and `vbq` writers + - use `push(SequencingRecord)`. +- **`bq::Encoder`** from the public API + - `bq` and `vbq` now share one internal encoder; + - sequence-length validation happens in `bq::Writer::push`. +- **`vbq::BlockIndex::from_vbq`** + - indexes are embedded in VBQ files - removing this holdover from way back when; + - use `MmapReader::load_index`. +- **Redundant header constructors** + - `bq::FileHeader::{new, new_extended, set_bitsize}` and `vbq::FileHeader::{new, with_capacity, set_bitsize}`; + - use the `FileHeaderBuilder`s. +- **Never-constructed error variants** + - the five `WriteError` flag-mismatch variants and `CbqError::MissingSequenceOnSequencingRecord`; + - `BuilderError` folds into `HeaderError::MissingSequenceLength` and `FastxEncodingError` into `WriteError`. +- **`&R` / `&mut R` impls of `BinseqRecord`** + - the `auto_impl` dependency is dropped and nothing used the reference impls; + - pass records by value (they are already references anyways). +- Dead accessors throughout (`RefRecord::{config, set_id}`, `cbq` writer `usage` / `bytes_written`, and similar zero-caller items), plus the redundant `streaming` and `write` examples. + +### Changed + +- `SequencingRecord` fields are now public and the getters are removed +- FASTX encoding now drains only _completed_ blocks per batch and does a final drain per thread + - CBQ blocks stay full across batches for better compression and less time under the global writer lock. +- Dependencies: `byteorder` replaced with `std` little-endian conversions (byte-identical output), `num_cpus` replaced with `std::thread::available_parallelism`, `auto_impl` dropped, and `memchr` moved to dev-dependencies. +- The `paraseq` feature is no longer enabled by default, reducing compile times when FASTX encoding is not needed. Enable it with `features = ["paraseq"]` to use `encode_fastx` (which remains visible on docs.rs with a feature banner). +- `anyhow` is now actually an optional dependency: + - the CBQ Elias-Fano codec is handled directly without the auto-derive from thiserror + - fixes the optional feature which actually never compiled when disabled. +- Internal simplification: + - one shared `bq`/`vbq` encoder + - `BatchRecord` wraps `RefRecord` + - `cbq` `Span` replaced by `Range` + - unreachable `pub` items demoted to `pub(crate)`. + +### Fixed + +- The `error` test module was missing `#[cfg(test)]`, so a test-only enum compiled into release builds. +- `cbq::Index::average_block_size` reports `0.0` instead of `NaN` for files with fewer than two blocks. +- `encode_fastx` with single non-paired input no longer records second read's length as `xlen` + - made fixed-length BQ output look paired and fail on interleaved encoding. + - The extended length now only probes when builder is configured paired. + ## [0.9.6] - 2026-08-12 ### Changed diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..f7e9efd --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,1029 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bgzip" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b64fd8980fb64af5951bc05de7772b598150a6f7eac42ec17f73e8489915f99b" +dependencies = [ + "flate2", + "log", + "rayon", + "thiserror 1.0.69", +] + +[[package]] +name = "binseq" +version = "0.10.0" +dependencies = [ + "anyhow", + "bitnuc 0.4.1", + "bitnuc 0.5.4", + "bytemuck", + "clap", + "itoa", + "memchr", + "memmap2", + "paraseq", + "parking_lot", + "rand", + "sucds", + "tempfile", + "thiserror 2.0.20", + "zstd", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "bitnuc" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7871516cbb4e097e220623917e1491c995ee58c8018d929d4b1e76c378002bf" + +[[package]] +name = "bitnuc" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ff0fcddd0c1e70392497c43846f73f5345d0588bd1aee9a16338c13f77223ba" +dependencies = [ + "fearless_simd", + "thiserror 2.0.20", +] + +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0e56a716f1e132ff6bf4bdac1c944a3fcdc1cae65f70a4a2a1ac3b401d2d1f" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "bzip2" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c" +dependencies = [ + "libbz2-rs-sys", +] + +[[package]] +name = "cc" +version = "1.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "clap" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "crc32fast" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "defmt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror 2.0.20", +] + +[[package]] +name = "either" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" + +[[package]] +name = "env_filter" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "fearless_simd" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4beca3cb2444e3304ac30843cc091f44ed58932353cd492ce740067bfce6b12" + +[[package]] +name = "find-msvc-tools" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jiff" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" +dependencies = [ + "defmt", + "jiff-core", + "jiff-static", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", +] + +[[package]] +name = "jiff-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" +dependencies = [ + "defmt", +] + +[[package]] +name = "jiff-static" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + +[[package]] +name = "libbz2-rs-sys" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "liblzma" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fe0a34ca854fd4f20c07f696fc8675aec78f87d88d29f5e10257a7490a1b2e1" +dependencies = [ + "liblzma-sys", +] + +[[package]] +name = "liblzma-sys" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0dad045e4b1b7b170be4b60b54b780cafb4490165461bac7d1cf7b703f61d5f" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "memmap2" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +dependencies = [ + "libc", +] + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "niffler" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01baa3696744e580712b28344fe25ed4ac1e8c4b76cc5189a34e713924c25208" +dependencies = [ + "bgzip", + "bzip2", + "cfg-if", + "flate2", + "liblzma", + "thiserror 2.0.20", + "zstd", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "paraseq" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f0c7259499c750423489d3fe955dfc60183f998f203a87a6f2d4c56872a2f4" +dependencies = [ + "anyhow", + "crossbeam-channel", + "either", + "env_logger", + "itertools", + "log", + "memchr", + "niffler", + "num_cpus", + "parking_lot", + "smallvec", + "thiserror 2.0.20", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.13.1", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags 2.13.1", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "sucds" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd324eaa05be64f105ea5269bb8aabd70e5dd57fa5c673b167f451b07d6c0dcd" +dependencies = [ + "anyhow", + "num-traits", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl 2.0.20", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "zerocopy" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index bb976a0..81ef10b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "binseq" -version = "0.9.6" +version = "0.10.0" edition = "2024" description = "A high efficiency binary format for sequencing data" license = "MIT" @@ -11,16 +11,12 @@ categories = ["science::bioinformatics", "encoding", "data-structures"] keywords = ["binary", "nucleotide", "sequencing", "genomics", "fastq"] [dependencies] -anyhow = {version = "1.0.103", optional = true} -auto_impl = "1.3.0" +anyhow = { version = "1.0.103", optional = true } bitnuc-deprec = { package ="bitnuc", version = "0.4.1" } bitnuc = { version = "0.5.1" } bytemuck = { version = "1.25.1", features = ["derive", "extern_crate_alloc"] } -byteorder = "1.5.0" itoa = "1.0.18" -memchr = "2.8.3" memmap2 = "0.9.11" -num_cpus = "1.17.0" paraseq = { version = "0.4.14", optional = true } parking_lot = {version = "0.12.5", optional = true } rand = { version = "0.9.5", features = ["small_rng"] } @@ -29,6 +25,7 @@ thiserror = "2.0.20" zstd = { version = "0.13.3", features = ["zstdmt"] } [dev-dependencies] +memchr = "2.8.3" anyhow = "1.0.103" parking_lot = "0.12.5" clap = { version = "4.6.2", features = ["derive"] } @@ -36,7 +33,7 @@ paraseq = "0.4.14" tempfile = "3.27.0" [features] -default = ["paraseq", "anyhow"] +default = [] anyhow = ["dep:anyhow"] paraseq = ["dep:paraseq", "dep:parking_lot"] @@ -47,3 +44,11 @@ cast_precision_loss = "allow" missing_errors_doc = "allow" struct_excessive_bools = "allow" fn_params_excessive_bools = "allow" + +[[example]] +name = "auto-write" +required-features = ["paraseq"] + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--cfg", "docsrs"] diff --git a/examples/network_streaming.rs b/examples/network_streaming.rs index 71ca2f9..10d0415 100644 --- a/examples/network_streaming.rs +++ b/examples/network_streaming.rs @@ -2,8 +2,8 @@ use std::io::{BufReader, BufWriter}; use std::net::{TcpListener, TcpStream}; use std::thread; -use binseq::bq::{FileHeader, FileHeaderBuilder, StreamReader, StreamWriterBuilder}; -use binseq::{BinseqRecord, Policy, Result}; +use binseq::bq::{FileHeader, FileHeaderBuilder, StreamReader, WriterBuilder}; +use binseq::{BinseqRecord, Policy, Result, SequencingRecordBuilder}; fn server(header: FileHeader, sequence: &[u8]) -> Result<()> { // Create a listener on localhost:3000 @@ -14,19 +14,22 @@ fn server(header: FileHeader, sequence: &[u8]) -> Result<()> { let (stream, _) = listener.accept().expect("Failed to accept connection"); println!("Client connected"); - let stream = BufWriter::new(stream); + // Buffer the network stream for efficient writes + let stream = BufWriter::with_capacity(16384, stream); - // Create a stream writer with the network stream as destination - let mut writer = StreamWriterBuilder::default() + // Create a writer with the network stream as destination + let mut writer = WriterBuilder::default() .header(header) .policy(Policy::RandomDraw) - .buffer_capacity(16384) // Larger buffer for network I/O .build(stream)?; // Write sequences in a loop for i in 0..10 { - #[allow(deprecated)] - writer.write_record(Some(i), sequence)?; + let record = SequencingRecordBuilder::default() + .s_seq(sequence) + .flag(i) + .build()?; + writer.push(record)?; println!("Server: Sent record {i}"); // Simulate delay between records diff --git a/examples/parallel_range.rs b/examples/parallel_range.rs index 2d7eacf..0c2f2bf 100644 --- a/examples/parallel_range.rs +++ b/examples/parallel_range.rs @@ -1,7 +1,11 @@ -use binseq::{BinseqReader, BinseqRecord, ParallelProcessor, ParallelReader, Result}; +use std::path::PathBuf; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; +use clap::Parser; + +use binseq::{BinseqReader, BinseqRecord, ParallelProcessor, ParallelReader, Result}; + #[derive(Clone)] struct RangeProcessor { counter: Arc, @@ -63,78 +67,72 @@ impl ParallelProcessor for RangeProcessor { } } -fn main() -> Result<()> { - let args: Vec = std::env::args().collect(); - if args.len() < 2 { - eprintln!( - "Usage: {} [num_threads] [start] [end]", - args[0] - ); - eprintln!("Example: {} data/subset.bq 4 1000 5000", args[0]); - std::process::exit(1); - } +#[derive(Parser)] +struct Cli { + path: PathBuf, + + #[clap(short = 'T', long, default_value_t = 0)] + threads: usize, - let file_path = &args[1]; - let num_threads = args - .get(2) - .unwrap_or(&"4".to_string()) - .parse::() - .map_err(|e| binseq::Error::from(anyhow::Error::from(e)))?; + /// Optional start index for processing range + #[clap(long, default_value_t = 0)] + start: usize, + + /// Optional end index for processing range + #[clap(long, default_value_t = 10_000)] + end: usize, +} + +fn main() -> Result<()> { + let args = Cli::parse(); // Create reader to get total record count - let reader = BinseqReader::new(file_path)?; + let reader = BinseqReader::new(&args.path)?; let total_records = reader.num_records()?; - println!("File: {file_path}"); + println!("File: {}", args.path.display()); println!("Total records in file: {total_records}"); - // Parse range arguments or use defaults - let start = args - .get(3) - .map(|s| s.parse::()) - .transpose() - .map_err(|e| binseq::Error::from(anyhow::Error::from(e)))? - .unwrap_or(0); - let end = args - .get(4) - .map(|s| s.parse::()) - .transpose() - .map_err(|e| binseq::Error::from(anyhow::Error::from(e)))? - .unwrap_or(total_records.min(10_000)); // Default to first 10k records - // Validate range - if start >= total_records { - eprintln!("Error: Start index {start} is >= total records {total_records}"); + if args.start >= total_records { + eprintln!( + "Error: Start index {} is >= total records {total_records}", + args.start + ); std::process::exit(1); } - if end > total_records { + if args.end > total_records { eprintln!( - "Warning: End index {end} is > total records {total_records}, clamping to {total_records}" + "Warning: End index {} is > total records {total_records}, clamping to {total_records}", + args.end ); } - let end = end.min(total_records); + let end = args.end.min(total_records); - if start >= end { - eprintln!("Error: Start index {start} must be < end index {end}"); + if args.start >= end { + eprintln!( + "Error: Start index {} must be < end index {end}", + args.start + ); std::process::exit(1); } println!( "Processing range: {} to {} ({} records)", - start, + args.start, end, - end - start + end - args.start ); - println!("Using {num_threads} threads"); + println!("Using {} threads", args.threads); println!(); // Demonstrate processing the full file println!("=== Processing full file ==="); - let reader_full = BinseqReader::new(file_path)?; + let reader_full = BinseqReader::new(&args.path)?; let processor_full = RangeProcessor::new(0, total_records); let start_time = std::time::Instant::now(); - reader_full.process_parallel(processor_full.clone(), num_threads)?; + reader_full.process_parallel(processor_full.clone(), args.threads)?; let elapsed_full = start_time.elapsed(); println!("Full file processing completed!"); @@ -144,16 +142,16 @@ fn main() -> Result<()> { // Demonstrate processing a specific range println!("=== Processing specific range ==="); - let reader_range = BinseqReader::new(file_path)?; - let processor_range = RangeProcessor::new(start, end); + let reader_range = BinseqReader::new(args.path)?; + let processor_range = RangeProcessor::new(args.start, end); let start_time = std::time::Instant::now(); - reader_range.process_parallel_range(processor_range.clone(), num_threads, start..end)?; + reader_range.process_parallel_range(processor_range.clone(), args.threads, args.start..end)?; let elapsed_range = start_time.elapsed(); println!("Range processing completed!"); println!("Records processed: {}", processor_range.count()); - println!("Expected records: {}", end - start); + println!("Expected records: {}", end - args.start); println!("Time taken: {elapsed_range:.2?}"); // Compare performance diff --git a/examples/streaming.rs b/examples/streaming.rs deleted file mode 100644 index 001ad47..0000000 --- a/examples/streaming.rs +++ /dev/null @@ -1,56 +0,0 @@ -use std::io::{BufReader, Cursor}; - -use binseq::bq::{FileHeaderBuilder, StreamReader, StreamWriterBuilder}; -use binseq::{BinseqRecord, Policy, Result}; - -fn main() -> Result<()> { - // Create a header for sequences of length 100 - let header = FileHeaderBuilder::new().slen(100).build()?; - - // Create some example sequence data - let sequence = b"ACGT".repeat(25); // 100 nucleotides - - // Create a stream writer with a memory buffer as destination - let mut writer = StreamWriterBuilder::default() - .header(header) - .policy(Policy::RandomDraw) // Use random nucleotides for invalid bases - .buffer_capacity(4096) // Use 4K buffer - .build(Cursor::new(Vec::new()))?; - - // Write the sequence with flag 0 - #[allow(deprecated)] - writer.write_record(Some(0), &sequence)?; - - // Write the sequence with flag 1 - #[allow(deprecated)] - writer.write_record(Some(1), &sequence)?; - - // Flush and get the buffer - let buffer = writer.into_inner()?; - let buffer_inner = buffer.into_inner(); - - println!("Wrote {} bytes to buffer", buffer_inner.len()); - - // Now read from the buffer using the streaming reader - let cursor = Cursor::new(buffer_inner); - let buf_reader = BufReader::new(cursor); - - // Create a stream reader - let mut reader = StreamReader::new(buf_reader); - - // Read and display the header - let header = reader.read_header()?; - println!("Read header: sequence length = {}", header.slen); - - // Read records one by one - let mut count = 0; - while let Some(record) = reader.next_record() { - let record = record?; - println!("Record {}: flag = {:?}", count, record.flag()); - count += 1; - } - - println!("Read {count} records"); - - Ok(()) -} diff --git a/examples/write.rs b/examples/write.rs deleted file mode 100644 index f83a273..0000000 --- a/examples/write.rs +++ /dev/null @@ -1,292 +0,0 @@ -use std::{ - io::{BufWriter, Read}, - sync::Arc, -}; - -use anyhow::{Result, bail}; -use binseq::BitSize; -use binseq::{ - SequencingRecordBuilder, - write::{BinseqWriter, BinseqWriterBuilder, Format}, -}; -use clap::Parser; -use paraseq::{ - Record, fastx, - prelude::{IntoProcessError, PairedParallelProcessor, ParallelProcessor, ParallelReader}, -}; -use parking_lot::Mutex; - -type BoxedWriter = Box; - -#[derive(Parser)] -struct Args { - /// Input FASTX to encode into BINSEQ format - #[clap(required = true)] - input: String, - - /// Input FASTX to encode into BINSEQ format (R2) - #[clap(required = false)] - input2: Option, - - /// Output file path for BINSEQ format - #[clap(short = 'o', long)] - output: Option, - - /// Default prefix for writing BINSEQ: `.` - #[clap(short = 'p', long, default_value = "output")] - prefix: String, - - /// Format of the output BINSEQ file - /// - /// [bq: bq|BQ|b, vbq: vbq|VBQ|v, cbq: cbq|CBQ|c] - #[clap(short = 'f', long)] - format: Option, - - /// Exclude quality information in BINSEQ output - /// - /// (bq ignores quality always) - #[clap(short = 'Q', long)] - exclude_quality: bool, - - /// Exclude sequence headers in BINSEQ output - /// - /// (bq ignores headers always) - #[clap(short = 'H', long)] - exclude_headers: bool, - - /// Compression level for BINSEQ output (0: auto) - #[clap(long, default_value_t = 0)] - compression_level: i32, - - /// Default BITSIZE for BINSEQ output (2: 2bit, 4: 4bit) - #[clap(long, default_value_t = 2)] - bitsize: u8, - - /// Default BLOCKSIZE in KB for BINSEQ output (vbq,cbq) - #[clap(long, default_value_t = 128)] - blocksize: usize, - - /// Number of threads to use for parallel processing, 0: all available - #[clap(short = 'T', long, default_value = "0")] - threads: usize, -} -impl Args { - /// Determines the output format based on the file extension or the provided format - fn format(&self) -> Format { - if let Some(format) = self.format { - format - } else if let Some(output) = &self.output { - match output.split('.').next_back() { - Some("bq") => Format::Bq, - Some("vbq") => Format::Vbq, - Some("cbq") => Format::Cbq, - _ => Format::default(), - } - } else { - Format::default() - } - } - fn bitsize(&self) -> BitSize { - match self.bitsize { - 4 => BitSize::Four, - _ => BitSize::Two, - } - } - - /// Creates an output file handle - fn ohandle(&self) -> Result { - let path = if let Some(output) = &self.output { - output.clone() - } else { - format!("{}{}", self.prefix, self.format().extension()) - }; - let ofile = std::fs::File::create(path).map(BufWriter::new)?; - Ok(Box::new(ofile)) - } - - fn is_paired(&self) -> bool { - self.input2.is_some() - } -} - -/// Calculates the sequence length of the first record in the reader -fn get_seq_len(reader: &mut fastx::Reader) -> Result { - let mut rset = reader.new_record_set(); - rset.fill(reader)?; - - let slen = if let Some(record) = rset.iter().next() { - let record = record?; - record.seq().len() - } else { - bail!("No records found in reader"); - }; - - reader.reload(&mut rset)?; - - Ok(slen) -} - -#[derive(Clone)] -struct Encoder { - /// global writer - writer: Arc>>, - thread_writer: BinseqWriter>, -} -impl Encoder { - pub fn new(writer: BinseqWriter) -> Result { - let thread_writer = writer.new_headless_buffer()?; - Ok(Self { - writer: Arc::new(Mutex::new(writer)), - thread_writer, - }) - } - pub fn finish(&mut self) -> Result<()> { - self.writer.lock().finish()?; - Ok(()) - } -} -impl ParallelProcessor for Encoder { - fn process_record(&mut self, record: Rf) -> paraseq::Result<()> { - let seq = record.seq(); - let seq_record = SequencingRecordBuilder::default() - .s_header(record.id()) - .s_seq(&seq) - .opt_s_qual(record.qual()) - .build() - .map_err(IntoProcessError::into_process_error)?; - self.thread_writer - .push(seq_record) - .map_err(IntoProcessError::into_process_error)?; - Ok(()) - } - fn on_batch_complete(&mut self) -> paraseq::Result<()> { - // Drain only the already-compressed completed blocks under the lock. - // The incomplete block keeps accumulating on this thread, and its - // compression happens off-lock when it next fills up in `push`. - self.writer - .lock() - .ingest_completed(&mut self.thread_writer) - .map_err(IntoProcessError::into_process_error)?; - Ok(()) - } - - fn on_thread_complete(&mut self) -> paraseq::Result<()> { - // Flush this thread's residual incomplete block into the global writer. - self.writer - .lock() - .ingest(&mut self.thread_writer) - .map_err(IntoProcessError::into_process_error)?; - Ok(()) - } -} - -impl PairedParallelProcessor for Encoder { - fn process_record_pair(&mut self, record1: Rf, record2: Rf) -> paraseq::Result<()> { - let sseq = record1.seq(); - let xseq = record2.seq(); - let seq_record = SequencingRecordBuilder::default() - .s_header(record1.id()) - .s_seq(&sseq) - .opt_s_qual(record1.qual()) - .x_header(record2.id()) - .x_seq(&xseq) - .opt_x_qual(record2.qual()) - .build() - .map_err(IntoProcessError::into_process_error)?; - - self.thread_writer - .push(seq_record) - .map_err(IntoProcessError::into_process_error)?; - Ok(()) - } - - fn on_batch_complete(&mut self) -> paraseq::Result<()> { - // Drain only the already-compressed completed blocks under the lock. - // The incomplete block keeps accumulating on this thread, and its - // compression happens off-lock when it next fills up in `push`. - self.writer - .lock() - .ingest_completed(&mut self.thread_writer) - .map_err(IntoProcessError::into_process_error)?; - Ok(()) - } - - fn on_thread_complete(&mut self) -> paraseq::Result<()> { - // Flush this thread's residual incomplete block into the global writer. - self.writer - .lock() - .ingest(&mut self.thread_writer) - .map_err(IntoProcessError::into_process_error)?; - Ok(()) - } -} - -fn encode_paired(args: &Args) -> Result<()> { - let mut r1 = fastx::Reader::from_path(&args.input)?; - let mut r2 = fastx::Reader::from_path(args.input2.as_ref().expect("Missing input2"))?; - let ohandle = args.ohandle()?; - - // prepare writer - let writer = { - let format = args.format(); - let mut builder = BinseqWriterBuilder::new(format) - .headers(!args.exclude_headers) - .quality(!args.exclude_quality) - .compression_level(args.compression_level) - .bitsize(args.bitsize()) - .paired(true) - .block_size(args.blocksize * 1024); - - // BQ requires a fixed sequence length from init time - if matches!(format, Format::Bq) { - builder = builder.slen(get_seq_len(&mut r1)? as u32); - builder = builder.xlen(get_seq_len(&mut r2)? as u32); - } - - builder.build(ohandle)? - }; - - let mut encoder = Encoder::new(writer)?; - r1.process_parallel_paired(r2, &mut encoder, args.threads)?; - encoder.finish()?; - - Ok(()) -} - -fn encode_single(args: &Args) -> Result<()> { - let mut reader = fastx::Reader::from_path(&args.input)?; - let ohandle = args.ohandle()?; - - // prepare writer - let writer = { - let format = args.format(); - let mut builder = BinseqWriterBuilder::new(format) - .headers(!args.exclude_headers) - .quality(!args.exclude_quality) - .compression_level(args.compression_level) - .bitsize(args.bitsize()) - .block_size(args.blocksize * 1024); - - // BQ requires a fixed sequence length from init time - if matches!(format, Format::Bq) { - builder = builder.slen(get_seq_len(&mut reader)? as u32); - } - - builder.build(ohandle)? - }; - - let mut encoder = Encoder::new(writer)?; - reader.process_parallel(&mut encoder, args.threads)?; - encoder.finish()?; - - Ok(()) -} - -fn main() -> Result<()> { - let args = Args::parse(); - if args.is_paired() { - encode_paired(&args) - } else { - encode_single(&args) - } -} diff --git a/src/bq/header.rs b/src/bq/header.rs index dd84593..f55d3ab 100644 --- a/src/bq/header.rs +++ b/src/bq/header.rs @@ -5,10 +5,12 @@ //! sequence length, and other information necessary for proper interpretation of the data. use bitnuc_deprec::BitSize; -use byteorder::{ByteOrder, LittleEndian}; use std::io::{Read, Write}; -use crate::error::{BuilderError, HeaderError, Result}; +use crate::{ + error::{HeaderError, Result}, + utils::read_u32_le, +}; /// Current magic number: "BSEQ" in ASCII (in little-endian byte order) /// @@ -86,7 +88,7 @@ impl FileHeaderBuilder { slen: if let Some(slen) = self.slen { slen } else { - return Err(BuilderError::MissingSlen.into()); + return Err(HeaderError::MissingSequenceLength.into()); }, xlen: self.xlen.unwrap_or(0), bits: self.bitsize.unwrap_or_default(), @@ -141,67 +143,6 @@ pub struct FileHeader { pub reserved: [u8; 17], } impl FileHeader { - /// Creates a new header with the specified sequence length - /// - /// This constructor initializes a standard header with the given sequence length, - /// setting the magic number and format version to their default values. - /// The extended sequence length (xlen) is set to 0. - /// - /// # Arguments - /// - /// * `bits` - The number of bits per nucleotide (currently 2 or 4) - /// * `slen` - The length of sequences in the file - /// * `flags` - The flags for the header - /// - /// # Returns - /// - /// A new `FileHeader` instance - #[must_use] - pub fn new(bits: BitSize, slen: u32, flags: bool) -> Self { - Self { - magic: MAGIC, - format: FORMAT, - slen, - xlen: 0, - bits, - flags, - reserved: RESERVED, - } - } - - /// Creates a new header with both primary and extended sequence lengths - /// - /// This constructor initializes a header for files that contain both primary - /// and secondary sequence data, such as quality scores or annotations. - /// - /// # Arguments - /// - /// * `bits` - The number of bits per nucleotide (currently 2 or 4) - /// * `slen` - The length of primary sequences in the file - /// * `xlen` - The length of secondary/extended sequences in the file - /// * `flags` - The flags for the header - /// - /// # Returns - /// - /// A new `FileHeader` instance with extended sequence information - #[must_use] - pub fn new_extended(bits: BitSize, slen: u32, xlen: u32, flags: bool) -> Self { - Self { - magic: MAGIC, - format: FORMAT, - slen, - xlen, - bits, - flags, - reserved: RESERVED, - } - } - - /// Sets the bitsize of the header - pub fn set_bitsize(&mut self, bits: BitSize) { - self.bits = bits; - } - /// Checks if the file is paired #[must_use] pub fn is_paired(&self) -> bool { @@ -229,7 +170,7 @@ impl FileHeader { /// * The format version is unsupported /// * The reserved bytes are invalid pub fn from_bytes(buffer: &[u8; SIZE_HEADER]) -> Result { - let magic = LittleEndian::read_u32(&buffer[0..4]); + let magic = read_u32_le(&buffer[0..4]); if magic != MAGIC { return Err(HeaderError::InvalidMagicNumber(magic).into()); } @@ -237,8 +178,8 @@ impl FileHeader { if format != FORMAT { return Err(HeaderError::InvalidFormatVersion(format).into()); } - let slen = LittleEndian::read_u32(&buffer[5..9]); - let xlen = LittleEndian::read_u32(&buffer[9..13]); + let slen = read_u32_le(&buffer[5..9]); + let xlen = read_u32_le(&buffer[9..13]); let bits = match buffer[13] { 0 | 2 | 42 => BitSize::Two, 4 => BitSize::Four, @@ -307,10 +248,10 @@ impl FileHeader { /// Returns an error if writing to the writer fails (typically an I/O error). pub fn write_bytes(&self, writer: &mut W) -> Result<()> { let mut buffer = [0u8; SIZE_HEADER]; - LittleEndian::write_u32(&mut buffer[0..4], self.magic); + buffer[0..4].copy_from_slice(&self.magic.to_le_bytes()); buffer[4] = self.format; - LittleEndian::write_u32(&mut buffer[5..9], self.slen); - LittleEndian::write_u32(&mut buffer[9..13], self.xlen); + buffer[5..9].copy_from_slice(&self.slen.to_le_bytes()); + buffer[9..13].copy_from_slice(&self.xlen.to_le_bytes()); buffer[13] = self.bits.into(); buffer[14] = self.flags.into(); buffer[15..32].copy_from_slice(&self.reserved); @@ -373,38 +314,11 @@ mod tests { assert!(result.is_err()); } - // ==================== FileHeader Constructor Tests ==================== - - #[test] - fn test_header_new() { - let header = FileHeader::new(BitSize::Two, 100, true); - assert_eq!(header.slen, 100); - assert_eq!(header.xlen, 0); - assert!(header.flags); - assert!(!header.is_paired()); - } - - #[test] - fn test_header_new_extended() { - let header = FileHeader::new_extended(BitSize::Four, 100, 50, false); - assert_eq!(header.slen, 100); - assert_eq!(header.xlen, 50); - assert_eq!(header.bits, BitSize::Four); - assert!(header.is_paired()); - } - - #[test] - fn test_set_bitsize() { - let mut header = FileHeader::new(BitSize::Two, 100, false); - header.set_bitsize(BitSize::Four); - assert_eq!(header.bits, BitSize::Four); - } - // ==================== from_bytes Tests ==================== #[test] fn test_from_bytes_invalid_format_version() { - let header = FileHeader::new(BitSize::Two, 32, false); + let header = FileHeaderBuilder::new().slen(32).build().unwrap(); let mut buffer = [0u8; SIZE_HEADER]; let mut cursor = std::io::Cursor::new(&mut buffer[..]); header.write_bytes(&mut cursor).unwrap(); @@ -415,7 +329,11 @@ mod tests { #[test] fn test_from_bytes_four_bit_size() { - let header = FileHeader::new(BitSize::Four, 32, false); + let header = FileHeaderBuilder::new() + .bitsize(BitSize::Four) + .slen(32) + .build() + .unwrap(); let mut buffer = [0u8; SIZE_HEADER]; let mut cursor = std::io::Cursor::new(&mut buffer[..]); header.write_bytes(&mut cursor).unwrap(); @@ -425,7 +343,7 @@ mod tests { #[test] fn test_from_bytes_invalid_bitsize() { - let header = FileHeader::new(BitSize::Two, 32, false); + let header = FileHeaderBuilder::new().slen(32).build().unwrap(); let mut buffer = [0u8; SIZE_HEADER]; let mut cursor = std::io::Cursor::new(&mut buffer[..]); header.write_bytes(&mut cursor).unwrap(); @@ -452,7 +370,7 @@ mod tests { #[test] fn test_from_buffer_valid() { - let header = FileHeader::new(BitSize::Two, 32, false); + let header = FileHeaderBuilder::new().slen(32).build().unwrap(); let mut buffer = Vec::new(); header.write_bytes(&mut buffer).unwrap(); buffer.extend_from_slice(&[0u8; 16]); // trailing data beyond header @@ -464,7 +382,12 @@ mod tests { #[test] fn test_from_reader_valid() { - let header = FileHeader::new_extended(BitSize::Two, 32, 16, true); + let header = FileHeaderBuilder::new() + .slen(32) + .xlen(16) + .flags(true) + .build() + .unwrap(); let mut buffer = Vec::new(); header.write_bytes(&mut buffer).unwrap(); let mut cursor = std::io::Cursor::new(buffer); diff --git a/src/bq/mod.rs b/src/bq/mod.rs index 47fa622..3811e48 100644 --- a/src/bq/mod.rs +++ b/src/bq/mod.rs @@ -109,18 +109,17 @@ //! //! ``` //! use binseq::{Policy, Result, BinseqRecord, SequencingRecordBuilder}; -//! use binseq::bq::{FileHeaderBuilder, StreamReader, StreamWriterBuilder}; -//! use std::io::{BufReader, Cursor}; +//! use binseq::bq::{FileHeaderBuilder, StreamReader, WriterBuilder}; +//! use std::io::{BufReader, BufWriter, Cursor}; //! //! fn main() -> Result<()> { //! // Create a header for sequences of length 100 //! let header = FileHeaderBuilder::new().slen(100).build()?; //! -//! // Create a stream writer -//! let mut writer = StreamWriterBuilder::default() +//! // Create a buffered writer over any `Write` destination +//! let mut writer = WriterBuilder::default() //! .header(header) -//! .buffer_capacity(8192) -//! .build(Cursor::new(Vec::new()))?; +//! .build(BufWriter::new(Cursor::new(Vec::new())))?; //! //! // Write sequences //! let sequence = b"ACGT".repeat(25); // 100 nucleotides @@ -131,7 +130,7 @@ //! writer.push(record)?; //! //! // Get the inner buffer -//! let buffer = writer.into_inner()?; +//! let buffer = writer.into_inner().into_inner().map_err(std::io::Error::from)?; //! let data = buffer.into_inner(); //! //! // Create a stream reader @@ -243,4 +242,4 @@ mod writer; pub use header::{FILE_MAGIC, FileHeader, FileHeaderBuilder, SIZE_HEADER}; pub use reader::{MmapReader, RefRecord, StreamReader}; -pub use writer::{Encoder, StreamWriter, StreamWriterBuilder, Writer, WriterBuilder}; +pub use writer::{Writer, WriterBuilder}; diff --git a/src/bq/reader.rs b/src/bq/reader.rs index 679b210..30a6b86 100644 --- a/src/bq/reader.rs +++ b/src/bq/reader.rs @@ -61,7 +61,7 @@ impl<'a> RefRecord<'a> { /// /// Panics if the buffer length doesn't match the expected size from the config #[must_use] - pub fn new(id: u64, buffer: &'a [u64], qbuf: &'a [u8], config: RecordConfig) -> Self { + pub(crate) fn new(id: u64, buffer: &'a [u64], qbuf: &'a [u8], config: RecordConfig) -> Self { assert_eq!(buffer.len(), config.record_size_u64()); Self { id, @@ -72,18 +72,6 @@ impl<'a> RefRecord<'a> { header_len: 0, } } - /// Returns the record's configuration - /// - /// The configuration defines the layout and size of the record's components. - #[must_use] - pub fn config(&self) -> RecordConfig { - self.config - } - - pub fn set_id(&mut self, id: &[u8]) { - self.header_len = id.len(); - self.header_buf[..self.header_len].copy_from_slice(id); - } } impl BinseqRecord for RefRecord<'_> { @@ -139,65 +127,39 @@ impl BinseqRecord for RefRecord<'_> { } /// A reference to a record in the map with a precomputed decoded buffer slice -pub struct BatchRecord<'a> { - /// Unprocessed buffer slice (with flags) - buffer: &'a [u64], +pub(crate) struct BatchRecord<'a> { + /// The underlying record view (encoded buffer, config, header) + inner: RefRecord<'a>, /// Decoded buffer slice dbuf: &'a [u8], - /// Record ID - id: u64, - /// The configuration that defines the layout and size of record components - config: RecordConfig, - /// A reusable pre-initialized quality score buffer - qbuf: &'a [u8], - /// Cached index string for the sequence header - header_buf: [u8; 20], - /// Length of the header in bytes - header_len: usize, } impl BinseqRecord for BatchRecord<'_> { fn bitsize(&self) -> BitSize { - self.config.bitsize + self.inner.bitsize() } fn index(&self) -> u64 { - self.id + self.inner.index() } - /// Clear the buffer and fill it with the sequence header fn sheader(&self) -> &[u8] { - &self.header_buf[..self.header_len] + self.inner.sheader() } - - /// Clear the buffer and fill it with the extended header fn xheader(&self) -> &[u8] { - self.sheader() + self.inner.xheader() } - fn flag(&self) -> Option { - if self.config.flags { - Some(self.buffer[0]) - } else { - None - } + self.inner.flag() } fn slen(&self) -> u64 { - self.config.slen + self.inner.slen() } fn xlen(&self) -> u64 { - self.config.xlen + self.inner.xlen() } fn sbuf(&self) -> &[u64] { - if self.config.flags { - &self.buffer[1..=(self.config.schunk as usize)] - } else { - &self.buffer[..(self.config.schunk as usize)] - } + self.inner.sbuf() } fn xbuf(&self) -> &[u64] { - if self.config.flags { - &self.buffer[1 + self.config.schunk as usize..] - } else { - &self.buffer[self.config.schunk as usize..] - } + self.inner.xbuf() } fn decode_s(&self, dbuf: &mut Vec) -> Result<()> { dbuf.extend_from_slice(self.sseq()); @@ -209,31 +171,33 @@ impl BinseqRecord for BatchRecord<'_> { } /// Override this method since we can make use of block information fn sseq(&self) -> &[u8] { - let scalar = self.config.scalar(); + let config = &self.inner.config; + let scalar = config.scalar(); let mut lbound = 0; - let mut rbound = self.config.slen(); - if self.config.flags { + let mut rbound = config.slen(); + if config.flags { lbound += scalar; rbound += scalar; } - &self.dbuf[lbound..rbound] + self.dbuf.get(lbound..rbound).unwrap_or_default() } /// Override this method since we can make use of block information fn xseq(&self) -> &[u8] { - let scalar = self.config.scalar(); - let mut lbound = scalar * self.config.schunk(); - let mut rbound = lbound + self.config.xlen(); - if self.config.flags { + let config = &self.inner.config; + let scalar = config.scalar(); + let mut lbound = scalar * config.schunk(); + let mut rbound = lbound + config.xlen(); + if config.flags { lbound += scalar; rbound += scalar; } - &self.dbuf[lbound..rbound] + self.dbuf.get(lbound..rbound).unwrap_or_default() } fn squal(&self) -> &[u8] { - &self.qbuf[..self.config.slen()] + self.inner.squal() } fn xqual(&self) -> &[u8] { - &self.qbuf[..self.config.xlen()] + self.inner.xqual() } } @@ -244,7 +208,7 @@ impl BinseqRecord for BatchRecord<'_> { /// It handles the translation between sequence lengths in base pairs /// and the number of u64 chunks needed to store the compressed data. #[derive(Clone, Copy)] -pub struct RecordConfig { +pub(crate) struct RecordConfig { /// The primary sequence length in base pairs slen: u64, /// The extended sequence length in base pairs @@ -312,13 +276,6 @@ impl RecordConfig { ) } - /// Returns whether this record contains extended sequence data - /// - /// A record is considered paired if it has a non-zero extended sequence length. - pub fn paired(&self) -> bool { - self.xlen > 0 - } - /// Returns the primary sequence length in base pairs /// /// This method returns the length of the primary sequence in base pairs. @@ -828,7 +785,7 @@ impl StreamReader { /// /// This constant defines how many records each thread processes at a time /// during parallel processing operations. -pub const BATCH_SIZE: usize = 1024; +pub(crate) const BATCH_SIZE: usize = 1024; /// Parallel processing implementation for memory-mapped readers impl ParallelReader for MmapReader { @@ -887,11 +844,7 @@ impl ParallelReader for MmapReader { range: Range, ) -> Result<()> { // Calculate the number of threads to use - let num_threads = if num_threads == 0 { - num_cpus::get() - } else { - num_threads.min(num_cpus::get()) - }; + let num_threads = crate::parallel::clamp_threads(num_threads); // Validate range let num_records = self.num_records(); @@ -972,13 +925,15 @@ impl ParallelReader for MmapReader { // initialize the record let record = BatchRecord { - buffer: &ebuf[ebuf_start..(ebuf_start + rsize_u64)], + inner: RefRecord { + buffer: &ebuf[ebuf_start..(ebuf_start + rsize_u64)], + qbuf: &qbuf, + id: idx as u64, + config: reader.config, + header_buf, + header_len, + }, dbuf: &dbuf[dbuf_start..(dbuf_start + dbuf_rsize)], - qbuf: &qbuf, - id: idx as u64, - config: reader.config, - header_buf, - header_len, }; // process the record diff --git a/src/bq/writer.rs b/src/bq/writer.rs index e457b13..641073f 100644 --- a/src/bq/writer.rs +++ b/src/bq/writer.rs @@ -7,224 +7,22 @@ //! - Efficient buffering and encoding //! - Headless mode for parallel writing -use std::io::{BufWriter, Write}; - -use byteorder::{LittleEndian, WriteBytesExt}; -use rand::{SeedableRng, rngs::SmallRng}; +use std::io::Write; use super::FileHeader; use crate::{ - Policy, RNG_SEED, SequencingRecord, + Policy, SequencingRecord, + encoder::Encoder, error::{Result, WriteError}, }; -/// Writes a single flag value to a writer in little-endian format -/// -/// # Arguments -/// -/// * `writer` - Any type that implements the `Write` trait -/// * `flag` - The 64-bit flag value to write -/// -/// # Returns -/// -/// * `Ok(())` - If the flag was successfully written -/// * `Err(Error)` - If writing to the writer failed -pub fn write_flag(writer: &mut W, flag: u64) -> Result<()> { - writer.write_u64::(flag)?; - Ok(()) -} - /// Writes a buffer of u64 values to a writer in little-endian format -/// -/// This function is used to write encoded sequence data to the output. -/// Each u64 in the buffer contains up to 32 nucleotides in 2-bit format. -/// -/// # Arguments -/// -/// * `writer` - Any type that implements the `Write` trait -/// * `ebuf` - The buffer of u64 values to write -/// -/// # Returns -/// -/// * `Ok(())` - If the buffer was successfully written -/// * `Err(Error)` - If writing to the writer failed -pub fn write_buffer(writer: &mut W, ebuf: &[u64]) -> Result<()> { +fn write_buffer(writer: &mut W, ebuf: &[u64]) -> Result<()> { ebuf.iter() - .try_for_each(|&x| writer.write_u64::(x))?; + .try_for_each(|&x| writer.write_all(&x.to_le_bytes()))?; Ok(()) } -/// Encodes nucleotide sequences into a compact 2-bit binary format -/// -/// The `Encoder` handles the conversion of nucleotide sequences (A, C, G, T) -/// into a compact binary representation where each nucleotide is stored using -/// 2 bits. It also handles invalid nucleotides according to a configurable policy. -/// -/// The encoder maintains internal buffers to avoid repeated allocations during -/// encoding operations. These buffers are reused across multiple encode calls -/// and are cleared automatically when needed. -#[derive(Clone)] -pub struct Encoder { - /// Header containing sequence length and format information - header: FileHeader, - - /// Buffers for storing encoded nucleotides in 2-bit format - /// Each u64 can store 32 nucleotides (64 bits / 2 bits per nucleotide) - sbuffer: Vec, // Primary sequence buffer - xbuffer: Vec, // Extended sequence buffer - - /// Temporary buffers for handling invalid nucleotides - /// These store the processed sequences after policy application - s_ibuf: Vec, // Primary sequence invalid buffer - x_ibuf: Vec, // Extended sequence invalid buffer - - /// Policy for handling invalid nucleotides during encoding - policy: Policy, - - /// Random number generator for the `RandomDraw` policy - /// Seeded with `RNG_SEED` for reproducibility - rng: SmallRng, -} -impl Encoder { - /// Creates a new encoder with default invalid nucleotide policy - /// - /// # Arguments - /// - /// * `header` - The header defining sequence lengths and format - /// - /// # Examples - /// - /// ``` - /// # use binseq::bq::{FileHeaderBuilder, Encoder}; - /// let header = FileHeaderBuilder::new().slen(100).build().unwrap(); - /// let encoder = Encoder::new(header); - /// ``` - #[must_use] - pub fn new(header: FileHeader) -> Self { - Self::with_policy(header, Policy::default()) - } - - /// Creates a new encoder with a specific invalid nucleotide policy - /// - /// # Arguments - /// - /// * `header` - The header defining sequence lengths and format - /// * `policy` - The policy for handling invalid nucleotides - /// - /// # Examples - /// - /// ``` - /// # use binseq::bq::{FileHeaderBuilder, Encoder}; - /// # use binseq::Policy; - /// let header = FileHeaderBuilder::new().slen(100).build().unwrap(); - /// let encoder = Encoder::with_policy(header, Policy::SetToA); - /// ``` - #[must_use] - pub fn with_policy(header: FileHeader, policy: Policy) -> Self { - Self { - header, - policy, - sbuffer: Vec::default(), - xbuffer: Vec::default(), - s_ibuf: Vec::default(), - x_ibuf: Vec::default(), - rng: SmallRng::seed_from_u64(RNG_SEED), - } - } - - /// Returns whether the header is paired-end. - #[must_use] - pub fn is_paired(&self) -> bool { - self.header.is_paired() - } - - /// Encodes a single sequence as 2-bit. - /// - /// Will return `None` if the sequence is invalid and the policy does not allow correction. - pub fn encode_single(&mut self, primary: &[u8]) -> Result> { - if primary.len() != self.header.slen as usize { - return Err(WriteError::UnexpectedSequenceLength { - expected: self.header.slen, - got: primary.len(), - } - .into()); - } - - // Fill the buffer with the 2-bit representation of the nucleotides - self.clear(); - if self.header.bits.encode(primary, &mut self.sbuffer).is_err() { - self.clear(); - if self - .policy - .handle(primary, &mut self.s_ibuf, &mut self.rng)? - { - self.header.bits.encode(&self.s_ibuf, &mut self.sbuffer)?; - } else { - return Ok(None); - } - } - - Ok(Some(&self.sbuffer)) - } - - /// Encodes a pair of sequences as 2-bit. - /// - /// Will return `None` if either sequence is invalid and the policy does not allow correction. - pub fn encode_paired( - &mut self, - primary: &[u8], - extended: &[u8], - ) -> Result> { - if primary.len() != self.header.slen as usize { - return Err(WriteError::UnexpectedSequenceLength { - expected: self.header.slen, - got: primary.len(), - } - .into()); - } - if extended.len() != self.header.xlen as usize { - return Err(WriteError::UnexpectedSequenceLength { - expected: self.header.xlen, - got: extended.len(), - } - .into()); - } - - self.clear(); - if self.header.bits.encode(primary, &mut self.sbuffer).is_err() - || self - .header - .bits - .encode(extended, &mut self.xbuffer) - .is_err() - { - self.clear(); - if self - .policy - .handle(primary, &mut self.s_ibuf, &mut self.rng)? - && self - .policy - .handle(extended, &mut self.x_ibuf, &mut self.rng)? - { - self.header.bits.encode(&self.s_ibuf, &mut self.sbuffer)?; - self.header.bits.encode(&self.x_ibuf, &mut self.xbuffer)?; - } else { - return Ok(None); - } - } - - Ok(Some((&self.sbuffer, &self.xbuffer))) - } - - /// Clear all buffers and reset the encoder. - pub fn clear(&mut self) { - self.sbuffer.clear(); - self.xbuffer.clear(); - self.s_ibuf.clear(); - self.x_ibuf.clear(); - } -} - /// Builder for creating configured `Writer` instances /// /// This builder provides a flexible way to create writers with various @@ -305,6 +103,9 @@ pub struct Writer { /// The underlying writer for output inner: W, + /// Header defining sequence lengths and format + header: FileHeader, + /// Encoder for converting sequences to binary format encoder: Encoder, @@ -352,85 +153,25 @@ impl Writer { } Ok(Self { inner, - encoder: Encoder::with_policy(header, policy), + header, + encoder: Encoder::with_policy(header.bits, policy), headless, }) } /// Returns whether the header is paired-end. pub fn is_paired(&self) -> bool { - self.encoder.is_paired() + self.header.is_paired() } /// Returns the header of the writer pub fn header(&self) -> FileHeader { - self.encoder.header + self.header } /// Returns the N-policy of the writer pub fn policy(&self) -> Policy { - self.encoder.policy - } - - /// Writes a single record to the output - /// - /// This method encodes and writes a primary sequence along with an associated flag. - /// - /// # Arguments - /// - /// * `flag` - A 64-bit flag value associated with the sequence - /// * `primary` - The nucleotide sequence to write - /// - /// # Returns - /// - /// * `Ok(true)` if the record was written successfully - /// * `Ok(false)` if the record was not written because it was empty - /// * `Err(WriteError::FlagSet)` if the flag is set but no flag value is provided - #[deprecated] - pub fn write_record(&mut self, flag: Option, primary: &[u8]) -> Result { - let has_flag = self.encoder.header.flags; - if let Some(sbuffer) = self.encoder.encode_single(primary)? { - if has_flag { - write_flag(&mut self.inner, flag.unwrap_or(0))?; - } - write_buffer(&mut self.inner, sbuffer)?; - Ok(true) - } else { - Ok(false) - } - } - - /// Writes a paired record to the output - /// - /// This method writes a paired record to the output. It takes a flag, primary sequence, and extended sequence as input. - /// If the flag is set but no flag value is provided, it returns an error. - /// Otherwise, it writes the encoded single and extended sequences to the output and returns true. - /// - /// # Arguments - /// * `flag` - The flag value to write to the output - /// * `primary` - The primary sequence to encode and write to the output - /// * `extended` - The extended sequence to encode and write to the output - /// - /// # Returns - /// * `Result` - A result indicating whether the write was successful or not - #[deprecated] - pub fn write_paired_record( - &mut self, - flag: Option, - primary: &[u8], - extended: &[u8], - ) -> Result { - let has_flag = self.encoder.header.flags; - if let Some((sbuffer, xbuffer)) = self.encoder.encode_paired(primary, extended)? { - if has_flag { - write_flag(&mut self.inner, flag.unwrap_or(0))?; - } - write_buffer(&mut self.inner, sbuffer)?; - write_buffer(&mut self.inner, xbuffer)?; - Ok(true) - } else { - Ok(false) - } + self.encoder.policy() } /// Writes a record using the unified [`SequencingRecord`] API @@ -470,23 +211,43 @@ impl Writer { /// # } /// ``` pub fn push(&mut self, record: SequencingRecord) -> Result { - let has_flag = self.encoder.header.flags; + let has_flag = self.header.flags; if has_flag { - write_flag(&mut self.inner, record.flag().unwrap_or(0))?; + self.inner + .write_all(&record.flag.unwrap_or(0).to_le_bytes())?; } // Check paired status - writer can require paired (record must have R2), // but if writer is single-end, we simply ignore any R2 data in the record. - if self.encoder.header.is_paired() && !record.is_paired() { + if self.header.is_paired() && !record.is_paired() { return Err(WriteError::ConfigurationMismatch { attribute: "paired", - expected: self.encoder.header.is_paired(), + expected: self.header.is_paired(), actual: record.is_paired(), } .into()); } - if self.encoder.header.is_paired() { + // BQ records are fixed-length: validate against the header + if record.s_seq.len() != self.header.slen as usize { + return Err(WriteError::UnexpectedSequenceLength { + expected: self.header.slen, + got: record.s_seq.len(), + } + .into()); + } + if self.header.is_paired() { + let xlen = record.x_seq.unwrap_or_default().len(); + if xlen != self.header.xlen as usize { + return Err(WriteError::UnexpectedSequenceLength { + expected: self.header.xlen, + got: xlen, + } + .into()); + } + } + + if self.header.is_paired() { if let Some((sbuffer, xbuffer)) = self .encoder .encode_paired(record.s_seq, record.x_seq.unwrap_or_default())? @@ -549,21 +310,6 @@ impl Writer { Ok(()) } - /// Creates a new encoder with the same configuration as this writer - /// - /// This is useful when you need a separate encoder instance for parallel - /// processing or other scenarios where you need independent encoding. - /// The new encoder is initialized with a cleared state. - /// - /// # Returns - /// - /// A new `Encoder` instance with the same configuration but cleared buffers - pub fn new_encoder(&self) -> Encoder { - let mut encoder = self.encoder.clone(); - encoder.clear(); - encoder - } - /// Checks if this writer is in headless mode /// /// In headless mode, the writer does not write the header to the output. @@ -599,195 +345,6 @@ impl Writer { } } -/// A streaming writer for binary sequence data -/// -/// This writer buffers data before writing it to the underlying writer, -/// providing efficient streaming capabilities suitable for: -/// - Writing to network connections -/// - Processing very large datasets -/// - Pipeline processing -/// -/// The `StreamWriter` is a specialized version of `Writer` that -/// adds internal buffering and is optimized for streaming scenarios. -pub struct StreamWriter { - /// The underlying writer for processing sequences - writer: Writer>, -} - -impl StreamWriter { - /// Creates a new `StreamWriter` with the default buffer size - /// - /// This constructor initializes a `StreamWriter` with an 8K buffer - /// for efficient writing to the underlying writer. - /// - /// # Arguments - /// - /// * `inner` - The writer to write binary sequence data to - /// * `header` - The header defining sequence lengths and format - /// * `policy` - The policy for handling invalid nucleotides - /// * `headless` - Whether to skip writing the header - /// - /// # Returns - /// - /// * `Ok(StreamWriter)` - A new streaming writer - /// * `Err(Error)` - If initialization fails - pub fn new(inner: W, header: FileHeader, policy: Policy, headless: bool) -> Result { - Self::with_capacity(inner, 8192, header, policy, headless) - } - - /// Creates a new `StreamWriter` with a specified buffer capacity - /// - /// This constructor allows customizing the buffer size based on - /// expected usage patterns and performance requirements. - /// - /// # Arguments - /// - /// * `inner` - The writer to write binary sequence data to - /// * `capacity` - The size of the internal buffer in bytes - /// * `header` - The header defining sequence lengths and format - /// * `policy` - The policy for handling invalid nucleotides - /// * `headless` - Whether to skip writing the header - /// - /// # Returns - /// - /// * `Ok(StreamWriter)` - A new streaming writer with the specified buffer capacity - /// * `Err(Error)` - If initialization fails - pub fn with_capacity( - inner: W, - capacity: usize, - header: FileHeader, - policy: Policy, - headless: bool, - ) -> Result { - let buffered = BufWriter::with_capacity(capacity, inner); - let writer = Writer::new(buffered, header, policy, headless)?; - - Ok(Self { writer }) - } - - #[deprecated(note = "use `push` method with SequencingRecord instead")] - pub fn write_record(&mut self, flag: Option, primary: &[u8]) -> Result { - #[allow(deprecated)] - self.writer.write_record(flag, primary) - } - - #[deprecated(note = "use `push` method with SequencingRecord instead")] - pub fn write_paired_record( - &mut self, - flag: Option, - primary: &[u8], - extended: &[u8], - ) -> Result { - #[allow(deprecated)] - self.writer.write_paired_record(flag, primary, extended) - } - - /// Writes a record using the unified [`SequencingRecord`] API - pub fn push(&mut self, record: SequencingRecord) -> Result { - self.writer.push(record) - } - - /// Flushes any buffered data to the underlying writer - /// - /// # Returns - /// - /// * `Ok(())` - If the flush was successful - /// * `Err(Error)` - If flushing failed - pub fn flush(&mut self) -> Result<()> { - self.writer.flush() - } - - /// Consumes the streaming writer and returns the inner writer after flushing - /// - /// This method is useful when you need access to the underlying writer - /// after all writing is complete. - /// - /// # Returns - /// - /// * `Ok(W)` - The inner writer after flushing all data - /// * `Err(Error)` - If flushing failed - pub fn into_inner(self) -> Result { - // First unwrap the writer inner (BufWriter) - let bufw = self.writer.into_inner(); - // Now unwrap the BufWriter to get W - match bufw.into_inner() { - Ok(inner) => Ok(inner), - Err(e) => Err(std::io::Error::from(e).into()), - } - } -} - -/// Builder for `StreamWriter` instances -/// -/// This builder provides a convenient way to create and configure `StreamWriter` -/// instances with custom buffer sizes and other settings. -#[derive(Default)] -pub struct StreamWriterBuilder { - /// Required header defining sequence lengths and format - header: Option, - /// Optional policy for handling invalid nucleotides - policy: Option, - /// Optional headless mode for parallel writing scenarios - headless: Option, - /// Optional buffer capacity setting - buffer_capacity: Option, -} - -impl StreamWriterBuilder { - /// Sets the header for the writer - #[must_use] - pub fn header(mut self, header: FileHeader) -> Self { - self.header = Some(header); - self - } - - /// Sets the policy for handling invalid nucleotides - #[must_use] - pub fn policy(mut self, policy: Policy) -> Self { - self.policy = Some(policy); - self - } - - /// Sets headless mode (whether to skip writing the header) - #[must_use] - pub fn headless(mut self, headless: bool) -> Self { - self.headless = Some(headless); - self - } - - /// Sets the buffer capacity for the writer - #[must_use] - pub fn buffer_capacity(mut self, capacity: usize) -> Self { - self.buffer_capacity = Some(capacity); - self - } - - /// Builds a `StreamWriter` with the configured settings - /// - /// # Arguments - /// - /// * `inner` - The writer to write binary sequence data to - /// - /// # Returns - /// - /// * `Ok(StreamWriter)` - A new streaming writer with the specified configuration - /// * `Err(Error)` - If building the writer fails - pub fn build(self, inner: W) -> Result> { - let Some(header) = self.header else { - return Err(WriteError::MissingHeader.into()); - }; - - let capacity = self.buffer_capacity.unwrap_or(8192); - StreamWriter::with_capacity( - inner, - capacity, - header, - self.policy.unwrap_or_default(), - self.headless.unwrap_or(false), - ) - } -} - #[cfg(test)] mod testing { @@ -848,132 +405,37 @@ mod testing { Ok(()) } - #[test] - fn test_stream_writer() -> Result<()> { - let inner = Vec::new(); - let writer = StreamWriterBuilder::default() - .header(FileHeaderBuilder::new().slen(32).build()?) - .buffer_capacity(16384) - .build(inner)?; - - // Convert back to Vec to verify it works - let inner = writer.into_inner()?; - assert_eq!(inner.len(), SIZE_HEADER); - Ok(()) - } - - // ==================== Encoder Tests ==================== - - #[test] - fn test_encoder_new() { - let header = FileHeaderBuilder::new().slen(8).build().unwrap(); - let encoder = Encoder::new(header); - assert!(matches!(encoder.policy, Policy::IgnoreSequence)); - } - - #[test] - fn test_encoder_encode_single_wrong_length() { - let header = FileHeaderBuilder::new().slen(8).build().unwrap(); - let mut encoder = Encoder::new(header); - let result = encoder.encode_single(b"ACGT"); - assert!(result.is_err()); - } + // ==================== Length Validation Tests ==================== #[test] - fn test_encoder_encode_single_invalid_ignored() { - let header = FileHeaderBuilder::new().slen(8).build().unwrap(); - let mut encoder = Encoder::with_policy(header, Policy::IgnoreSequence); - let result = encoder.encode_single(b"ACGTNNNN").unwrap(); - assert!(result.is_none()); - } - - #[test] - fn test_encoder_encode_single_invalid_corrected() { - let header = FileHeaderBuilder::new().slen(8).build().unwrap(); - let mut encoder = Encoder::with_policy(header, Policy::SetToA); - let result = encoder.encode_single(b"ACGTNNNN").unwrap(); - assert!(result.is_some()); - } - - #[test] - fn test_encoder_encode_paired_wrong_primary_length() { - let header = FileHeaderBuilder::new().slen(8).xlen(8).build().unwrap(); - let mut encoder = Encoder::new(header); - let result = encoder.encode_paired(b"ACGT", b"ACGTACGT"); - assert!(result.is_err()); - } - - #[test] - fn test_encoder_encode_paired_wrong_extended_length() { - let header = FileHeaderBuilder::new().slen(8).xlen(8).build().unwrap(); - let mut encoder = Encoder::new(header); - let result = encoder.encode_paired(b"ACGTACGT", b"ACGT"); - assert!(result.is_err()); - } - - #[test] - fn test_encoder_encode_paired_invalid_ignored() { - let header = FileHeaderBuilder::new().slen(8).xlen(8).build().unwrap(); - let mut encoder = Encoder::with_policy(header, Policy::IgnoreSequence); - let result = encoder.encode_paired(b"ACGTNNNN", b"ACGTACGT").unwrap(); - assert!(result.is_none()); - } - - #[test] - fn test_encoder_encode_paired_invalid_corrected() { - let header = FileHeaderBuilder::new().slen(8).xlen(8).build().unwrap(); - let mut encoder = Encoder::with_policy(header, Policy::SetToA); - let result = encoder.encode_paired(b"ACGTNNNN", b"NNNNACGT").unwrap(); - assert!(result.is_some()); - } - - // ==================== WriterBuilder Tests ==================== - - #[test] - fn test_writer_builder_missing_header() { - let result = WriterBuilder::default().build(Vec::new()); - assert!(result.is_err()); - } - - // ==================== Deprecated Writer Methods ==================== - - #[test] - #[allow(deprecated)] - fn test_write_record_deprecated() -> Result<()> { + fn test_push_wrong_length() -> Result<()> { let mut writer = WriterBuilder::default() .header(FileHeaderBuilder::new().slen(8).build()?) .build(Vec::new())?; - let wrote = writer.write_record(None, b"ACGTACGT")?; - assert!(wrote); + let record = SequencingRecordBuilder::default().s_seq(b"ACGT").build()?; + assert!(writer.push(record).is_err()); Ok(()) } #[test] - #[allow(deprecated)] - fn test_write_record_deprecated_skipped() -> Result<()> { + fn test_push_wrong_extended_length() -> Result<()> { let mut writer = WriterBuilder::default() - .header(FileHeaderBuilder::new().slen(8).build()?) + .header(FileHeaderBuilder::new().slen(8).xlen(8).build()?) .build(Vec::new())?; - let wrote = writer.write_record(None, b"NNNNNNNN")?; - assert!(!wrote); + let record = SequencingRecordBuilder::default() + .s_seq(b"ACGTACGT") + .x_seq(b"ACGT") + .build()?; + assert!(writer.push(record).is_err()); Ok(()) } + // ==================== WriterBuilder Tests ==================== + #[test] - #[allow(deprecated)] - fn test_write_paired_record_deprecated() -> Result<()> { - let mut writer = WriterBuilder::default() - .header( - FileHeaderBuilder::new() - .slen(8) - .xlen(8) - .flags(true) - .build()?, - ) - .build(Vec::new())?; - let wrote = writer.write_paired_record(Some(5), b"ACGTACGT", b"TTGGCCAA")?; - assert!(wrote); - Ok(()) + fn test_writer_builder_missing_header() { + let result = WriterBuilder::default().build(Vec::new()); + assert!(result.is_err()); } // ==================== push() Tests ==================== @@ -1057,16 +519,6 @@ mod testing { // ==================== Writer Misc Tests ==================== - #[test] - fn test_new_encoder() -> Result<()> { - let writer = WriterBuilder::default() - .header(FileHeaderBuilder::new().slen(8).build()?) - .build(Vec::new())?; - let encoder = writer.new_encoder(); - assert!(encoder.sbuffer.is_empty()); - Ok(()) - } - #[test] fn test_writer_flush() -> Result<()> { let mut writer = WriterBuilder::default() @@ -1108,73 +560,4 @@ mod testing { assert!(matches!(writer.policy(), Policy::SetToA)); Ok(()) } - - // ==================== StreamWriter Tests ==================== - - #[test] - fn test_stream_writer_new() -> Result<()> { - let writer = StreamWriter::new( - Vec::new(), - FileHeaderBuilder::new().slen(8).build()?, - Policy::default(), - false, - )?; - let inner = writer.into_inner()?; - assert_eq!(inner.len(), SIZE_HEADER); - Ok(()) - } - - #[test] - #[allow(deprecated)] - fn test_stream_writer_deprecated_methods() -> Result<()> { - let mut writer = StreamWriter::new( - Vec::new(), - FileHeaderBuilder::new().slen(8).xlen(8).build()?, - Policy::default(), - false, - )?; - assert!(writer.write_record(None, b"ACGTACGT")?); - assert!(writer.write_paired_record(None, b"ACGTACGT", b"TTGGCCAA")?); - writer.flush()?; - Ok(()) - } - - #[test] - fn test_stream_writer_push() -> Result<()> { - let mut writer = StreamWriter::new( - Vec::new(), - FileHeaderBuilder::new().slen(8).build()?, - Policy::default(), - false, - )?; - let record = SequencingRecordBuilder::default() - .s_seq(b"ACGTACGT") - .build()?; - assert!(writer.push(record)?); - writer.flush()?; - let inner = writer.into_inner()?; - assert_eq!(inner.len(), SIZE_HEADER + 8); - Ok(()) - } - - // ==================== StreamWriterBuilder Tests ==================== - - #[test] - fn test_stream_writer_builder_missing_header() { - let result = StreamWriterBuilder::default().build(Vec::new()); - assert!(result.is_err()); - } - - #[test] - fn test_stream_writer_builder_with_policy_and_headless() -> Result<()> { - let inner = Vec::new(); - let writer = StreamWriterBuilder::default() - .header(FileHeaderBuilder::new().slen(8).build()?) - .policy(Policy::SetToA) - .headless(true) - .build(inner)?; - let inner = writer.into_inner()?; - assert!(inner.is_empty()); - Ok(()) - } } diff --git a/src/cbq/core/block.rs b/src/cbq/core/block.rs index 1045fd7..d4768fd 100644 --- a/src/cbq/core/block.rs +++ b/src/cbq/core/block.rs @@ -10,7 +10,7 @@ use crate::cbq::core::utils::sized_compress; use crate::error::{CbqError, WriteError}; use crate::{BinseqRecord, BitSize, DEFAULT_QUALITY_SCORE, Result}; -use super::utils::{Span, calculate_offsets, extension_read, resize_uninit, slice_and_increment}; +use super::utils::{calculate_offsets, extension_read, resize_uninit, slice_and_increment, span}; use super::{BlockHeader, BlockRange, FileHeader}; use crate::SequencingRecord; @@ -70,8 +70,6 @@ pub struct ColumnarBlock { /// Total nucleotides in this block pub(crate) nuclen: usize, - /// Number of npos positions - pub(crate) num_npos: usize, /// Current size of this block (virtual) current_size: usize, @@ -113,7 +111,6 @@ impl ColumnarBlock { self.num_sequences = 0; self.num_records = 0; self.current_size = 0; - self.num_npos = 0; self.len_nef = 0; } @@ -220,7 +217,7 @@ impl ColumnarBlock { /// Note: this does not check if quality scores are different lengths from sequence fn add_quality(&mut self, record: &SequencingRecord) -> Result<()> { if self.header.has_qualities() { - let Some(squal) = record.s_qual() else { + let Some(squal) = record.s_qual else { return Err(WriteError::ConfigurationMismatch { attribute: "s_qual", expected: true, @@ -231,7 +228,7 @@ impl ColumnarBlock { self.qual.extend_from_slice(squal); if self.header.is_paired() { - let Some(xqual) = record.x_qual() else { + let Some(xqual) = record.x_qual else { return Err(WriteError::ConfigurationMismatch { attribute: "x_qual", expected: true, @@ -369,15 +366,19 @@ impl ColumnarBlock { /// Find all positions of 'N' in the sequence fn fill_npos(&mut self) -> Result<()> { bitnuc::ambiguous_bases(&self.seq, &mut self.npos); - self.num_npos = self.npos.len(); // build Elias-Fano encoding for N positions if self.npos.is_empty() { self.ef = None; Ok(()) } else { - let mut ef_builder = EliasFanoBuilder::new(self.seq.len(), self.npos.len())?; - ef_builder.extend(self.npos.iter().map(|idx| *idx as usize))?; + #[allow(clippy::redundant_closure_for_method_calls)] + let mut ef_builder = EliasFanoBuilder::new(self.seq.len(), self.npos.len()) + .map_err(|e| e.into_boxed_dyn_error())?; + #[allow(clippy::redundant_closure_for_method_calls)] + ef_builder + .extend(self.npos.iter().map(|idx| *idx as usize)) + .map_err(|e| e.into_boxed_dyn_error())?; let ef = ef_builder.build(); self.ef = Some(ef); @@ -408,7 +409,9 @@ impl ColumnarBlock { // compress N-positions (Elias-Fano encoded) if let Some(ef) = self.ef.as_ref() { - ef.serialize_into(&mut self.ef_bytes)?; + #[allow(clippy::redundant_closure_for_method_calls)] + ef.serialize_into(&mut self.ef_bytes) + .map_err(|e| e.into_boxed_dyn_error())?; self.len_nef = self.ef_bytes.len(); sized_compress(&mut self.z_npos, &self.ef_bytes, cctx)?; } @@ -461,8 +464,9 @@ impl ColumnarBlock { self.ef_bytes.reserve(self.len_nef); copy_decode(self.z_npos.as_slice(), &mut self.ef_bytes)?; - let ef = EliasFano::deserialize_from(self.ef_bytes.as_slice())?; - self.num_npos = ef.len(); + #[allow(clippy::redundant_closure_for_method_calls)] + let ef = EliasFano::deserialize_from(self.ef_bytes.as_slice()) + .map_err(|e| e.into_boxed_dyn_error())?; self.ef = Some(ef); } @@ -629,8 +633,9 @@ impl ColumnarBlock { .map_err(|e| io::Error::other(zstd_safe::get_error_name(e)))?; // reinitialize the EliasFano encoding - let ef = EliasFano::deserialize_from(self.ef_bytes.as_slice())?; - self.num_npos = ef.len(); + #[allow(clippy::redundant_closure_for_method_calls)] + let ef = EliasFano::deserialize_from(self.ef_bytes.as_slice()) + .map_err(|e| e.into_boxed_dyn_error())?; self.ef = Some(ef); } @@ -773,10 +778,9 @@ impl<'a> Iterator for RefRecordIter<'a> { self.index }; - let sseq_span = - Span::new_u64(self.block.l_seq_offsets[seq_idx], self.block.l_seq[seq_idx]); + let sseq_span = span(self.block.l_seq_offsets[seq_idx], self.block.l_seq[seq_idx]); let sheader_span = if self.has_headers { - Some(Span::new_u64( + Some(span( self.block.l_header_offsets[seq_idx], self.block.l_headers[seq_idx], )) @@ -784,7 +788,7 @@ impl<'a> Iterator for RefRecordIter<'a> { None }; let xseq_span = if self.is_paired { - Some(Span::new_u64( + Some(span( self.block.l_seq_offsets[seq_idx + 1], self.block.l_seq[seq_idx + 1], )) @@ -792,7 +796,7 @@ impl<'a> Iterator for RefRecordIter<'a> { None }; let xheader_span = if self.is_paired && self.has_headers { - Some(Span::new_u64( + Some(span( self.block.l_header_offsets[seq_idx + 1], self.block.l_headers[seq_idx + 1], )) @@ -847,7 +851,7 @@ impl RefRecordIndex { } /// A reference to a record in a [`ColumnarBlock`](crate::cbq::ColumnarBlock) that implements the [`BinseqRecord`](crate::BinseqRecord) trait -#[derive(Clone, Copy)] +#[derive(Clone)] pub struct RefRecord<'a> { /// A reference to the block containing this record block: &'a ColumnarBlock, @@ -862,16 +866,16 @@ pub struct RefRecord<'a> { global_index: usize, /// Span of the primary sequence within the block - sseq_span: Span, + sseq_span: std::ops::Range, /// Span of the extended sequence within the block - xseq_span: Option, + xseq_span: Option>, /// Span of the primary header within the block - sheader_span: Option, + sheader_span: Option>, /// Span of the extended header within the block - xheader_span: Option, + xheader_span: Option>, /// A buffer to the name of this record when not storing headers rr_index: RefRecordIndex, @@ -894,16 +898,16 @@ impl BinseqRecord for RefRecord<'_> { } fn sheader(&self) -> &[u8] { - if let Some(span) = self.sheader_span { - &self.block.headers[span.range()] + if let Some(span) = &self.sheader_span { + &self.block.headers[span.clone()] } else { self.rr_index.as_bytes() } } fn xheader(&self) -> &[u8] { - if let Some(span) = self.xheader_span { - &self.block.headers[span.range()] + if let Some(span) = &self.xheader_span { + &self.block.headers[span.clone()] } else { self.rr_index.as_bytes() } @@ -922,7 +926,7 @@ impl BinseqRecord for RefRecord<'_> { } fn xlen(&self) -> u64 { - self.xseq_span.map_or(0, |span| span.len() as u64) + self.xseq_span.as_ref().map_or(0, |span| span.len() as u64) } fn decode_s(&self, buf: &mut Vec) -> crate::Result<()> { @@ -936,12 +940,17 @@ impl BinseqRecord for RefRecord<'_> { } fn sseq(&self) -> &[u8] { - &self.block.seq[self.sseq_span.range()] + self.block + .seq + .get(self.sseq_span.clone()) + .unwrap_or_default() } fn xseq(&self) -> &[u8] { self.xseq_span - .map_or(&[], |span| &self.block.seq[span.range()]) + .as_ref() + .and_then(|span| self.block.seq.get(span.clone())) + .unwrap_or_default() } fn has_quality(&self) -> bool { @@ -950,7 +959,7 @@ impl BinseqRecord for RefRecord<'_> { fn squal(&self) -> &[u8] { if self.has_quality() { - &self.block.qual[self.sseq_span.range()] + &self.block.qual[self.sseq_span.clone()] } else { &self.qbuf[..self.slen() as usize] } @@ -958,9 +967,9 @@ impl BinseqRecord for RefRecord<'_> { fn xqual(&self) -> &[u8] { if self.has_quality() - && let Some(span) = self.xseq_span + && let Some(span) = &self.xseq_span { - &self.block.qual[span.range()] + &self.block.qual[span.clone()] } else { &self.qbuf[..self.xlen() as usize] } diff --git a/src/cbq/core/block_header.rs b/src/cbq/core/block_header.rs index a63e9e7..3f86895 100644 --- a/src/cbq/core/block_header.rs +++ b/src/cbq/core/block_header.rs @@ -59,7 +59,6 @@ impl BlockHeader { } /// Calculate the length of the block in bytes. - #[allow(dead_code)] #[must_use] pub fn block_len(&self) -> usize { (self.len_z_seq_len diff --git a/src/cbq/core/header.rs b/src/cbq/core/header.rs index 287d197..5860a58 100644 --- a/src/cbq/core/header.rs +++ b/src/cbq/core/header.rs @@ -139,11 +139,6 @@ pub struct FileHeaderBuilder { } impl FileHeaderBuilder { - pub fn with_compression_level(&mut self, compression_level: usize) -> &mut Self { - self.compression_level = Some(compression_level); - self - } - pub fn with_optional_compression_level( &mut self, compression_level: Option, @@ -152,6 +147,10 @@ impl FileHeaderBuilder { self } + pub fn with_compression_level(&mut self, compression_level: usize) -> &mut Self { + self.with_optional_compression_level(Some(compression_level)) + } + pub fn with_block_size(&mut self, block_size: usize) -> &mut Self { self.block_size = Some(block_size); self diff --git a/src/cbq/core/index.rs b/src/cbq/core/index.rs index e61657f..29ed26b 100644 --- a/src/cbq/core/index.rs +++ b/src/cbq/core/index.rs @@ -150,31 +150,25 @@ impl Index { self.ranges.len() } - #[must_use] - pub fn iter_blocks(&self) -> BlockIter<'_> { - BlockIter { - index: self, - pos: 0, - } + pub fn iter_blocks(&self) -> impl Iterator + '_ { + self.ranges.iter().copied() } + /// Returns the average block size in bytes (0.0 for indexes with fewer than two blocks) #[must_use] pub fn average_block_size(&self) -> f64 { - let mut block_iter = self.iter_blocks(); - let Some(mut last_block) = block_iter.next() else { + if self.ranges.len() < 2 { return 0.0; - }; - let mut total_size = 0.0; - let mut count = 0; - for block in block_iter { - let last_block_size = block.offset - last_block.offset; - total_size += last_block_size as f64; - count += 1; - last_block = block; } - total_size / f64::from(count) + let total: u64 = self + .ranges + .windows(2) + .map(|pair| pair[1].offset - pair[0].offset) + .sum(); + total as f64 / (self.ranges.len() - 1) as f64 } + /// Prints a debug summary of each block range to stdout pub fn pprint(&self) { for block in self.iter_blocks() { println!("{block:?}"); @@ -182,24 +176,6 @@ impl Index { } } -pub struct BlockIter<'a> { - index: &'a Index, - pos: usize, -} -impl Iterator for BlockIter<'_> { - type Item = BlockRange; - - fn next(&mut self) -> Option { - if self.pos >= self.index.num_blocks() { - None - } else { - let block = self.index.ranges[self.pos]; - self.pos += 1; - Some(block) - } - } -} - /// A struct representing a block range in a CBQ file and stored in the [`Index`](crate::cbq::Index) /// /// This is stored identically in memory and on disk. diff --git a/src/cbq/core/utils.rs b/src/cbq/core/utils.rs index eb7503b..16fbc44 100644 --- a/src/cbq/core/utils.rs +++ b/src/cbq/core/utils.rs @@ -80,25 +80,7 @@ pub(crate) fn calculate_offsets(values: &[u64], offsets: &mut Vec) { } } -#[derive(Clone, Copy, Debug)] -pub struct Span { - offset: usize, - length: usize, -} -impl Span { - pub fn new(offset: usize, length: usize) -> Self { - Span { offset, length } - } - - pub fn new_u64(offset: u64, length: u64) -> Self { - Span::new(offset as usize, length as usize) - } - - pub fn range(&self) -> std::ops::Range { - self.offset..self.offset + self.length - } - - pub fn len(&self) -> usize { - self.length - } +/// Build a `Range` from a u64 offset and length. +pub(crate) fn span(offset: u64, length: u64) -> std::ops::Range { + offset as usize..(offset + length) as usize } diff --git a/src/cbq/read.rs b/src/cbq/read.rs index 9d16cf2..9d864ce 100644 --- a/src/cbq/read.rs +++ b/src/cbq/read.rs @@ -231,11 +231,7 @@ impl ParallelReader for MmapReader { num_threads: usize, range: std::ops::Range, ) -> crate::Result<()> { - let num_threads = if num_threads == 0 { - num_cpus::get() - } else { - num_threads.min(num_cpus::get()) - }; + let num_threads = crate::parallel::clamp_threads(num_threads); // validate range let total_records = self.num_records(); @@ -258,53 +254,14 @@ impl ParallelReader for MmapReader { return Ok(()); // nothing to do } - // Distribute blocks evenly across threads, giving extra blocks to first threads - let base_blocks_per_thread = num_blocks / num_threads; - let extra_blocks = num_blocks % num_threads; - + // Distribute blocks evenly across threads let mut handles = Vec::new(); - for thread_id in 0..num_threads { - // Threads 0..extra_blocks get one extra block - let blocks_for_this_thread = if thread_id < extra_blocks { - base_blocks_per_thread + 1 - } else { - base_blocks_per_thread - }; - - // Calculate cumulative start position - let start_block_idx = if thread_id < extra_blocks { - thread_id * (base_blocks_per_thread + 1) - } else { - extra_blocks * (base_blocks_per_thread + 1) - + (thread_id - extra_blocks) * base_blocks_per_thread - }; - let end_block_idx = start_block_idx + blocks_for_this_thread; - - // Skip threads with no work (happens when num_threads > num_blocks) - if blocks_for_this_thread == 0 { - continue; - } - + for t_block_ranges in relevant_blocks.chunks(num_blocks.div_ceil(num_threads)) { + let t_block_ranges = t_block_ranges.to_vec(); + let range = range.clone(); let mut t_reader = self.clone(); let mut t_proc = processor.clone(); - // pull all block ranges for this thread - let t_block_ranges = relevant_blocks - .iter() - .skip(start_block_idx) - .take(end_block_idx - start_block_idx) - .copied() - .collect::>(); - - // eprintln!( - // "Thread {} block range: {}-{}. First block Cumulative Records: {}. Last block Cumulative Records: {}", - // thread_id, - // start_block_idx, - // end_block_idx, - // t_block_ranges[0].cumulative_records, - // t_block_ranges.last().unwrap().cumulative_records - // ); - let thread_handle = thread::spawn(move || -> crate::Result<()> { for b_range in t_block_ranges { t_reader.load_block(b_range)?; @@ -582,39 +539,6 @@ mod tests { assert!(result.is_err(), "Should fail on invalid file format"); } - // ==================== Block Header Iterator Tests ==================== - - #[test] - fn test_iter_block_headers() { - let reader = MmapReader::new(TEST_CBQ_FILE).unwrap(); - - let headers: Vec<_> = reader - .iter_block_headers() - .take(5) - .collect::>>() - .unwrap(); - - assert!(!headers.is_empty(), "Should have at least one block header"); - - for header in headers { - assert!(header.num_records > 0, "Block should have records"); - } - } - - #[test] - fn test_iter_block_headers_count() { - let reader = MmapReader::new(TEST_CBQ_FILE).unwrap(); - - let header_count = reader - .iter_block_headers() - .collect::>>() - .unwrap() - .len(); - - let num_blocks = reader.num_blocks(); - assert_eq!(header_count, num_blocks, "Should iterate all block headers"); - } - // ==================== Empty Range Tests ==================== #[test] diff --git a/src/cbq/write.rs b/src/cbq/write.rs index e94df6c..90cb821 100644 --- a/src/cbq/write.rs +++ b/src/cbq/write.rs @@ -84,11 +84,6 @@ impl ColumnarBlockWriter { self.block.header } - /// Calculate the usage of the block as a percentage - pub fn usage(&self) -> f64 { - self.block.usage() - } - /// Push a record to the writer /// /// Returns `Ok(true)` if the record was written successfully. @@ -211,12 +206,6 @@ impl ColumnarBlockWriter> { pub fn clear_incomplete_data(&mut self) { self.block.clear(); } - - /// Returns the number of bytes written to the inner data structure - #[must_use] - pub fn bytes_written(&self) -> usize { - self.inner.len() - } } #[cfg(test)] diff --git a/src/encoder.rs b/src/encoder.rs new file mode 100644 index 0000000..23ff7da --- /dev/null +++ b/src/encoder.rs @@ -0,0 +1,145 @@ +//! Shared nucleotide encoder used by the BQ and VBQ writers. + +use bitnuc_deprec::BitSize; +use rand::{SeedableRng, rngs::SmallRng}; + +use crate::{Policy, RNG_SEED, error::Result}; + +/// Encodes nucleotide sequences into a compact multi-bit binary format. +/// +/// The encoder maintains internal buffers to avoid repeated allocations and +/// handles invalid nucleotides according to a configurable [`Policy`]. +#[derive(Clone)] +pub(crate) struct Encoder { + /// Bitsize of the nucleotides + bitsize: BitSize, + + /// Reusable buffers for encoded nucleotides + sbuffer: Vec, + xbuffer: Vec, + + /// Reusable buffers for invalid nucleotide sequences + s_ibuf: Vec, + x_ibuf: Vec, + + /// Invalid Nucleotide Policy + policy: Policy, + + /// Random Number Generator (seeded with `RNG_SEED` for reproducibility) + rng: SmallRng, +} + +impl Encoder { + /// Initialize a new encoder with the given policy. + pub fn with_policy(bitsize: BitSize, policy: Policy) -> Self { + Self { + bitsize, + policy, + sbuffer: Vec::default(), + xbuffer: Vec::default(), + s_ibuf: Vec::default(), + x_ibuf: Vec::default(), + rng: SmallRng::seed_from_u64(RNG_SEED), + } + } + + /// Returns the invalid-nucleotide policy of this encoder. + pub fn policy(&self) -> Policy { + self.policy + } + + /// Encodes a single sequence. + /// + /// Will return `None` if the sequence is invalid and the policy does not allow correction. + pub fn encode_single(&mut self, primary: &[u8]) -> Result> { + self.clear(); + if self.bitsize.encode(primary, &mut self.sbuffer).is_err() { + self.clear(); + if self + .policy + .handle(primary, &mut self.s_ibuf, &mut self.rng)? + { + self.bitsize.encode(&self.s_ibuf, &mut self.sbuffer)?; + } else { + return Ok(None); + } + } + Ok(Some(&self.sbuffer)) + } + + /// Encodes a pair of sequences. + /// + /// Will return `None` if either sequence is invalid and the policy does not allow correction. + pub fn encode_paired( + &mut self, + primary: &[u8], + extended: &[u8], + ) -> Result> { + self.clear(); + if self.bitsize.encode(primary, &mut self.sbuffer).is_err() + || self.bitsize.encode(extended, &mut self.xbuffer).is_err() + { + self.clear(); + if self + .policy + .handle(primary, &mut self.s_ibuf, &mut self.rng)? + && self + .policy + .handle(extended, &mut self.x_ibuf, &mut self.rng)? + { + self.bitsize.encode(&self.s_ibuf, &mut self.sbuffer)?; + self.bitsize.encode(&self.x_ibuf, &mut self.xbuffer)?; + } else { + return Ok(None); + } + } + Ok(Some((&self.sbuffer, &self.xbuffer))) + } + + /// Clear all buffers and reset the encoder. + pub fn clear(&mut self) { + self.sbuffer.clear(); + self.xbuffer.clear(); + self.s_ibuf.clear(); + self.x_ibuf.clear(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_encode_single_invalid_ignored() { + let mut encoder = Encoder::with_policy(BitSize::Two, Policy::IgnoreSequence); + assert!(encoder.encode_single(b"ACGTNNNN").unwrap().is_none()); + } + + #[test] + fn test_encode_single_invalid_corrected() { + let mut encoder = Encoder::with_policy(BitSize::Two, Policy::SetToA); + assert!(encoder.encode_single(b"ACGTNNNN").unwrap().is_some()); + } + + #[test] + fn test_encode_paired_invalid_ignored() { + let mut encoder = Encoder::with_policy(BitSize::Two, Policy::IgnoreSequence); + assert!( + encoder + .encode_paired(b"ACGTNNNN", b"ACGTACGT") + .unwrap() + .is_none() + ); + } + + #[test] + fn test_encode_paired_invalid_corrected() { + let mut encoder = Encoder::with_policy(BitSize::Two, Policy::SetToA); + assert!( + encoder + .encode_paired(b"ACGTNNNN", b"NNNNACGT") + .unwrap() + .is_some() + ); + } +} diff --git a/src/error.rs b/src/error.rs index ef20cbb..27d4089 100644 --- a/src/error.rs +++ b/src/error.rs @@ -6,7 +6,6 @@ pub type Result = std::result::Result; /// The main error type for the binseq library, encompassing all possible error cases /// that can occur during binary sequence operations. #[derive(thiserror::Error, Debug)] -#[error(transparent)] pub enum Error { /// Errors related to file and block headers #[error("Error processing header: {0}")] @@ -24,10 +23,6 @@ pub enum Error { #[error("Error reading file: {0}")] ReadError(#[from] ReadError), - /// Errors that occur during build operations - #[error("Error building file: {0}")] - BuilderError(#[from] BuilderError), - /// Errors related to file indexing #[error("Error processing Index: {0}")] IndexError(#[from] IndexError), @@ -60,10 +55,6 @@ pub enum Error { /// Generic errors for other unexpected situations #[error("Generic error: {0}")] GenericError(#[from] Box), - - #[cfg(feature = "paraseq")] - #[error("Fastx encoding error: {0}")] - FastxEncodingError(#[from] FastxEncodingError), } /// Errors specific to processing and validating binary sequence headers @@ -98,6 +89,10 @@ pub enum HeaderError { /// * Second `usize` - The expected number of bytes according to the header #[error("Invalid number of bytes provided: {0}. Expected: {1}")] InvalidSize(usize, usize), + + /// A required sequence length was not provided when building the header + #[error("Missing sequence length")] + MissingSequenceLength, } /// Errors that can occur while reading binary sequence data @@ -162,12 +157,6 @@ pub enum ReadError { MissingIndexEndMagic, } -#[derive(thiserror::Error, Debug)] -pub enum BuilderError { - #[error("Missing sequence length")] - MissingSlen, -} - /// Errors that can occur while writing binary sequence data #[derive(thiserror::Error, Debug)] pub enum WriteError { @@ -213,26 +202,6 @@ pub enum WriteError { #[error("Missing header in writer builder")] MissingHeader, - /// When trying to write data without quality scores but the header specifies they should be present - #[error("Quality flag is set in header but trying to write without quality scores.")] - QualityFlagSet, - - /// When trying to write data without a pair but the header specifies paired records - #[error("Paired flag is set in header but trying to write without record pair.")] - PairedFlagSet, - - /// When trying to write quality scores but the header specifies they are not present - #[error("Quality flag not set in header but trying to write quality scores.")] - QualityFlagNotSet, - - /// When trying to write paired data but the header doesn't specify paired records - #[error("Paired flag not set in header but trying to write with record pair.")] - PairedFlagNotSet, - - /// When trying to write data without headers but the header specifies they should be present - #[error("Header flag is set in header but trying to write without headers.")] - HeaderFlagSet, - /// When a record is too large to fit in a block of the configured size /// /// The first parameter is the record size, the second is the maximum block size @@ -257,6 +226,16 @@ pub enum WriteError { /// When building a `SequencingRecord` without a primary sequence #[error("SequencingRecordBuilder requires a primary sequence (s_seq)")] MissingSequence, + + /// FASTX encoding was attempted on an empty input file + #[cfg(feature = "paraseq")] + #[error("Empty FASTX file")] + EmptyFastxFile, + + /// FASTX encoding was attempted without any input + #[cfg(feature = "paraseq")] + #[error("Builder not provided with any input")] + MissingInput, } /// Errors related to VBQ file indexing @@ -316,19 +295,6 @@ pub enum CbqError { #[error("Unable to cast bytes to Index - likely an alignment error")] IndexCastingError, - - #[error("SequenceRecordBuilder failed on build due to missing primary sequence (`s_seq`)")] - MissingSequenceOnSequencingRecord, -} - -#[cfg(feature = "paraseq")] -#[derive(thiserror::Error, Debug)] -pub enum FastxEncodingError { - #[error("Empty FASTX file")] - EmptyFastxFile, - - #[error("Builder not provided with any input")] - MissingInput, } #[derive(thiserror::Error, Debug)] @@ -353,12 +319,11 @@ where } } +#[cfg(test)] mod testing { - #[allow(unused)] use super::*; use thiserror::Error; - #[allow(unused)] #[derive(Error, Debug)] pub enum MyError { #[error("Custom error: {0}")] @@ -372,248 +337,9 @@ mod testing { assert!(matches!(binseq_error, Error::GenericError(_))); } - // ==================== HeaderError Tests ==================== - - #[test] - fn test_header_error_invalid_magic_number() { - let error = HeaderError::InvalidMagicNumber(0xDEAD_BEEF); - let error_str = format!("{error}"); - assert!(error_str.contains("0xdeadbeef") || error_str.contains("3735928559")); - } - - #[test] - fn test_header_error_invalid_format_version() { - let error = HeaderError::InvalidFormatVersion(99); - let error_str = format!("{error}"); - assert!(error_str.contains("99")); - } - - #[test] - fn test_header_error_invalid_bit_size() { - let error = HeaderError::InvalidBitSize(8); - let error_str = format!("{error}"); - assert!(error_str.contains('8')); - assert!(error_str.contains("[2,4]")); - } - - #[test] - fn test_header_error_invalid_size() { - let error = HeaderError::InvalidSize(100, 200); - let error_str = format!("{error}"); - assert!(error_str.contains("100")); - assert!(error_str.contains("200")); - } - - // ==================== ReadError Tests ==================== - - #[test] - fn test_read_error_out_of_range() { - let error = ReadError::OutOfRange { - requested_index: 150, - max_index: 100, - }; - let error_str = format!("{error}"); - assert!(error_str.contains("150")); - assert!(error_str.contains("100")); - } - - #[test] - fn test_read_error_file_truncation() { - let error = ReadError::FileTruncation(12345); - let error_str = format!("{error}"); - assert!(error_str.contains("12345")); - } - #[test] - fn test_read_error_partial_record() { - let error = ReadError::PartialRecord(42); - let error_str = format!("{error}"); - assert!(error_str.contains("42")); - } - - #[test] - fn test_read_error_invalid_block_magic_number() { - let error = ReadError::InvalidBlockMagicNumber(0x0BAD_C0DE, 1000); - let error_str = format!("{error}"); - assert!(error_str.contains("1000")); - } - - // ==================== WriteError Tests ==================== - - #[test] - fn test_write_error_configuration_mismatch() { - let error = WriteError::ConfigurationMismatch { - attribute: "paired", - expected: true, - actual: false, - }; - let error_str = format!("{error}"); - assert!(error_str.contains("paired")); - assert!(error_str.contains("true")); - assert!(error_str.contains("false")); - } - - #[test] - fn test_write_error_unexpected_sequence_length() { - let error = WriteError::UnexpectedSequenceLength { - expected: 100, - got: 150, - }; - let error_str = format!("{error}"); - assert!(error_str.contains("100")); - assert!(error_str.contains("150")); - } - - #[test] - fn test_write_error_invalid_nucleotide_sequence() { - let error = WriteError::InvalidNucleotideSequence("ACGTNX".to_string()); - let error_str = format!("{error}"); - assert!(error_str.contains("ACGTNX")); - } - - #[test] - fn test_write_error_record_size_exceeds_max() { - let error = WriteError::RecordSizeExceedsMaximumBlockSize(2000, 1024); - let error_str = format!("{error}"); - assert!(error_str.contains("2000")); - assert!(error_str.contains("1024")); - } - - #[test] - fn test_write_error_missing_sequence_length() { - let error = WriteError::MissingSequenceLength { - exp_primary: true, - exp_extended: false, - obs_primary: false, - obs_extended: false, - }; - let error_str = format!("{error}"); - assert!(error_str.contains("Missing required sequence length")); - } - - // ==================== CbqError Tests ==================== - - #[test] - fn test_cbq_error_exceeds_maximum_block_size() { - let error = CbqError::ExceedsMaximumBlockSize { - max_block_size: 1024, - record_size: 2048, - }; - let error_str = format!("{error}"); - assert!(error_str.contains("1024")); - assert!(error_str.contains("2048")); - } - - #[test] - fn test_cbq_error_block_full() { - let error = CbqError::BlockFull { - current_size: 900, - record_size: 200, - block_size: 1024, - }; - let error_str = format!("{error}"); - assert!(error_str.contains("900")); - assert!(error_str.contains("200")); - assert!(error_str.contains("1024")); - } - - #[test] - fn test_cbq_error_cannot_ingest_block() { - let error = CbqError::CannotIngestBlock { - self_block_size: 1024, - other_block_size: 2048, - }; - let error_str = format!("{error}"); - assert!(error_str.contains("1024")); - assert!(error_str.contains("2048")); - } - - // ==================== BuilderError Tests ==================== - - #[test] - fn test_builder_error_missing_slen() { - let error = BuilderError::MissingSlen; - let error_str = format!("{error}"); - assert!(error_str.contains("Missing sequence length")); - } - - // ==================== FormatError Tests ==================== - - #[test] - fn test_format_error_unrecognized_magic_bytes() { - let error = FormatError::UnrecognizedMagicBytes("test.xyz".to_string()); - let error_str = format!("{error}"); - assert!(error_str.contains("test.xyz")); - } - - // ==================== Error Conversion Tests ==================== - - #[test] - fn test_error_from_header_error() { - let header_error = HeaderError::InvalidMagicNumber(0x1234); - let error: Error = header_error.into(); + fn test_error_from_sub_error() { + let error: Error = HeaderError::InvalidMagicNumber(0x1234).into(); assert!(matches!(error, Error::HeaderError(_))); } - - #[test] - fn test_error_from_write_error() { - let write_error = WriteError::MissingHeader; - let error: Error = write_error.into(); - assert!(matches!(error, Error::WriteError(_))); - } - - #[test] - fn test_error_from_read_error() { - let read_error = ReadError::EndOfStream; - let error: Error = read_error.into(); - assert!(matches!(error, Error::ReadError(_))); - } - - #[test] - fn test_error_from_index_error() { - let index_error = IndexError::InvalidMagicNumber(0x5678); - let error: Error = index_error.into(); - assert!(matches!(error, Error::IndexError(_))); - } - - #[test] - fn test_error_from_cbq_error() { - let cbq_error = CbqError::InvalidBlockHeaderMagic; - let error: Error = cbq_error.into(); - assert!(matches!(error, Error::CbqError(_))); - } - - #[test] - fn test_error_from_builder_error() { - let builder_error = BuilderError::MissingSlen; - let error: Error = builder_error.into(); - assert!(matches!(error, Error::BuilderError(_))); - } - - #[test] - fn test_error_debug_output() { - let error = Error::WriteError(WriteError::MissingHeader); - let debug_str = format!("{error:?}"); - assert!(debug_str.contains("WriteError")); - } - - // ==================== Fastx Error Tests (conditional) ==================== - - #[cfg(feature = "paraseq")] - #[test] - fn test_fastx_error_empty_file() { - use super::FastxEncodingError; - let error = FastxEncodingError::EmptyFastxFile; - let error_str = format!("{error}"); - assert!(error_str.contains("Empty FASTX file")); - } - - #[cfg(feature = "paraseq")] - #[test] - fn test_fastx_error_missing_input() { - use super::FastxEncodingError; - let error = FastxEncodingError::MissingInput; - let error_str = format!("{error}"); - assert!(error_str.contains("not provided with any input")); - } } diff --git a/src/lib.rs b/src/lib.rs index 7a3ccb6..5511d60 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,5 @@ +#![cfg_attr(docsrs, feature(doc_cfg))] +#![cfg_attr(docsrs, doc(auto_cfg))] #![doc = include_str!("../README.md")] //! //! # BINSEQ @@ -81,6 +83,9 @@ /// BQ - fixed length records, no quality scores pub mod bq; +/// Shared nucleotide encoder for BQ/VBQ writers +mod encoder; + /// Error definitions pub mod error; diff --git a/src/parallel.rs b/src/parallel.rs index eedd96c..eaf3a88 100644 --- a/src/parallel.rs +++ b/src/parallel.rs @@ -88,32 +88,15 @@ impl BinseqReader { /// Process records in parallel within a specified range /// - /// This method allows parallel processing of a subset of records within the file, - /// defined by a start and end index. The range is distributed across the specified - /// number of threads. - /// - /// # Arguments - /// - /// * `processor` - The processor to use for each record - /// * `num_threads` - The number of threads to spawn - /// * `start` - The starting record index (inclusive) - /// * `end` - The ending record index (exclusive) - /// - /// # Returns - /// - /// * `Ok(())` - If all records were processed successfully - /// * `Err(Error)` - If an error occurred during processing + /// Inherent convenience for [`ParallelReader::process_parallel_range`], callable + /// without importing the trait. pub fn process_parallel_range( self, processor: P, num_threads: usize, range: Range, ) -> Result<()> { - match self { - Self::Bq(reader) => reader.process_parallel_range(processor, num_threads, range), - Self::Vbq(reader) => reader.process_parallel_range(processor, num_threads, range), - Self::Cbq(reader) => reader.process_parallel_range(processor, num_threads, range), - } + ::process_parallel_range(self, processor, num_threads, range) } } impl ParallelReader for BinseqReader { @@ -140,6 +123,18 @@ impl ParallelReader for BinseqReader { } } +/// Clamp a requested thread count to the available parallelism. +/// +/// A request of 0 means "use all available cores". +pub(crate) fn clamp_threads(num_threads: usize) -> usize { + let available = std::thread::available_parallelism().map_or(1, std::num::NonZero::get); + if num_threads == 0 { + available + } else { + num_threads.min(available) + } +} + /// Trait for BINSEQ readers that can process records in parallel /// /// This is implemented by the **reader** not by the **processor**. @@ -224,14 +219,12 @@ pub trait ParallelProcessor: Send + Clone { /// Called when a thread finishes processing its batch /// Default implementation does nothing - #[allow(unused_variables)] fn on_batch_complete(&mut self) -> Result<()> { Ok(()) } /// Called when a thread finished processing all its batches /// Default implementation does nothing - #[allow(unused_variables)] fn on_thread_complete(&mut self) -> Result<()> { Ok(()) } @@ -239,7 +232,6 @@ pub trait ParallelProcessor: Send + Clone { /// Set the thread ID for this processor /// /// Each thread should call this method with its own unique ID. - #[allow(unused_variables)] fn set_tid(&mut self, _tid: usize) { // Default implementation does nothing } diff --git a/src/policy.rs b/src/policy.rs index c09d326..3e9a515 100644 --- a/src/policy.rs +++ b/src/policy.rs @@ -84,13 +84,7 @@ impl Policy { for &n in sequence { ibuf.push(match n { b'A' | b'C' | b'G' | b'T' => n, - _ => match rng.random_range(0..4) { - 0 => b'A', - 1 => b'C', - 2 => b'G', - 3 => b'T', - _ => unreachable!(), - }, + _ => b"ACGT"[rng.random_range(0..4)], }); } } @@ -176,37 +170,28 @@ mod tests { use rand::SeedableRng; use rand::rngs::StdRng; - // ==================== Basic Policy Tests ==================== + fn run(policy: Policy, seq: &[u8]) -> (Result, Vec) { + let mut output = Vec::new(); + let mut rng = StdRng::seed_from_u64(RNG_SEED); + let result = policy.handle(seq, &mut output, &mut rng); + (result, output) + } #[test] fn test_default_policy() { - let policy = Policy::default(); - assert!(matches!(policy, Policy::IgnoreSequence)); + assert!(matches!(Policy::default(), Policy::IgnoreSequence)); } #[test] fn test_ignore_sequence_policy() { - let policy = Policy::IgnoreSequence; - let sequence = b"ACGTNX"; - let mut output = Vec::new(); - let mut rng = StdRng::seed_from_u64(RNG_SEED); - - let should_process = policy.handle(sequence, &mut output, &mut rng).unwrap(); - - assert!(!should_process); // Should return false to skip this sequence - assert!(output.is_empty()); // Output buffer should be empty + let (result, output) = run(Policy::IgnoreSequence, b"ACGTNX"); + assert!(!result.unwrap()); // Should return false to skip this sequence + assert!(output.is_empty()); } #[test] fn test_break_on_invalid_policy() { - let policy = Policy::BreakOnInvalid; - let sequence = b"ACGTNX"; - let mut output = Vec::new(); - let mut rng = StdRng::seed_from_u64(RNG_SEED); - - let result = policy.handle(sequence, &mut output, &mut rng); - - assert!(result.is_err()); + let (result, _) = run(Policy::BreakOnInvalid, b"ACGTNX"); assert!(matches!( result.unwrap_err(), crate::error::Error::WriteError(WriteError::InvalidNucleotideSequence(_)) @@ -214,237 +199,55 @@ mod tests { } #[test] - fn test_break_on_invalid_with_valid_sequence() { - let policy = Policy::BreakOnInvalid; - let sequence = b"ACGT"; - let mut output = Vec::new(); - let mut rng = StdRng::seed_from_u64(RNG_SEED); - - let result = policy.handle(sequence, &mut output, &mut rng); - - // Valid sequences should error because handle() doesn't validate for BreakOnInvalid - // It only returns an error immediately - assert!(result.is_err()); - } - - // ==================== Set-to-Specific-Nucleotide Tests ==================== - - #[test] - fn test_set_to_a_policy() { - let policy = Policy::SetToA; - let sequence = b"ACGTNX"; - let mut output = Vec::new(); - let mut rng = StdRng::seed_from_u64(RNG_SEED); - - let should_process = policy.handle(sequence, &mut output, &mut rng).unwrap(); - - assert!(should_process); // Should return true to process this sequence - assert_eq!(output, b"ACGTAA"); // N and X should be replaced with A - } - - #[test] - fn test_set_to_c_policy() { - let policy = Policy::SetToC; - let sequence = b"ACGTNX"; - let mut output = Vec::new(); - let mut rng = StdRng::seed_from_u64(RNG_SEED); - - let should_process = policy.handle(sequence, &mut output, &mut rng).unwrap(); - - assert!(should_process); - assert_eq!(output, b"ACGTCC"); // N and X should be replaced with C - } - - #[test] - fn test_set_to_g_policy() { - let policy = Policy::SetToG; - let sequence = b"ACGTNX"; - let mut output = Vec::new(); - let mut rng = StdRng::seed_from_u64(RNG_SEED); - - let should_process = policy.handle(sequence, &mut output, &mut rng).unwrap(); - - assert!(should_process); - assert_eq!(output, b"ACGTGG"); // N and X should be replaced with G - } - - #[test] - fn test_set_to_t_policy() { - let policy = Policy::SetToT; - let sequence = b"ACGTNX"; - let mut output = Vec::new(); - let mut rng = StdRng::seed_from_u64(RNG_SEED); - - let should_process = policy.handle(sequence, &mut output, &mut rng).unwrap(); - - assert!(should_process); - assert_eq!(output, b"ACGTTT"); // N and X should be replaced with T - } - - #[test] - fn test_all_valid_nucleotides_unchanged() { - let policy = Policy::SetToA; - let sequence = b"ACGTACGT"; - let mut output = Vec::new(); - let mut rng = StdRng::seed_from_u64(RNG_SEED); - - let should_process = policy.handle(sequence, &mut output, &mut rng).unwrap(); - - assert!(should_process); - assert_eq!(output, b"ACGTACGT"); // All valid, should remain unchanged + fn test_set_to_policies() { + // (policy, input, expected output) + let cases: &[(Policy, &[u8], &[u8])] = &[ + (Policy::SetToA, b"ACGTNX", b"ACGTAA"), + (Policy::SetToC, b"ACGTNX", b"ACGTCC"), + (Policy::SetToG, b"ACGTNX", b"ACGTGG"), + (Policy::SetToT, b"ACGTNX", b"ACGTTT"), + // valid nucleotides remain unchanged + (Policy::SetToA, b"ACGTACGT", b"ACGTACGT"), + // empty sequence + (Policy::SetToA, b"", b""), + // all invalid + (Policy::SetToG, b"NNNXXX", b"GGGGGG"), + // lowercase and ambiguous codes are invalid + (Policy::SetToA, b"acgt", b"AAAA"), + (Policy::SetToC, b"AcGt", b"ACGC"), + (Policy::SetToT, b"RYWSMK", b"TTTTTT"), + ]; + for &(policy, seq, expected) in cases { + let (result, output) = run(policy, seq); + assert!(result.unwrap(), "{policy:?} on {seq:?}"); + assert_eq!(output, expected, "{policy:?} on {seq:?}"); + } } - // ==================== Random Draw Tests ==================== - #[test] fn test_random_draw_policy() { - let policy = Policy::RandomDraw; - let sequence = b"ACGTNX"; - let mut output = Vec::new(); - let mut rng = StdRng::seed_from_u64(RNG_SEED); - - let should_process = policy.handle(sequence, &mut output, &mut rng).unwrap(); - - assert!(should_process); - assert_eq!(output.len(), 6); // Same length as input - // First 4 nucleotides should be unchanged - assert_eq!(&output[0..4], b"ACGT"); - // Last 2 should be valid nucleotides (A, C, G, or T) - assert!(matches!(output[4], b'A' | b'C' | b'G' | b'T')); - assert!(matches!(output[5], b'A' | b'C' | b'G' | b'T')); + let (result, output) = run(Policy::RandomDraw, b"ACGTNX"); + assert!(result.unwrap()); + assert_eq!(&output[0..4], b"ACGT"); // valid prefix unchanged + assert!(output[4..].iter().all(|n| b"ACGT".contains(n))); } #[test] fn test_random_draw_deterministic_with_seed() { - let policy = Policy::RandomDraw; - let sequence = b"NNNN"; - let mut output1 = Vec::new(); - let mut output2 = Vec::new(); - let mut rng1 = StdRng::seed_from_u64(RNG_SEED); - let mut rng2 = StdRng::seed_from_u64(RNG_SEED); - - policy.handle(sequence, &mut output1, &mut rng1).unwrap(); - policy.handle(sequence, &mut output2, &mut rng2).unwrap(); - - // Same seed should produce same output - assert_eq!(output1, output2); + let (_, output1) = run(Policy::RandomDraw, b"NNNN"); + let (_, output2) = run(Policy::RandomDraw, b"NNNN"); + assert_eq!(output1, output2); // same seed, same output } - // ==================== Buffer Clearing Tests ==================== - #[test] fn test_buffer_cleared_before_processing() { let policy = Policy::SetToA; - let sequence = b"ACGT"; - let mut output = vec![b'X', b'Y', b'Z']; // Pre-fill buffer + let mut output = vec![b'X', b'Y', b'Z']; // pre-filled buffer let mut rng = StdRng::seed_from_u64(RNG_SEED); - - policy.handle(sequence, &mut output, &mut rng).unwrap(); - - // Buffer should be cleared and only contain new data - assert_eq!(output, b"ACGT"); - } - - #[test] - fn test_multiple_calls_clear_buffer() { - let policy = Policy::SetToA; - let mut output = Vec::new(); - let mut rng = StdRng::seed_from_u64(RNG_SEED); - policy.handle(b"ACGT", &mut output, &mut rng).unwrap(); assert_eq!(output, b"ACGT"); policy.handle(b"TT", &mut output, &mut rng).unwrap(); - assert_eq!(output, b"TT"); // Should only contain second sequence - } - - // ==================== Edge Case Tests ==================== - - #[test] - fn test_empty_sequence() { - let policy = Policy::SetToA; - let sequence = b""; - let mut output = Vec::new(); - let mut rng = StdRng::seed_from_u64(RNG_SEED); - - let should_process = policy.handle(sequence, &mut output, &mut rng).unwrap(); - - assert!(should_process); - assert!(output.is_empty()); - } - - #[test] - fn test_all_invalid_nucleotides() { - let policy = Policy::SetToG; - let sequence = b"NNNXXX"; - let mut output = Vec::new(); - let mut rng = StdRng::seed_from_u64(RNG_SEED); - - let should_process = policy.handle(sequence, &mut output, &mut rng).unwrap(); - - assert!(should_process); - assert_eq!(output, b"GGGGGG"); // All should be replaced with G - } - - #[test] - fn test_policy_clone() { - let policy1 = Policy::SetToA; - let policy2 = policy1; - - // Should be able to use both (tests Copy trait) - let mut output = Vec::new(); - let mut rng = StdRng::seed_from_u64(RNG_SEED); - - policy1.handle(b"NT", &mut output, &mut rng).unwrap(); - assert_eq!(output, b"AT"); - - policy2.handle(b"NT", &mut output, &mut rng).unwrap(); - assert_eq!(output, b"AT"); - } - - #[test] - fn test_policy_debug() { - let policy = Policy::SetToA; - let debug_str = format!("{policy:?}"); - assert!(debug_str.contains("SetToA")); - } - - // ==================== Various Invalid Character Tests ==================== - - #[test] - fn test_lowercase_nucleotides_treated_as_invalid() { - let policy = Policy::SetToA; - let sequence = b"acgt"; // lowercase - let mut output = Vec::new(); - let mut rng = StdRng::seed_from_u64(RNG_SEED); - - policy.handle(sequence, &mut output, &mut rng).unwrap(); - - // Lowercase nucleotides should be treated as invalid - assert_eq!(output, b"AAAA"); - } - - #[test] - fn test_mixed_case_nucleotides() { - let policy = Policy::SetToC; - let sequence = b"AcGt"; - let mut output = Vec::new(); - let mut rng = StdRng::seed_from_u64(RNG_SEED); - - policy.handle(sequence, &mut output, &mut rng).unwrap(); - - assert_eq!(output, b"ACGC"); // Only uppercase are valid - } - - #[test] - fn test_ambiguous_nucleotide_codes() { - let policy = Policy::SetToT; - let sequence = b"RYWSMK"; // R, Y, W, S, M, K are ambiguous codes - let mut output = Vec::new(); - let mut rng = StdRng::seed_from_u64(RNG_SEED); - - policy.handle(sequence, &mut output, &mut rng).unwrap(); - - assert_eq!(output, b"TTTTTT"); // All ambiguous codes replaced with T + assert_eq!(output, b"TT"); // cleared again on the next call } } diff --git a/src/record/binseq_record.rs b/src/record/binseq_record.rs index 7641799..9ed9ef6 100644 --- a/src/record/binseq_record.rs +++ b/src/record/binseq_record.rs @@ -1,4 +1,3 @@ -use auto_impl::auto_impl; use bitnuc_deprec::BitSize; use crate::Result; @@ -11,7 +10,6 @@ use crate::Result; /// Implemented by [`bq::RefRecord`](crate::bq::RefRecord) and [`vbq::RefRecord`](crate::vbq::RefRecord). /// /// Used to interact with [`ParallelProcessor`](crate::ParallelProcessor) for easy parallel processing. -#[auto_impl(&, &mut)] pub trait BinseqRecord { /// Returns the bitsize of the record (number of bits per nucleotide) fn bitsize(&self) -> BitSize; diff --git a/src/record/sequencing_record.rs b/src/record/sequencing_record.rs index b5da435..7437d78 100644 --- a/src/record/sequencing_record.rs +++ b/src/record/sequencing_record.rs @@ -20,13 +20,13 @@ use crate::{BitSize, Result, error::WriteError}; /// ``` #[derive(Clone, Copy, Default)] pub struct SequencingRecord<'a> { - pub(crate) s_seq: &'a [u8], - pub(crate) s_qual: Option<&'a [u8]>, - pub(crate) s_header: Option<&'a [u8]>, - pub(crate) x_seq: Option<&'a [u8]>, - pub(crate) x_qual: Option<&'a [u8]>, - pub(crate) x_header: Option<&'a [u8]>, - pub(crate) flag: Option, + pub s_seq: &'a [u8], + pub s_qual: Option<&'a [u8]>, + pub s_header: Option<&'a [u8]>, + pub x_seq: Option<&'a [u8]>, + pub x_qual: Option<&'a [u8]>, + pub x_header: Option<&'a [u8]>, + pub flag: Option, } impl<'a> SequencingRecord<'a> { @@ -52,55 +52,6 @@ impl<'a> SequencingRecord<'a> { } } - /// Returns the primary sequence - #[inline] - #[must_use] - pub fn s_seq(&self) -> &'a [u8] { - self.s_seq - } - - /// Returns the primary quality scores if present - #[inline] - #[must_use] - pub fn s_qual(&self) -> Option<&'a [u8]> { - self.s_qual - } - - /// Returns the primary header if present - #[inline] - #[must_use] - pub fn s_header(&self) -> Option<&'a [u8]> { - self.s_header - } - - /// Returns the extended/paired sequence if present - #[inline] - #[must_use] - pub fn x_seq(&self) -> Option<&'a [u8]> { - self.x_seq - } - - /// Returns the extended quality scores if present - #[inline] - #[must_use] - pub fn x_qual(&self) -> Option<&'a [u8]> { - self.x_qual - } - - /// Returns the extended header if present - #[inline] - #[must_use] - pub fn x_header(&self) -> Option<&'a [u8]> { - self.x_header - } - - /// Returns the flag if present - #[inline] - #[must_use] - pub fn flag(&self) -> Option { - self.flag - } - /// Returns the configured size of this record for CBQ format. /// /// CBQ uses columnar storage so there are no per-record length prefixes. @@ -419,25 +370,25 @@ mod tests { Some(b"x_hdr"), Some(7), ); - assert_eq!(record.s_seq(), b"ACGT"); - assert_eq!(record.s_qual(), Some(b"IIII".as_slice())); - assert_eq!(record.s_header(), Some(b"s_hdr".as_slice())); - assert_eq!(record.x_seq(), Some(b"TGCA".as_slice())); - assert_eq!(record.x_qual(), Some(b"FFFF".as_slice())); - assert_eq!(record.x_header(), Some(b"x_hdr".as_slice())); - assert_eq!(record.flag(), Some(7)); + assert_eq!(record.s_seq, b"ACGT"); + assert_eq!(record.s_qual, Some(b"IIII".as_slice())); + assert_eq!(record.s_header, Some(b"s_hdr".as_slice())); + assert_eq!(record.x_seq, Some(b"TGCA".as_slice())); + assert_eq!(record.x_qual, Some(b"FFFF".as_slice())); + assert_eq!(record.x_header, Some(b"x_hdr".as_slice())); + assert_eq!(record.flag, Some(7)); } #[test] fn test_new_constructor_minimal() { let record = SequencingRecord::new(b"ACGT", None, None, None, None, None, None); - assert_eq!(record.s_seq(), b"ACGT"); - assert_eq!(record.s_qual(), None); - assert_eq!(record.s_header(), None); - assert_eq!(record.x_seq(), None); - assert_eq!(record.x_qual(), None); - assert_eq!(record.x_header(), None); - assert_eq!(record.flag(), None); + assert_eq!(record.s_seq, b"ACGT"); + assert_eq!(record.s_qual, None); + assert_eq!(record.s_header, None); + assert_eq!(record.x_seq, None); + assert_eq!(record.x_qual, None); + assert_eq!(record.x_header, None); + assert_eq!(record.flag, None); assert!(!record.is_paired()); assert!(!record.has_flags()); assert!(!record.has_headers()); @@ -447,35 +398,24 @@ mod tests { // ==================== SequencingRecordBuilder opt_* setters ==================== #[test] - fn test_builder_opt_setters_some() { + fn test_builder_opt_setters() { let record = SequencingRecordBuilder::default() .s_seq(b"ACGT") + .opt_s_qual(Some(b"FFFF")) .opt_s_header(Some(b"s_hdr")) .opt_x_seq(Some(b"TGCA")) - .opt_x_qual(Some(b"FFFF")) - .opt_flag(Some(9)) - .build() - .unwrap(); - assert_eq!(record.s_header(), Some(b"s_hdr".as_slice())); - assert_eq!(record.x_seq(), Some(b"TGCA".as_slice())); - assert_eq!(record.x_qual(), Some(b"FFFF".as_slice())); - assert_eq!(record.flag(), Some(9)); - } - - #[test] - fn test_builder_opt_setters_none() { - let record = SequencingRecordBuilder::default() - .s_seq(b"ACGT") - .opt_s_header(None) - .opt_x_seq(None) .opt_x_qual(None) - .opt_flag(None) + .opt_x_header(None) + .opt_flag(Some(42)) .build() .unwrap(); - assert_eq!(record.s_header(), None); - assert_eq!(record.x_seq(), None); - assert_eq!(record.x_qual(), None); - assert_eq!(record.flag(), None); + // field access and getters are equivalent + assert_eq!(record.s_qual, Some(b"FFFF".as_slice())); + assert_eq!(record.s_header, Some(b"s_hdr".as_slice())); + assert_eq!(record.x_seq, Some(b"TGCA".as_slice())); + assert_eq!(record.x_qual, None); + assert_eq!(record.x_header, None); + assert_eq!(record.flag, Some(42)); } #[test] diff --git a/src/utils/fastx.rs b/src/utils/fastx.rs index 3d13007..a0459c2 100644 --- a/src/utils/fastx.rs +++ b/src/utils/fastx.rs @@ -17,7 +17,7 @@ use parking_lot::Mutex; use crate::{ BinseqWriter, BinseqWriterBuilder, IntoBinseqError, Result, SequencingRecordBuilder, - error::FastxEncodingError, + error::WriteError, }; type BoxedRead = Box; @@ -189,14 +189,20 @@ impl FastxEncoderBuilder { // build interleaved reader let mut reader = fastx::Reader::from_path(path).map_err(IntoBinseqError::into_binseq_error)?; - let (slen, xlen) = detect_seq_len(&mut reader, true)?; + // Only probe for an extended length when the writer is configured + // as paired (i.e. the single input is interleaved); otherwise a + // second read's length would mark fixed-length formats as paired. + let (slen, xlen) = detect_seq_len(&mut reader, self.builder.paired)?; self.builder = self.builder.slen(slen as u32).xlen(xlen as u32); (reader, None) } Some(FastxInput::Stdin) => { let mut reader = fastx::Reader::from_stdin().map_err(IntoBinseqError::into_binseq_error)?; - let (slen, xlen) = detect_seq_len(&mut reader, true)?; + // Only probe for an extended length when the writer is configured + // as paired (i.e. the single input is interleaved); otherwise a + // second read's length would mark fixed-length formats as paired. + let (slen, xlen) = detect_seq_len(&mut reader, self.builder.paired)?; self.builder = self.builder.slen(slen as u32).xlen(xlen as u32); (reader, None) } @@ -211,100 +217,48 @@ impl FastxEncoderBuilder { self.builder = self.builder.slen(slen as u32).xlen(xlen as u32); (reader1, Some(reader2)) } - None => return Err(FastxEncodingError::MissingInput.into()), + None => return Err(WriteError::MissingInput.into()), }; let writer = self.builder.build(self.output)?; - if writer.is_paired() { - if let Some(r2) = r2 { - encode_paired(writer, r1, r2, self.threads)?; - } else { - encode_interleaved(writer, r1, self.threads)?; - } - } else { - encode_single_file(writer, r1, self.threads)?; + let paired = writer.is_paired(); + let mut encoder = Encoder::new(writer)?; + match (paired, r2) { + (true, Some(r2)) => r1.process_parallel_paired(r2, &mut encoder, self.threads), + (true, None) => r1.process_parallel_interleaved(&mut encoder, self.threads), + (false, _) => r1.process_parallel(&mut encoder, self.threads), } + .map_err(IntoBinseqError::into_binseq_error)?; + encoder.finish()?; Ok(()) } } -/// Encode single-end reads from a file -fn encode_single_file( - writer: BinseqWriter, - reader: fastx::Reader, - threads: usize, -) -> Result<()> { - let mut encoder = Encoder::new(writer)?; - reader - .process_parallel(&mut encoder, threads) - .map_err(IntoBinseqError::into_binseq_error)?; - encoder.finish()?; - Ok(()) -} - -/// Encode paired-end reads from interleaved file -fn encode_interleaved( - writer: BinseqWriter, - reader: fastx::Reader, - threads: usize, -) -> Result<()> { - let mut encoder = Encoder::new(writer)?; - reader - .process_parallel_interleaved(&mut encoder, threads) - .map_err(IntoBinseqError::into_binseq_error)?; - encoder.finish()?; - Ok(()) -} - -/// Encode paired-end reads from files -fn encode_paired( - writer: BinseqWriter, - r1: fastx::Reader, - r2: fastx::Reader, - threads: usize, -) -> Result<()> { - let mut encoder = Encoder::new(writer)?; - r1.process_parallel_paired(r2, &mut encoder, threads) - .map_err(IntoBinseqError::into_binseq_error)?; - encoder.finish()?; - Ok(()) -} - fn detect_seq_len( reader: &mut fastx::Reader, interleaved: bool, ) -> Result<(usize, usize)> { - // Initialze the record set + // Initialize the record set let mut rset = reader.new_record_set(); rset.fill(reader) .map_err(IntoBinseqError::into_binseq_error)?; - let (slen, xlen) = if interleaved { + let (slen, xlen) = { let mut rset_iter = rset.iter(); - let Some(Ok(slen)) = rset_iter.next().map(|r| -> Result { - let rec = r.map_err(IntoBinseqError::into_binseq_error)?; - Ok(rec.seq().len()) - }) else { - return Err(FastxEncodingError::EmptyFastxFile.into()); - }; - let Some(Ok(xlen)) = rset_iter.next().map(|r| -> Result { - let rec = r.map_err(IntoBinseqError::into_binseq_error)?; + let mut next_len = || -> Result { + let rec = rset_iter + .next() + .ok_or(WriteError::EmptyFastxFile)? + .map_err(IntoBinseqError::into_binseq_error)?; Ok(rec.seq().len()) - }) else { - return Err(FastxEncodingError::EmptyFastxFile.into()); }; + + let slen = next_len()?; + let xlen = if interleaved { next_len()? } else { 0 }; (slen, xlen) - } else { - let mut rset_iter = rset.iter(); - let Some(Ok(slen)) = rset_iter.next().map(|r| -> Result { - let rec = r.map_err(IntoBinseqError::into_binseq_error)?; - Ok(rec.seq().len()) - }) else { - return Err(FastxEncodingError::EmptyFastxFile.into()); - }; - (slen, 0) }; + reader .reload(&mut rset) .map_err(IntoBinseqError::into_binseq_error)?; @@ -355,6 +309,15 @@ impl ParallelProcessor for Encoder { } fn on_batch_complete(&mut self) -> paraseq::Result<()> { + // Only drain completed blocks mid-stream (keeps CBQ blocks full) + self.writer + .lock() + .ingest_completed(&mut self.thread_writer) + .map_err(IntoProcessError::into_process_error)?; + Ok(()) + } + + fn on_thread_complete(&mut self) -> paraseq::Result<()> { self.writer .lock() .ingest(&mut self.thread_writer) @@ -384,6 +347,15 @@ impl PairedParallelProcessor for Encoder { } fn on_batch_complete(&mut self) -> paraseq::Result<()> { + // Only drain completed blocks mid-stream (keeps CBQ blocks full) + self.writer + .lock() + .ingest_completed(&mut self.thread_writer) + .map_err(IntoProcessError::into_process_error)?; + Ok(()) + } + + fn on_thread_complete(&mut self) -> paraseq::Result<()> { self.writer .lock() .ingest(&mut self.thread_writer) diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 031c492..bdd0233 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -5,3 +5,13 @@ pub mod fastx; #[cfg(feature = "paraseq")] pub use fastx::FastxEncoderBuilder; + +/// Read a little-endian u64 from the start of a byte slice +pub(crate) fn read_u64_le(b: &[u8]) -> u64 { + u64::from_le_bytes(b[..8].try_into().unwrap()) +} + +/// Read a little-endian u32 from the start of a byte slice +pub(crate) fn read_u32_le(b: &[u8]) -> u32 { + u32::from_le_bytes(b[..4].try_into().unwrap()) +} diff --git a/src/vbq/header.rs b/src/vbq/header.rs index 8d5fbf9..bec355a 100644 --- a/src/vbq/header.rs +++ b/src/vbq/header.rs @@ -15,9 +15,11 @@ use std::io::{Read, Write}; use bitnuc_deprec::BitSize; -use byteorder::{ByteOrder, LittleEndian}; -use crate::error::{HeaderError, ReadError, Result}; +use crate::{ + error::{HeaderError, ReadError, Result}, + utils::{read_u32_le, read_u64_le}, +}; /// Magic number for file identification: "VSEQ" in ASCII (0x51455356) /// @@ -119,15 +121,16 @@ impl FileHeaderBuilder { } #[must_use] pub fn build(self) -> FileHeader { - FileHeader::with_capacity( - self.block.unwrap_or(BLOCK_SIZE), - self.qual.unwrap_or(false), - self.compressed.unwrap_or(false), - self.paired.unwrap_or(false), - self.bitsize.unwrap_or_default(), - self.headers.unwrap_or(false), - self.flags.unwrap_or(false), - ) + FileHeader { + block: self.block.unwrap_or(BLOCK_SIZE), + qual: self.qual.unwrap_or(false), + compressed: self.compressed.unwrap_or(false), + paired: self.paired.unwrap_or(false), + bits: self.bitsize.unwrap_or_default(), + headers: self.headers.unwrap_or(false), + flags: self.flags.unwrap_or(false), + ..FileHeader::default() + } } } @@ -214,103 +217,21 @@ impl Default for FileHeader { /// - Does not include sequence headers /// - Uses 2-bit nucleotide encoding fn default() -> Self { - Self::with_capacity( - BLOCK_SIZE, - false, - false, - false, - BitSize::default(), - false, - false, - ) - } -} -impl FileHeader { - /// Creates a new VBQ header with the default block size - /// - /// # Parameters - /// - /// * `qual` - Whether to include quality scores with sequences - /// * `compressed` - Whether to use ZSTD compression for blocks - /// * `paired` - Whether records contain paired sequences - /// * `bitsize` - Number of bits per nucleotide (2 or 4) - /// * `headers` - Whether to include sequence headers with records - /// - /// # Example - /// - /// ```rust - /// use binseq::vbq::FileHeaderBuilder; - /// - /// // Create header with quality scores and compression, without paired sequences - /// let header = FileHeaderBuilder::new() - /// .qual(true) - /// .compressed(true) - /// .build(); - /// ``` - #[must_use] - pub fn new( - qual: bool, - compressed: bool, - paired: bool, - bitsize: BitSize, - headers: bool, - flags: bool, - ) -> Self { - Self::with_capacity( - BLOCK_SIZE, qual, compressed, paired, bitsize, headers, flags, - ) - } - - /// Creates a new VBQ header with a custom block size - /// - /// # Parameters - /// - /// * `block` - Custom block size in bytes (virtual/uncompressed size) - /// * `qual` - Whether to include quality scores with sequences - /// * `compressed` - Whether to use ZSTD compression for blocks - /// * `paired` - Whether records contain paired sequences - /// - /// # Example - /// - /// ```rust - /// use binseq::vbq::FileHeaderBuilder; - /// - /// // Create header with a 256KB block size, with quality scores and compression - /// let header = FileHeaderBuilder::new() - /// .block(256 * 1024) - /// .qual(true) - /// .compressed(true) - /// .build(); - /// ``` - #[must_use] - pub fn with_capacity( - block: u64, - qual: bool, - compressed: bool, - paired: bool, - bitsize: BitSize, - headers: bool, - flags: bool, - ) -> Self { Self { magic: MAGIC, format: FORMAT, - block, - qual, - compressed, - paired, - headers, - flags, - bits: bitsize, + block: BLOCK_SIZE, + qual: false, + compressed: false, + paired: false, + headers: false, + flags: false, + bits: BitSize::default(), reserved: RESERVED_BYTES, } } - - /// Sets the encoding bitsize for the header. - pub fn set_bitsize(&mut self, bits: BitSize) { - self.bits = bits; - } - +} +impl FileHeader { /// Creates a header from a 32-byte buffer /// /// This function parses a raw byte buffer into a `FileHeader` structure, @@ -330,7 +251,7 @@ impl FileHeader { /// * `HeaderError::InvalidFormatVersion` - If the format version is unsupported /// * `HeaderError::InvalidReservedBytes` - If the reserved bytes section is invalid pub fn from_bytes(buffer: &[u8; SIZE_HEADER]) -> Result { - let magic = LittleEndian::read_u32(&buffer[0..4]); + let magic = read_u32_le(&buffer[0..4]); if magic != MAGIC { return Err(HeaderError::InvalidMagicNumber(magic).into()); } @@ -338,7 +259,7 @@ impl FileHeader { if format != FORMAT { return Err(HeaderError::InvalidFormatVersion(format).into()); } - let block = LittleEndian::read_u64(&buffer[5..13]); + let block = read_u64_le(&buffer[5..13]); let qual = buffer[13] != 0; let compressed = buffer[14] != 0; let paired = buffer[15] != 0; @@ -387,9 +308,9 @@ impl FileHeader { /// * IO errors if writing to the writer fails pub fn write_bytes(&self, writer: &mut W) -> Result<()> { let mut buffer = [0u8; SIZE_HEADER]; - LittleEndian::write_u32(&mut buffer[0..4], self.magic); + buffer[0..4].copy_from_slice(&self.magic.to_le_bytes()); buffer[4] = self.format; - LittleEndian::write_u64(&mut buffer[5..13], self.block); + buffer[5..13].copy_from_slice(&self.block.to_le_bytes()); buffer[13] = self.qual.into(); buffer[14] = self.compressed.into(); buffer[15] = self.paired.into(); @@ -523,9 +444,9 @@ impl BlockHeader { /// * IO errors if writing to the writer fails pub fn write_bytes(&self, writer: &mut W) -> Result<()> { let mut buffer = [0u8; SIZE_BLOCK_HEADER]; - LittleEndian::write_u64(&mut buffer[0..8], self.magic); - LittleEndian::write_u64(&mut buffer[8..16], self.size); - LittleEndian::write_u32(&mut buffer[16..20], self.records); + buffer[0..8].copy_from_slice(&self.magic.to_le_bytes()); + buffer[8..16].copy_from_slice(&self.size.to_le_bytes()); + buffer[16..20].copy_from_slice(&self.records.to_le_bytes()); buffer[20..].copy_from_slice(&self.reserved); writer.write_all(&buffer)?; Ok(()) @@ -548,12 +469,12 @@ impl BlockHeader { /// /// * `ReadError::InvalidBlockMagicNumber` - If the magic number doesn't match "BLOCKSEQ" pub fn from_bytes(buffer: &[u8; SIZE_BLOCK_HEADER]) -> Result { - let magic = LittleEndian::read_u64(&buffer[0..8]); + let magic = read_u64_le(&buffer[0..8]); if magic != BLOCK_MAGIC { return Err(ReadError::InvalidBlockMagicNumber(magic, 0).into()); } - let size = LittleEndian::read_u64(&buffer[8..16]); - let records = LittleEndian::read_u32(&buffer[16..20]); + let size = read_u64_le(&buffer[8..16]); + let records = read_u32_le(&buffer[16..20]); Ok(Self::new(size, records)) } @@ -592,19 +513,6 @@ mod tests { // ==================== FileHeader Constructor Tests ==================== - #[test] - fn test_file_header_new() { - let header = FileHeader::new(true, true, true, BitSize::Four, true, true); - assert_eq!(header.block, BLOCK_SIZE); - assert!(header.qual); - assert!(header.compressed); - assert!(header.paired); - assert_eq!(header.bits, BitSize::Four); - assert!(header.headers); - assert!(header.flags); - assert!(header.is_paired()); - } - #[test] fn test_file_header_default() { let header = FileHeader::default(); @@ -612,18 +520,16 @@ mod tests { assert!(!header.is_paired()); } - #[test] - fn test_set_bitsize() { - let mut header = FileHeader::default(); - header.set_bitsize(BitSize::Four); - assert_eq!(header.bits, BitSize::Four); - } - // ==================== FileHeader from_bytes/from_reader Tests ==================== #[test] fn test_file_header_roundtrip() { - let header = FileHeader::new(true, false, true, BitSize::Two, true, true); + let header = FileHeaderBuilder::new() + .qual(true) + .paired(true) + .headers(true) + .flags(true) + .build(); let mut buffer = Vec::new(); header.write_bytes(&mut buffer).unwrap(); let mut cursor = std::io::Cursor::new(buffer); @@ -633,7 +539,7 @@ mod tests { #[test] fn test_file_header_from_bytes_four_bit() { - let header = FileHeader::new(false, false, false, BitSize::Four, false, false); + let header = FileHeaderBuilder::new().bitsize(BitSize::Four).build(); let mut buffer = [0u8; SIZE_HEADER]; { let mut cursor = std::io::Cursor::new(&mut buffer[..]); diff --git a/src/vbq/index.rs b/src/vbq/index.rs index 7b2c8e1..c530dca 100644 --- a/src/vbq/index.rs +++ b/src/vbq/index.rs @@ -32,19 +32,11 @@ //! - Cumulative record counts changed from `u32` to `u64` //! - Support for files with more than 4 billion records -use std::{ - fs::File, - io::{Cursor, Read, Write}, - path::Path, -}; +use std::io::{Cursor, Read, Write}; -use byteorder::{ByteOrder, LittleEndian}; +use crate::utils::{read_u32_le, read_u64_le}; use zstd::{Decoder, Encoder}; -use super::{ - BlockHeader, FileHeader, - header::{SIZE_BLOCK_HEADER, SIZE_HEADER}, -}; use crate::error::{IndexError, Result}; /// Size of `BlockRange` in bytes @@ -182,27 +174,16 @@ impl BlockRange { /// * `Err(_)` - If an error occurred during writing pub fn write_bytes(&self, writer: &mut W) -> Result<()> { let mut buf = [0; SIZE_BLOCK_RANGE]; - LittleEndian::write_u64(&mut buf[0..8], self.start_offset); - LittleEndian::write_u64(&mut buf[8..16], self.len); - LittleEndian::write_u32(&mut buf[16..20], self.block_records); - LittleEndian::write_u64(&mut buf[20..28], self.cumulative_records); + buf[0..8].copy_from_slice(&self.start_offset.to_le_bytes()); + buf[8..16].copy_from_slice(&self.len.to_le_bytes()); + buf[16..20].copy_from_slice(&self.block_records.to_le_bytes()); + buf[20..28].copy_from_slice(&self.cumulative_records.to_le_bytes()); buf[28..].copy_from_slice(&self.reservation); writer.write_all(&buf)?; Ok(()) } - /// Deserializes a `BlockRange` from a fixed-size buffer - /// - /// This method deserializes a `BlockRange` from a 32-byte buffer in the format - /// used by `write_bytes`. It's typically used when reading an index file. - /// - /// # Parameters - /// - /// * `buffer` - A fixed-size buffer containing a serialized `BlockRange` - /// - /// # Returns - /// - /// A new `BlockRange` with the values read from the buffer + /// Deserializes a `BlockRange` from a slice of bytes /// /// # Format /// @@ -212,39 +193,19 @@ impl BlockRange { /// - Bytes 16-19: `block_records` (u32, little endian) /// - Bytes 20-27: `cumulative_records` (u64, little endian) /// - Bytes 28-31: reservation (ignored, default value used) - #[must_use] - pub fn from_exact(buffer: &[u8; SIZE_BLOCK_RANGE]) -> Self { - Self { - start_offset: LittleEndian::read_u64(&buffer[0..8]), - len: LittleEndian::read_u64(&buffer[8..16]), - block_records: LittleEndian::read_u32(&buffer[16..20]), - cumulative_records: LittleEndian::read_u64(&buffer[20..28]), - reservation: INDEX_RESERVATION, - } - } - - /// Deserializes a `BlockRange` from a slice of bytes - /// - /// This is a convenience method that copies the first 32 bytes from the provided slice - /// into a fixed-size buffer and then calls `from_exact`. It's useful when reading from - /// a larger buffer that contains multiple serialized `BlockRange` instances. - /// - /// # Parameters - /// - /// * `buffer` - A slice containing at least 32 bytes with a serialized `BlockRange` - /// - /// # Returns - /// - /// A new `BlockRange` with the values read from the buffer /// /// # Panics /// - /// This method will panic if the buffer is less than 32 bytes long. + /// Panics if the buffer is less than 28 bytes long. #[must_use] pub fn from_bytes(buffer: &[u8]) -> Self { - let mut buf = [0; SIZE_BLOCK_RANGE]; - buf.copy_from_slice(buffer); - Self::from_exact(&buf) + Self { + start_offset: read_u64_le(&buffer[0..8]), + len: read_u64_le(&buffer[8..16]), + block_records: read_u32_le(&buffer[16..20]), + cumulative_records: read_u64_le(&buffer[20..28]), + reservation: INDEX_RESERVATION, + } } } @@ -257,22 +218,11 @@ impl BlockRange { /// The header has a fixed size of 32 bytes to ensure compatibility across versions. #[derive(Debug, Clone, Copy)] pub struct IndexHeader { - /// Magic number to designate the index file ("VBQINDEX" in ASCII) - /// - /// This is used to verify that a file is indeed a VBQ index file. - /// (8 bytes in serialized form) - magic: u64, - /// Total size of the indexed VBQ file in bytes /// - /// This is used to verify that the index matches the file it references. - /// (8 bytes in serialized form) + /// The serialized form also carries the `INDEX_MAGIC` magic number (8 bytes) + /// and 16 reserved bytes. bytes: u64, - - /// Reserved bytes for future extensions - /// - /// (16 bytes in serialized form) - reserved: [u8; INDEX_HEADER_SIZE - 16], } impl IndexHeader { /// Creates a new index header for a VBQ file of the specified size @@ -285,11 +235,7 @@ impl IndexHeader { /// /// A new `IndexHeader` instance with the appropriate magic number and size pub fn new(bytes: u64) -> Self { - Self { - magic: INDEX_MAGIC, - bytes, - reserved: [42; INDEX_HEADER_SIZE - 16], - } + Self { bytes } } /// Reads an index header from the provided reader /// @@ -312,30 +258,16 @@ impl IndexHeader { /// - Bytes 0-7: magic number (u64, little endian, must be `INDEX_MAGIC`) /// - Bytes 8-15: file size in bytes (u64, little endian) /// - Bytes 16-31: reserved for future extensions - pub fn from_reader(reader: &mut R) -> Result { - let mut buffer = [0; INDEX_HEADER_SIZE]; - reader.read_exact(&mut buffer)?; - let magic = LittleEndian::read_u64(&buffer[0..8]); - let bytes = LittleEndian::read_u64(&buffer[8..16]); - let Ok(reserved) = buffer[16..INDEX_HEADER_SIZE].try_into() else { - return Err(IndexError::InvalidReservedBytes.into()); - }; + pub fn from_bytes(buffer: &[u8]) -> Result { + let magic = read_u64_le(&buffer[0..8]); if magic != INDEX_MAGIC { return Err(IndexError::InvalidMagicNumber(magic).into()); } Ok(Self { - magic, - bytes, - reserved, + bytes: read_u64_le(&buffer[8..16]), }) } - pub fn from_bytes(bytes: &[u8]) -> Result { - let mut buffer = [0; INDEX_HEADER_SIZE]; - buffer.copy_from_slice(&bytes[..INDEX_HEADER_SIZE]); - Self::from_reader(&mut Cursor::new(buffer)) - } - /// Serializes the index header to a binary format and writes it to the provided writer /// /// This method serializes the `IndexHeader` to a fixed-size 32-byte structure and @@ -356,11 +288,10 @@ impl IndexHeader { /// - Bytes 0-7: magic number (u64, little endian) /// - Bytes 8-15: file size in bytes (u64, little endian) /// - Bytes 16-31: reserved for future extensions - pub fn write_bytes(&self, writer: &mut W) -> Result<()> { - let mut buffer = [0; INDEX_HEADER_SIZE]; - LittleEndian::write_u64(&mut buffer[0..8], self.magic); - LittleEndian::write_u64(&mut buffer[8..16], self.bytes); - buffer[16..].copy_from_slice(&self.reserved); + pub fn write_bytes(self, writer: &mut W) -> Result<()> { + let mut buffer = [42; INDEX_HEADER_SIZE]; + buffer[0..8].copy_from_slice(&INDEX_MAGIC.to_le_bytes()); + buffer[8..16].copy_from_slice(&self.bytes.to_le_bytes()); writer.write_all(&buffer)?; Ok(()) } @@ -374,22 +305,18 @@ impl IndexHeader { /// the file. /// /// The index is embedded at the end of VBQ files and can be loaded using -/// `MmapReader::load_index()` or created by scanning a VBQ file using -/// `BlockIndex::from_vbq()`. Once loaded, it provides information about block +/// `MmapReader::load_index()`. Once loaded, it provides information about block /// locations, sizes, and record counts. /// /// # Examples /// /// ```rust,no_run -/// use binseq::vbq::{BlockIndex, MmapReader}; +/// use binseq::vbq::MmapReader; /// use std::path::Path; /// -/// // Create an index from a VBQ file -/// let vbq_path = Path::new("example.vbq"); -/// let index = BlockIndex::from_vbq(vbq_path).unwrap(); -/// -/// // Use the index with a reader for parallel processing -/// let reader = MmapReader::new(vbq_path).unwrap(); +/// // Load the embedded index from a VBQ file +/// let reader = MmapReader::new(Path::new("example.vbq")).unwrap(); +/// let index = reader.load_index().unwrap(); /// println!("File contains {} blocks", index.n_blocks()); /// ``` #[derive(Debug, Clone)] @@ -480,85 +407,6 @@ impl BlockIndex { self.ranges.push(range); } - /// Creates a new index by scanning a VBQ file - /// - /// This method memory-maps the specified VBQ file and scans it block by block - /// to create an index. This is primarily used internally when embedding the index - /// into VBQ files during the write process. - /// - /// # Parameters - /// - /// * `path` - Path to the VBQ file to index - /// - /// # Returns - /// - /// * `Ok(Self)` - A new `BlockIndex` containing information about all blocks in the file - /// * `Err(_)` - If an error occurred during file opening, validation, or scanning - /// - /// # Examples - /// - /// ```rust,no_run - /// use binseq::vbq::BlockIndex; - /// use std::path::Path; - /// - /// // Create an index from a VBQ file - /// let index = BlockIndex::from_vbq(Path::new("example.vbq")).unwrap(); - /// - /// // Get statistics about the file - /// println!("File contains {} blocks", index.n_blocks()); - /// - /// // Analyze the record distribution - /// if let Some(last_range) = index.ranges().last() { - /// println!("Total records: {}", last_range.cumulative_records); - /// println!("Average records per block: {}", - /// last_range.cumulative_records as f64 / index.n_blocks() as f64); - /// } - /// ``` - /// - /// # Notes - /// - /// This method uses memory mapping for efficiency, which allows the operating system - /// to load only the needed portions of the file into memory as they are accessed. - pub fn from_vbq>(path: P) -> Result { - let file = File::open(path)?; - let mmap = unsafe { memmap2::Mmap::map(&file)? }; - let file_size = mmap.len(); - - // Read header from mapped memory (unused but checks for validity) - let _header = { - let mut header_bytes = [0u8; SIZE_HEADER]; - header_bytes.copy_from_slice(&mmap[..SIZE_HEADER]); - FileHeader::from_bytes(&header_bytes)? - }; - - // Initialize position after the header - let mut pos = SIZE_HEADER; - - // Initialize the collection - let index_header = IndexHeader::new(file_size as u64); - let mut index = BlockIndex::new(index_header); - - // Find all block headers - let mut record_total = 0; - while pos < mmap.len() { - let block_header = { - let mut header_bytes = [0u8; SIZE_BLOCK_HEADER]; - header_bytes.copy_from_slice(&mmap[pos..pos + SIZE_BLOCK_HEADER]); - BlockHeader::from_bytes(&header_bytes)? - }; - index.add_range(BlockRange::new( - pos as u64, - block_header.size, - block_header.records, - record_total, - )); - pos += SIZE_BLOCK_HEADER + block_header.size as usize; - record_total += u64::from(block_header.records); - } - - Ok(index) - } - pub fn from_bytes(bytes: &[u8]) -> Result { let index_header = IndexHeader::from_bytes(bytes)?; let buffer = { @@ -610,13 +458,14 @@ impl BlockIndex { &self.ranges } + /// Prints a tab-separated summary of each block range to stdout pub fn pprint(&self) { - self.ranges.iter().for_each(|range| { + for range in &self.ranges { println!( "{}\t{}\t{}\t{}", range.start_offset, range.len, range.block_records, range.cumulative_records ); - }); + } } /// Returns the total number of records in the dataset @@ -634,69 +483,6 @@ impl BlockIndex { mod tests { use super::*; - /// Writes a minimal raw VBQ file (file header + block headers/data, with - /// **no** embedded index) to `path`, suitable for `BlockIndex::from_vbq`. - /// - /// `from_vbq` scans block-by-block from just after the file header to - /// EOF, so it requires the file to end exactly at the last block's data - /// -- unlike files produced by `vbq::Writer`, which always append an - /// embedded index on `finish()` that `from_vbq` cannot parse as a block. - fn write_raw_vbq_file(path: &str, blocks: &[(u64, u32)]) { - let mut buffer = Vec::new(); - FileHeader::default().write_bytes(&mut buffer).unwrap(); - for &(size, records) in blocks { - BlockHeader::new(size, records) - .write_bytes(&mut buffer) - .unwrap(); - buffer.extend(std::iter::repeat_n(0u8, size as usize)); - } - std::fs::write(path, buffer).unwrap(); - } - - // ==================== BlockIndex::from_vbq Tests ==================== - - #[test] - fn test_from_vbq() { - let path = "test_index_from_vbq.vbq"; - write_raw_vbq_file(path, &[(64, 3), (128, 5)]); - - let index = BlockIndex::from_vbq(path).unwrap(); - std::fs::remove_file(path).unwrap(); - - assert_eq!(index.n_blocks(), 2); - assert_eq!(index.num_records(), 8); - assert_eq!(index.ranges()[0].cumulative_records, 0); - assert_eq!(index.ranges()[1].cumulative_records, 3); - } - - #[test] - fn test_from_vbq_nonexistent_file() { - let result = BlockIndex::from_vbq("./data/does_not_exist.vbq"); - assert!(result.is_err()); - } - - #[test] - fn test_n_blocks() { - let path = "test_index_n_blocks.vbq"; - write_raw_vbq_file(path, &[(32, 1)]); - - let index = BlockIndex::from_vbq(path).unwrap(); - std::fs::remove_file(path).unwrap(); - - assert_eq!(index.n_blocks(), index.ranges().len()); - } - - #[test] - fn test_pprint() { - let path = "test_index_pprint.vbq"; - write_raw_vbq_file(path, &[(32, 1)]); - - // Just verify it doesn't panic - let index = BlockIndex::from_vbq(path).unwrap(); - std::fs::remove_file(path).unwrap(); - index.pprint(); - } - #[test] fn test_num_records_empty_index() { let index = BlockIndex::new(IndexHeader::new(0)); @@ -707,10 +493,10 @@ mod tests { // ==================== IndexHeader Tests ==================== #[test] - fn test_index_header_from_reader_invalid_magic() { + fn test_index_header_from_bytes_invalid_magic() { let mut buffer = [0u8; INDEX_HEADER_SIZE]; - LittleEndian::write_u64(&mut buffer[0..8], 0xDEAD_BEEF); - let result = IndexHeader::from_reader(&mut Cursor::new(buffer)); + buffer[0..8].copy_from_slice(&0xDEAD_BEEF_u64.to_le_bytes()); + let result = IndexHeader::from_bytes(&buffer); assert!(result.is_err()); } @@ -719,9 +505,8 @@ mod tests { let header = IndexHeader::new(12345); let mut buffer = Vec::new(); header.write_bytes(&mut buffer).unwrap(); - let parsed = IndexHeader::from_reader(&mut Cursor::new(buffer)).unwrap(); + let parsed = IndexHeader::from_bytes(&buffer).unwrap(); assert_eq!(parsed.bytes, 12345); - assert_eq!(parsed.magic, INDEX_MAGIC); } // ==================== BlockIndex round-trip through write_bytes/from_bytes ==================== diff --git a/src/vbq/reader.rs b/src/vbq/reader.rs index ef5b931..a4b2205 100644 --- a/src/vbq/reader.rs +++ b/src/vbq/reader.rs @@ -54,8 +54,8 @@ use std::ops::Range; use std::path::Path; use std::sync::Arc; +use crate::utils::read_u64_le; use bitnuc_deprec::BitSize; -use byteorder::{ByteOrder, LittleEndian}; use memmap2::Mmap; use zstd::zstd_safe; @@ -82,16 +82,22 @@ use crate::{ /// # Returns /// /// The number of 64-bit words required to encode the sequence -fn encoded_sequence_len(len: u64, bitsize: BitSize) -> usize { +/// Number of bases packed into each u64 word for the given bitsize +fn bases_per_word(bitsize: BitSize) -> usize { match bitsize { - BitSize::Two => len.div_ceil(32) as usize, - BitSize::Four => len.div_ceil(16) as usize, + BitSize::Two => 32, + BitSize::Four => 16, } } +/// Number of u64 words needed to store `len` bases at the given bitsize +fn encoded_sequence_len(len: u64, bitsize: BitSize) -> usize { + len.div_ceil(bases_per_word(bitsize) as u64) as usize +} + /// Represents a span (offset, length) into a buffer #[derive(Clone, Copy, Debug, Default)] -pub struct Span { +pub(crate) struct Span { offset: usize, len: usize, } @@ -370,7 +376,7 @@ impl RecordBlock { // Read flag let flag = if has_flags { - let flag = LittleEndian::read_u64(&bytes[pos..pos + 8]); + let flag = read_u64_le(&bytes[pos..pos + 8]); pos += 8; Some(flag) } else { @@ -378,9 +384,9 @@ impl RecordBlock { }; // Read lengths - let slen = LittleEndian::read_u64(&bytes[pos..pos + 8]); + let slen = read_u64_le(&bytes[pos..pos + 8]); pos += 8; - let xlen = LittleEndian::read_u64(&bytes[pos..pos + 8]); + let xlen = read_u64_le(&bytes[pos..pos + 8]); pos += 8; // Check for end of records @@ -395,7 +401,7 @@ impl RecordBlock { // Primary sequence - store span into sequences Vec let s_seq_span = Span::new(self.sequences.len(), s_seq_words); for _ in 0..s_seq_words { - let val = LittleEndian::read_u64(&bytes[pos..pos + 8]); + let val = read_u64_le(&bytes[pos..pos + 8]); self.sequences.push(val); pos += 8; } @@ -411,7 +417,7 @@ impl RecordBlock { // Primary header - store span into rbuf let s_header_span = if has_header { - let header_len = LittleEndian::read_u64(&bytes[pos..pos + 8]) as usize; + let header_len = read_u64_le(&bytes[pos..pos + 8]) as usize; pos += 8; let span = Span::new(pos, header_len); pos += header_len; @@ -423,7 +429,7 @@ impl RecordBlock { // Extended sequence - store span into sequences Vec let x_seq_span = Span::new(self.sequences.len(), x_seq_words); for _ in 0..x_seq_words { - let val = LittleEndian::read_u64(&bytes[pos..pos + 8]); + let val = read_u64_le(&bytes[pos..pos + 8]); self.sequences.push(val); pos += 8; } @@ -439,7 +445,7 @@ impl RecordBlock { // Extended header - store span into rbuf let x_header_span = if has_header && xlen > 0 { - let header_len = LittleEndian::read_u64(&bytes[pos..pos + 8]) as usize; + let header_len = read_u64_le(&bytes[pos..pos + 8]) as usize; pos += 8; let span = Span::new(pos, header_len); pos += header_len; @@ -496,24 +502,21 @@ impl RecordBlock { Ok(()) } - /// Get decoded primary sequence for a record by index - #[must_use] - pub fn get_decoded_s(&self, record_idx: usize) -> Option<&[u8]> { - let meta = self.records.get(record_idx)?; + /// Slice the decoded buffer for a record given a word span and base length + fn decoded(&self, word_offset: usize, len: usize) -> Option<&[u8]> { if self.dbuf.is_empty() { return None; } - - let bases_per_word = match self.bitsize { - BitSize::Two => 32, - BitSize::Four => 16, - }; - // Calculate offset in decoded buffer (accounting for padding) - let offset = meta.s_seq_span.offset * bases_per_word; - let len = meta.slen as usize; + let offset = word_offset * bases_per_word(self.bitsize); + self.dbuf.get(offset..offset + len) + } - Some(&self.dbuf[offset..offset + len]) + /// Get decoded primary sequence for a record by index + #[must_use] + pub fn get_decoded_s(&self, record_idx: usize) -> Option<&[u8]> { + let meta = self.records.get(record_idx)?; + self.decoded(meta.s_seq_span.offset, meta.slen as usize) } /// Get decoded extended sequence for a record by index @@ -523,19 +526,7 @@ impl RecordBlock { if meta.xlen == 0 { return Some(&[]); } - if self.dbuf.is_empty() { - return None; - } - - let bases_per_word = match self.bitsize { - BitSize::Two => 32, - BitSize::Four => 16, - }; - - let offset = meta.x_seq_span.offset * bases_per_word; - let len = meta.xlen as usize; - - Some(&self.dbuf[offset..offset + len]) + self.decoded(meta.x_seq_span.offset, meta.xlen as usize) } } @@ -1077,13 +1068,13 @@ impl MmapReader { let start_pos_index_size = start_pos_magic - 8; // Validate the magic number - let magic = LittleEndian::read_u64(&self.mmap[start_pos_magic..]); + let magic = read_u64_le(&self.mmap[start_pos_magic..]); if magic != INDEX_END_MAGIC { return Err(ReadError::MissingIndexEndMagic.into()); } // Get the index size - let index_size = LittleEndian::read_u64(&self.mmap[start_pos_index_size..start_pos_magic]); + let index_size = read_u64_le(&self.mmap[start_pos_index_size..start_pos_magic]); // Determine the start position of the index bytes let start_pos_index = start_pos_index_size - index_size as usize; @@ -1236,11 +1227,7 @@ impl ParallelReader for MmapReader { range: Range, ) -> Result<()> { // Calculate the number of threads to use - let num_threads = if num_threads == 0 { - num_cpus::get() - } else { - num_threads.min(num_cpus::get()) - }; + let num_threads = crate::parallel::clamp_threads(num_threads); // Generate or load the index first let index = self.load_index()?; diff --git a/src/vbq/writer.rs b/src/vbq/writer.rs index 75e0d8b..e5641be 100644 --- a/src/vbq/writer.rs +++ b/src/vbq/writer.rs @@ -64,16 +64,13 @@ use std::io::Write; -use bitnuc_deprec::BitSize; -use byteorder::{LittleEndian, WriteBytesExt}; -use rand::SeedableRng; -use rand::rngs::SmallRng; use zstd::stream::copy_encode; use super::header::{BlockHeader, FileHeader}; use crate::SequencingRecord; +use crate::encoder::Encoder; use crate::error::{Result, WriteError}; -use crate::policy::{Policy, RNG_SEED}; +use crate::policy::Policy; use crate::vbq::header::{SIZE_BLOCK_HEADER, SIZE_HEADER}; use crate::vbq::index::{INDEX_END_MAGIC, IndexHeader}; use crate::vbq::{BlockIndex, BlockRange}; @@ -393,7 +390,7 @@ impl Writer { /// Returns the N-policy of the writer pub fn policy(&self) -> Policy { - self.encoder.policy + self.encoder.policy() } /// Checks if the writer is configured for quality scores @@ -433,47 +430,45 @@ impl Writer { self.header.headers } - #[deprecated(note = "use `push` method with SequencingRecord instead")] - pub fn write_record( - &mut self, - flag: Option, - header: Option<&[u8]>, - sequence: &[u8], - quality: Option<&[u8]>, - ) -> Result { - let record = SequencingRecord::new(sequence, quality, header, None, None, None, flag); - self.push(record) + /// Flush the current block and record its range in the index. + /// + /// Free-standing over the individual fields so callers holding a borrow of + /// `self.encoder` (the encoded record buffers) can still flush. + fn flush_block_split( + inner: &mut W, + cblock: &mut BlockWriter, + ranges: &mut Vec, + bytes_written: &mut usize, + records_written: &mut usize, + ) -> Result<()> { + let block_header = cblock.flush(inner)?; + ranges.push(BlockRange::new( + *bytes_written as u64, + block_header.size, + block_header.records, + *records_written as u64, + )); + *bytes_written += block_header.size_with_header(); + *records_written += block_header.records as usize; + Ok(()) } - #[deprecated(note = "use `push` method with SequencingRecord instead")] - #[allow(clippy::too_many_arguments)] - pub fn write_paired_record( - &mut self, - flag: Option, - s_header: Option<&[u8]>, - s_sequence: &[u8], - s_qual: Option<&[u8]>, - x_header: Option<&[u8]>, - x_sequence: &[u8], - x_qual: Option<&[u8]>, - ) -> Result { - let record = SequencingRecord::new( - s_sequence, - s_qual, - s_header, - Some(x_sequence), - x_qual, - x_header, - flag, - ); - self.push(record) + /// Flush the current block and record its range in the index + fn flush_block(&mut self) -> Result<()> { + Self::flush_block_split( + &mut self.inner, + &mut self.cblock, + &mut self.ranges, + &mut self.bytes_written, + &mut self.records_written, + ) } /// Writes a record using the unified [`SequencingRecord`] API /// /// This method provides a consistent interface with BQ and CBQ writers. - /// It automatically routes to either `write_record` or `write_paired_record` - /// based on whether the record contains paired data. + /// It automatically routes to the single or paired write path based on + /// whether the record contains paired data. /// /// # Arguments /// @@ -552,46 +547,35 @@ impl Writer { self.header.bits, ); - if self.header.is_paired() { - // encode the sequences - if let Some((sbuffer, xbuffer)) = self - .encoder + // encode the sequence(s); a `None` means the record was skipped by policy + let encoded = if self.header.is_paired() { + self.encoder .encode_paired(record.s_seq, record.x_seq.unwrap_or_default())? - { - if self.cblock.exceeds_block_size(record_size)? { - impl_flush_block( - &mut self.inner, - &mut self.cblock, - &mut self.ranges, - &mut self.bytes_written, - &mut self.records_written, - )?; - } - - self.cblock.write_record(&record, sbuffer, Some(xbuffer))?; - Ok(true) - } else { - Ok(false) - } + .map(|(sbuffer, xbuffer)| (sbuffer, Some(xbuffer))) } else { - // encode the sequence - if let Some(sbuffer) = self.encoder.encode_single(record.s_seq)? { - if self.cblock.exceeds_block_size(record_size)? { - impl_flush_block( - &mut self.inner, - &mut self.cblock, - &mut self.ranges, - &mut self.bytes_written, - &mut self.records_written, - )?; - } - - self.cblock.write_record(&record, sbuffer, None)?; - Ok(true) - } else { - Ok(false) - } + self.encoder + .encode_single(record.s_seq)? + .map(|sbuffer| (sbuffer, None)) + }; + let Some((sbuffer, xbuffer)) = encoded else { + return Ok(false); + }; + + // flush the current block if this record would overflow it; done only + // once we know the record will actually be written, so policy-skipped + // records never fragment blocks (split-borrow: `encoded` holds the encoder) + if self.cblock.exceeds_block_size(record_size)? { + Self::flush_block_split( + &mut self.inner, + &mut self.cblock, + &mut self.ranges, + &mut self.bytes_written, + &mut self.records_written, + )?; } + + self.cblock.write_record(&record, sbuffer, xbuffer)?; + Ok(true) } /// Finishes writing and flushes all data to the underlying writer @@ -632,13 +616,7 @@ impl Writer { /// ``` pub fn finish(&mut self) -> Result<()> { // Flush any remaining data in the current block - impl_flush_block( - &mut self.inner, - &mut self.cblock, - &mut self.ranges, - &mut self.bytes_written, - &mut self.records_written, - )?; + self.flush_block()?; self.inner.flush()?; // Always write the index - this is critical for VBQ file validity @@ -764,7 +742,7 @@ impl Writer { Ok(()) } - pub fn write_index(&mut self) -> Result<()> { + fn write_index(&mut self) -> Result<()> { // Build the index let index_header = IndexHeader::new(self.bytes_written as u64); let block_index = BlockIndex { @@ -783,35 +761,15 @@ impl Writer { self.inner.write_all(&buffer)?; // Write the number of bytes written to the index - self.inner.write_u64::(n_bytes)?; + self.inner.write_all(&n_bytes.to_le_bytes())?; // Write the index footer magic - self.inner.write_u64::(INDEX_END_MAGIC)?; + self.inner.write_all(&INDEX_END_MAGIC.to_le_bytes())?; Ok(()) } } -fn impl_flush_block( - writer: &mut W, - cblock: &mut BlockWriter, - ranges: &mut Vec, - bytes_written: &mut usize, - records_written: &mut usize, -) -> Result<()> { - let block_header = cblock.flush(writer)?; - let range = BlockRange::new( - *bytes_written as u64, - block_header.size, - block_header.records, - *records_written as u64, - ); - ranges.push(range); - *bytes_written += block_header.size_with_header(); - *records_written += block_header.records as usize; - Ok(()) -} - impl Drop for Writer { fn drop(&mut self) { self.finish().expect("Writer: Failed to finish writing"); @@ -938,20 +896,20 @@ impl BlockWriter { } fn write_flag(&mut self, flag: u64) -> Result<()> { - self.ubuf.write_u64::(flag)?; + self.ubuf.write_all(&flag.to_le_bytes())?; self.pos += 8; Ok(()) } fn write_length(&mut self, length: u64) -> Result<()> { - self.ubuf.write_u64::(length)?; + self.ubuf.write_all(&length.to_le_bytes())?; self.pos += 8; Ok(()) } fn write_buffer(&mut self, ebuf: &[u64]) -> Result<()> { ebuf.iter() - .try_for_each(|&x| self.ubuf.write_u64::(x))?; + .try_for_each(|&x| self.ubuf.write_all(&x.to_le_bytes()))?; self.pos += 8 * ebuf.len(); Ok(()) } @@ -1111,105 +1069,42 @@ impl BlockWriter { } } -/// Encapsulates the logic for encoding sequences into a binary format. -#[derive(Clone)] -pub struct Encoder { - /// Bitsize of the nucleotides - bitsize: BitSize, - - /// Reusable buffers for all nucleotides (written as 2-bit after conversion) - sbuffer: Vec, - xbuffer: Vec, - - /// Reusable buffers for invalid nucleotide sequences - s_ibuf: Vec, - x_ibuf: Vec, - - /// Invalid Nucleotide Policy - policy: Policy, - - /// Random Number Generator - rng: SmallRng, -} - -impl Encoder { - /// Initialize a new encoder with the given policy. - pub fn with_policy(bitsize: BitSize, policy: Policy) -> Self { - Self { - bitsize, - policy, - sbuffer: Vec::default(), - xbuffer: Vec::default(), - s_ibuf: Vec::default(), - x_ibuf: Vec::default(), - rng: SmallRng::seed_from_u64(RNG_SEED), - } - } - - /// Encodes a single sequence as 2-bit. - /// - /// Will return `None` if the sequence is invalid and the policy does not allow correction. - pub fn encode_single(&mut self, primary: &[u8]) -> Result> { - // Fill the buffer with the bit representation of the nucleotides - self.clear(); - if self.bitsize.encode(primary, &mut self.sbuffer).is_err() { - self.clear(); - if self - .policy - .handle(primary, &mut self.s_ibuf, &mut self.rng)? - { - self.bitsize.encode(&self.s_ibuf, &mut self.sbuffer)?; - } else { - return Ok(None); - } - } - Ok(Some(&self.sbuffer)) - } - - /// Encodes a pair of sequences as 2-bit. - /// - /// Will return `None` if either sequence is invalid and the policy does not allow correction. - pub fn encode_paired( - &mut self, - primary: &[u8], - extended: &[u8], - ) -> Result> { - self.clear(); - if self.bitsize.encode(primary, &mut self.sbuffer).is_err() - || self.bitsize.encode(extended, &mut self.xbuffer).is_err() - { - self.clear(); - if self - .policy - .handle(primary, &mut self.s_ibuf, &mut self.rng)? - && self - .policy - .handle(extended, &mut self.x_ibuf, &mut self.rng)? - { - self.bitsize.encode(&self.s_ibuf, &mut self.sbuffer)?; - self.bitsize.encode(&self.x_ibuf, &mut self.xbuffer)?; - } else { - return Ok(None); - } - } - Ok(Some((&self.sbuffer, &self.xbuffer))) - } - - /// Clear all buffers and reset the encoder. - pub fn clear(&mut self) { - self.sbuffer.clear(); - self.xbuffer.clear(); - self.s_ibuf.clear(); - self.x_ibuf.clear(); - } -} - #[cfg(test)] mod tests { use super::*; use crate::SequencingRecordBuilder; + use crate::utils::read_u64_le; use crate::vbq::{FileHeaderBuilder, header::SIZE_HEADER}; + #[test] + fn test_policy_skipped_records_do_not_fragment_blocks() -> super::Result<()> { + // Two writers fed the same valid records must produce identical output, + // even when one is also fed invalid (policy-skipped) records that would + // overflow the current block: a skipped record must never trigger a flush. + let header = FileHeaderBuilder::new().block(2048).build(); + let mut plain = WriterBuilder::default().header(header).build(Vec::new())?; + let mut skipped = WriterBuilder::default().header(header).build(Vec::new())?; + + let valid = b"ACGT".repeat(25); // 100 bp + let invalid = b"N".repeat(300); // always skipped by IgnoreSequence + for _ in 0..200 { + let invalid_record = SequencingRecordBuilder::default().s_seq(&invalid).build()?; + assert!(!skipped.push(invalid_record)?); + + let valid_record = SequencingRecordBuilder::default().s_seq(&valid).build()?; + assert!(plain.push(valid_record)?); + let valid_record = SequencingRecordBuilder::default().s_seq(&valid).build()?; + assert!(skipped.push(valid_record)?); + } + plain.finish()?; + skipped.finish()?; + + assert!(plain.ranges.len() > 1, "test should span multiple blocks"); + assert_eq!(plain.ranges.len(), skipped.ranges.len()); + assert_eq!(plain.inner, skipped.inner); + Ok(()) + } + #[test] fn test_headless_writer() -> super::Result<()> { let writer = WriterBuilder::default().headless(true).build(Vec::new())?; @@ -1519,7 +1414,6 @@ mod tests { #[test] fn test_index_always_written_on_finish() -> super::Result<()> { use crate::vbq::index::INDEX_END_MAGIC; - use byteorder::{ByteOrder, LittleEndian}; // Create a writer with some records let header = FileHeaderBuilder::new().build(); @@ -1543,7 +1437,7 @@ mod tests { // Verify the file ends with the index magic number assert!(bytes.len() >= 8, "File is too short to contain index"); let magic_offset = bytes.len() - 8; - let magic = LittleEndian::read_u64(&bytes[magic_offset..]); + let magic = read_u64_le(&bytes[magic_offset..]); assert_eq!( magic, INDEX_END_MAGIC, "Index magic number not found at end of file" @@ -1552,7 +1446,7 @@ mod tests { // Verify we can read the index size assert!(bytes.len() >= 16, "File is too short to contain index size"); let size_offset = bytes.len() - 16; - let index_size = LittleEndian::read_u64(&bytes[size_offset..size_offset + 8]); + let index_size = read_u64_le(&bytes[size_offset..size_offset + 8]); assert!(index_size > 0, "Index size should be greater than 0"); // Verify the index size makes sense (should be less than total file size) @@ -1567,7 +1461,6 @@ mod tests { #[test] fn test_finish_idempotent() -> super::Result<()> { use crate::vbq::index::INDEX_END_MAGIC; - use byteorder::{ByteOrder, LittleEndian}; // Create a writer let header = FileHeaderBuilder::new().build(); @@ -1599,7 +1492,7 @@ mod tests { // Verify only one index magic number at the end let bytes = &writer.inner; let magic_offset = bytes.len() - 8; - let magic = LittleEndian::read_u64(&bytes[magic_offset..]); + let magic = read_u64_le(&bytes[magic_offset..]); assert_eq!(magic, INDEX_END_MAGIC); Ok(()) diff --git a/src/write.rs b/src/write.rs index 30dd1c2..f4b072f 100644 --- a/src/write.rs +++ b/src/write.rs @@ -268,13 +268,7 @@ impl BinseqWriterBuilder { bitsize: Some(header.bits), paired: header.is_paired(), flags: header.flags, - compression: false, - headers: false, - quality: false, - compression_level: None, - block_size: None, - headless: false, - policy: None, + ..Self::new(Format::Bq) } } @@ -283,8 +277,6 @@ impl BinseqWriterBuilder { pub fn from_vbq_header(header: vbq::FileHeader) -> Self { Self { format: Format::Vbq, - slen: None, - xlen: None, flags: header.flags, quality: header.qual, paired: header.paired, @@ -292,9 +284,7 @@ impl BinseqWriterBuilder { headers: header.headers, compression: header.compressed, block_size: Some(header.block as usize), - policy: None, - compression_level: None, - headless: false, + ..Self::new(Format::Vbq) } } @@ -309,18 +299,13 @@ impl BinseqWriterBuilder { paired: header.is_paired(), block_size: Some(header.block_size as usize), compression_level: Some(header.compression_level as i32), - compression: false, - slen: None, - xlen: None, - bitsize: None, - policy: None, - headless: false, + ..Self::new(Format::Cbq) } } /// Encode FASTX file(s) to BINSEQ format /// - /// This method returns a [`FastxEncoderBuilder`] that allows you to configure + /// This method returns a [`FastxEncoderBuilder`](crate::utils::FastxEncoderBuilder) that allows you to configure /// the input source and threading options before executing the encoding. /// /// This is an alternative to [`build`](Self::build) that directly processes @@ -336,19 +321,19 @@ impl BinseqWriterBuilder { /// use binseq::write::{BinseqWriterBuilder, Format}; /// use std::fs::File; /// - /// // Encode from stdin to VBQ - /// let writer = BinseqWriterBuilder::new(Format::Vbq) + /// // Encode from stdin to CBQ + /// let writer = BinseqWriterBuilder::new(Format::Cbq) /// .quality(true) /// .headers(true) - /// .encode_fastx(File::create("output.vbq")?) + /// .encode_fastx(File::create("output.cbq")?) /// .input_stdin() /// .threads(8) /// .run()?; /// /// // Encode paired-end reads - /// let writer = BinseqWriterBuilder::new(Format::Vbq) + /// let writer = BinseqWriterBuilder::new(Format::Cbq) /// .quality(true) - /// .encode_fastx(File::create("output.vbq")?) + /// .encode_fastx(File::create("output.cbq")?) /// .input_paired("R1.fastq", "R2.fastq") /// .run()?; /// # Ok::<(), binseq::Error>(()) @@ -608,10 +593,8 @@ impl BinseqWriter { /// - There's an I/O error during ingestion pub fn ingest_completed(&mut self, other: &mut BinseqWriter>) -> Result<()> { match (self, other) { - (Self::Bq(dst), BinseqWriter::Bq(src)) => dst.ingest(src), - (Self::Vbq(dst), BinseqWriter::Vbq(src)) => dst.ingest(src), (Self::Cbq(dst), BinseqWriter::Cbq(src)) => dst.ingest_completed(src), - _ => Err(WriteError::FormatMismatch.into()), + (dst, src) => dst.ingest(src), } } } @@ -995,444 +978,83 @@ mod tests { .unwrap() } - // ==================== VBQ Tests ==================== - - #[test] - fn test_vbq_single_minimal_writer_minimal_record() -> Result<()> { - // Writer: single-end, no quality, no headers, no flags - // Record: single-end, no quality, no headers, no flags - // Expected: success (correctly specified) - let mut writer = BinseqWriterBuilder::new(Format::Vbq) - .paired(false) - .quality(false) - .headers(false) - .flags(false) - .build(Cursor::new(Vec::new()))?; - - let record = minimal_single_record(); - assert!(writer.push(record)?); - writer.finish()?; - Ok(()) - } - - #[test] - fn test_vbq_single_minimal_writer_full_record() -> Result<()> { - // Writer: single-end, no quality, no headers, no flags - // Record: single-end, with quality, headers, flags - // Expected: success (over-specified - extra data ignored) - let mut writer = BinseqWriterBuilder::new(Format::Vbq) - .paired(false) - .quality(false) - .headers(false) - .flags(false) - .build(Cursor::new(Vec::new()))?; + // ==================== Writer x Record Specification Matrix ==================== - let record = full_single_record(); - assert!(writer.push(record)?); - writer.finish()?; - Ok(()) - } - - #[test] - fn test_vbq_single_full_writer_minimal_record() -> Result<()> { - // Writer: single-end, with quality, headers, flags - // Record: single-end, no quality, no headers, no flags - // Expected: error (under-specified) - let mut writer = BinseqWriterBuilder::new(Format::Vbq) - .paired(false) - .quality(true) - .headers(true) - .flags(true) - .build(Cursor::new(Vec::new()))?; - - let record = minimal_single_record(); - let result = writer.push(record); - assert!(result.is_err()); - Ok(()) - } - - #[test] - fn test_vbq_single_full_writer_full_record() -> Result<()> { - // Writer: single-end, with quality, headers, flags - // Record: single-end, with quality, headers, flags - // Expected: success (correctly specified) - let mut writer = BinseqWriterBuilder::new(Format::Vbq) - .paired(false) - .quality(true) - .headers(true) - .flags(true) - .build(Cursor::new(Vec::new()))?; - - let record = full_single_record(); - assert!(writer.push(record)?); - writer.finish()?; - Ok(()) - } - - #[test] - fn test_vbq_paired_writer_single_record() -> Result<()> { - // Writer: paired - // Record: single-end - // Expected: error (under-specified - missing R2) - let mut writer = BinseqWriterBuilder::new(Format::Vbq) - .paired(true) - .quality(false) - .headers(false) - .flags(false) - .build(Cursor::new(Vec::new()))?; - - let record = minimal_single_record(); - let result = writer.push(record); - assert!(result.is_err()); - Ok(()) - } - - #[test] - fn test_vbq_single_writer_paired_record() -> Result<()> { - // Writer: single-end - // Record: paired - // Expected: success (over-specified - R2 ignored) - let mut writer = BinseqWriterBuilder::new(Format::Vbq) - .paired(false) - .quality(false) - .headers(false) - .flags(false) - .build(Cursor::new(Vec::new()))?; - - let record = minimal_paired_record(); - assert!(writer.push(record)?); - writer.finish()?; - Ok(()) - } - - #[test] - fn test_vbq_paired_minimal_writer_paired_full_record() -> Result<()> { - // Writer: paired, no quality, no headers, no flags - // Record: paired, with quality, headers, flags - // Expected: success (over-specified) - let mut writer = BinseqWriterBuilder::new(Format::Vbq) - .paired(true) - .quality(false) - .headers(false) - .flags(false) - .build(Cursor::new(Vec::new()))?; - - let record = full_paired_record(); - assert!(writer.push(record)?); - writer.finish()?; - Ok(()) - } - - #[test] - fn test_vbq_paired_full_writer_paired_full_record() -> Result<()> { - // Writer: paired, with quality, headers, flags - // Record: paired, with quality, headers, flags - // Expected: success (correctly specified) - let mut writer = BinseqWriterBuilder::new(Format::Vbq) - .paired(true) - .quality(true) - .headers(true) - .flags(true) - .build(Cursor::new(Vec::new()))?; - - let record = full_paired_record(); - assert!(writer.push(record)?); - writer.finish()?; - Ok(()) - } - - // ==================== CBQ Tests ==================== - - #[test] - fn test_cbq_single_minimal_writer_minimal_record() -> Result<()> { - // Writer: single-end, no quality, no headers, no flags - // Record: single-end, no quality, no headers, no flags - // Expected: success (correctly specified) - let mut writer = BinseqWriterBuilder::new(Format::Cbq) - .paired(false) - .quality(false) - .headers(false) - .flags(false) - .build(Cursor::new(Vec::new()))?; - - let record = minimal_single_record(); - assert!(writer.push(record)?); - writer.finish()?; - Ok(()) - } - - #[test] - fn test_cbq_single_minimal_writer_full_record() -> Result<()> { - // Writer: single-end, no quality, no headers, no flags - // Record: single-end, with quality, headers, flags - // Expected: success (over-specified - extra data ignored) - let mut writer = BinseqWriterBuilder::new(Format::Cbq) - .paired(false) - .quality(false) - .headers(false) - .flags(false) - .build(Cursor::new(Vec::new()))?; - - let record = full_single_record(); - assert!(writer.push(record)?); - writer.finish()?; - Ok(()) - } - - #[test] - fn test_cbq_single_full_writer_minimal_record() -> Result<()> { - // Writer: single-end, with quality, headers, flags - // Record: single-end, no quality, no headers, no flags - // Expected: error (under-specified) - let mut writer = BinseqWriterBuilder::new(Format::Cbq) - .paired(false) - .quality(true) - .headers(true) - .flags(true) - .build(Cursor::new(Vec::new()))?; - - let record = minimal_single_record(); - let result = writer.push(record); - assert!(result.is_err()); - Ok(()) - } + type RecordFn = fn() -> SequencingRecord<'static>; #[test] - fn test_cbq_single_full_writer_full_record() -> Result<()> { - // Writer: single-end, with quality, headers, flags - // Record: single-end, with quality, headers, flags - // Expected: success (correctly specified) - let mut writer = BinseqWriterBuilder::new(Format::Cbq) - .paired(false) - .quality(true) - .headers(true) - .flags(true) - .build(Cursor::new(Vec::new()))?; - - let record = full_single_record(); - assert!(writer.push(record)?); - writer.finish()?; - Ok(()) - } - - #[test] - fn test_cbq_paired_writer_single_record() -> Result<()> { - // Writer: paired - // Record: single-end - // Expected: error (under-specified - missing R2) - let mut writer = BinseqWriterBuilder::new(Format::Cbq) - .paired(true) - .quality(false) - .headers(false) - .flags(false) - .build(Cursor::new(Vec::new()))?; - - let record = minimal_single_record(); - let result = writer.push(record); - assert!(result.is_err()); - Ok(()) - } - - #[test] - fn test_cbq_single_writer_paired_record() -> Result<()> { - // Writer: single-end - // Record: paired - // Expected: success (over-specified - R2 ignored) - let mut writer = BinseqWriterBuilder::new(Format::Cbq) - .paired(false) - .quality(false) - .headers(false) - .flags(false) - .build(Cursor::new(Vec::new()))?; - - let record = minimal_paired_record(); - assert!(writer.push(record)?); - writer.finish()?; - Ok(()) - } - - #[test] - fn test_cbq_paired_minimal_writer_paired_full_record() -> Result<()> { - // Writer: paired, no quality, no headers, no flags - // Record: paired, with quality, headers, flags - // Expected: success (over-specified) - let mut writer = BinseqWriterBuilder::new(Format::Cbq) - .paired(true) - .quality(false) - .headers(false) - .flags(false) - .build(Cursor::new(Vec::new()))?; - - let record = full_paired_record(); - assert!(writer.push(record)?); - writer.finish()?; - Ok(()) - } - - #[test] - fn test_cbq_paired_full_writer_paired_full_record() -> Result<()> { - // Writer: paired, with quality, headers, flags - // Record: paired, with quality, headers, flags - // Expected: success (correctly specified) - let mut writer = BinseqWriterBuilder::new(Format::Cbq) - .paired(true) - .quality(true) - .headers(true) - .flags(true) - .build(Cursor::new(Vec::new()))?; - - let record = full_paired_record(); - assert!(writer.push(record)?); - writer.finish()?; - Ok(()) - } - - // ==================== BQ Tests ==================== - // Note: BQ format has fixed-length sequences and doesn't support headers - - #[test] - fn test_bq_single_minimal_writer_minimal_record() -> Result<()> { - // Writer: single-end, no quality, no flags - // Record: single-end, no quality, no flags - // Expected: success (correctly specified) - let mut writer = BinseqWriterBuilder::new(Format::Bq) - .slen(32) - .paired(false) - .quality(false) - .flags(false) - .build(Cursor::new(Vec::new()))?; - - let record = minimal_single_record(); - assert!(writer.push(record)?); - writer.finish()?; - Ok(()) - } - - #[test] - fn test_bq_single_minimal_writer_full_record() -> Result<()> { - // Writer: single-end, no quality, no flags - // Record: single-end, with quality, headers, flags - // Expected: success (over-specified - extra data ignored) - let mut writer = BinseqWriterBuilder::new(Format::Bq) - .slen(32) - .paired(false) - .quality(false) - .flags(false) - .build(Cursor::new(Vec::new()))?; - - let record = full_single_record(); - assert!(writer.push(record)?); - writer.finish()?; - Ok(()) - } - - #[test] - fn test_bq_single_with_quality_writer_minimal_record() -> Result<()> { - // Writer: single-end, with quality (note: BQ ignores quality setting) - // Record: single-end, no quality - // Expected: success (BQ format doesn't support quality scores, setting is ignored) - let mut writer = BinseqWriterBuilder::new(Format::Bq) - .slen(32) - .paired(false) - .quality(true) // This is ignored for BQ format - .build(Cursor::new(Vec::new()))?; - - // BQ always reports has_quality as false - assert!(!writer.has_quality()); - - let record = minimal_single_record(); - assert!(writer.push(record)?); - writer.finish()?; - Ok(()) - } - - #[test] - fn test_bq_single_with_quality_writer_full_record() -> Result<()> { - // Writer: single-end, with quality - // Record: single-end, with quality - // Expected: success (correctly specified) - let mut writer = BinseqWriterBuilder::new(Format::Bq) - .slen(32) - .paired(false) - .quality(true) - .build(Cursor::new(Vec::new()))?; - - let record = full_single_record(); - assert!(writer.push(record)?); - writer.finish()?; - Ok(()) - } - - #[test] - fn test_bq_paired_writer_single_record() -> Result<()> { - // Writer: paired - // Record: single-end - // Expected: error (under-specified - missing R2) - let mut writer = BinseqWriterBuilder::new(Format::Bq) - .slen(32) - .xlen(32) - .paired(true) - .quality(false) - .build(Cursor::new(Vec::new()))?; - - let record = minimal_single_record(); - let result = writer.push(record); - assert!(result.is_err()); - Ok(()) - } - - #[test] - fn test_bq_single_writer_paired_record() -> Result<()> { - // Writer: single-end - // Record: paired - // Expected: success (over-specified - R2 ignored) - let mut writer = BinseqWriterBuilder::new(Format::Bq) - .slen(32) - .paired(false) - .quality(false) - .build(Cursor::new(Vec::new()))?; - - let record = minimal_paired_record(); - assert!(writer.push(record)?); - writer.finish()?; - Ok(()) - } - - #[test] - fn test_bq_paired_minimal_writer_paired_full_record() -> Result<()> { - // Writer: paired, no quality, no flags - // Record: paired, with quality, headers, flags - // Expected: success (over-specified) - let mut writer = BinseqWriterBuilder::new(Format::Bq) - .slen(32) - .xlen(32) - .paired(true) - .quality(false) - .flags(false) - .build(Cursor::new(Vec::new()))?; - - let record = full_paired_record(); - assert!(writer.push(record)?); - writer.finish()?; + fn test_record_specification_matrix() -> Result<()> { + // (format, paired, quality, headers, flags, record, expect_ok) + // + // - Under-specified: record is missing data the writer needs -> error + // - Over-specified: record has extra data the writer ignores -> success + // - Correctly-specified: record matches writer config exactly -> success + // BQ ignores quality/headers settings entirely. + #[rustfmt::skip] + let cases: &[(Format, bool, bool, bool, bool, RecordFn, bool)] = &[ + (Format::Vbq, false, false, false, false, minimal_single_record, true), + (Format::Vbq, false, false, false, false, full_single_record, true), + (Format::Vbq, false, true, true, true, minimal_single_record, false), + (Format::Vbq, false, true, true, true, full_single_record, true), + (Format::Vbq, true, false, false, false, minimal_single_record, false), + (Format::Vbq, false, false, false, false, minimal_paired_record, true), + (Format::Vbq, true, false, false, false, full_paired_record, true), + (Format::Vbq, true, true, true, true, full_paired_record, true), + (Format::Cbq, false, false, false, false, minimal_single_record, true), + (Format::Cbq, false, false, false, false, full_single_record, true), + (Format::Cbq, false, true, true, true, minimal_single_record, false), + (Format::Cbq, false, true, true, true, full_single_record, true), + (Format::Cbq, true, false, false, false, minimal_single_record, false), + (Format::Cbq, false, false, false, false, minimal_paired_record, true), + (Format::Cbq, true, false, false, false, full_paired_record, true), + (Format::Cbq, true, true, true, true, full_paired_record, true), + (Format::Bq, false, false, false, false, minimal_single_record, true), + (Format::Bq, false, false, false, false, full_single_record, true), + (Format::Bq, false, true, false, false, minimal_single_record, true), + (Format::Bq, false, true, false, false, full_single_record, true), + (Format::Bq, true, false, false, false, minimal_single_record, false), + (Format::Bq, false, false, false, false, minimal_paired_record, true), + (Format::Bq, true, false, false, false, full_paired_record, true), + (Format::Bq, true, true, false, true, full_paired_record, true), + ]; + + for &(format, paired, quality, headers, flags, record, expect_ok) in cases { + let mut builder = BinseqWriterBuilder::new(format) + .paired(paired) + .quality(quality) + .headers(headers) + .flags(flags); + // BQ is fixed-length: sequence lengths come from the header + if matches!(format, Format::Bq) { + builder = builder.slen(32); + if paired { + builder = builder.xlen(32); + } + } + let mut writer = builder.build(Cursor::new(Vec::new()))?; + + let case = format!( + "{format:?} paired={paired} quality={quality} headers={headers} flags={flags}" + ); + let result = writer.push(record()); + if expect_ok { + assert!(result.unwrap_or_else(|e| panic!("{case}: {e}")), "{case}"); + writer.finish()?; + } else { + assert!(result.is_err(), "{case}: expected error"); + } + } Ok(()) } #[test] - fn test_bq_paired_full_writer_paired_full_record() -> Result<()> { - // Writer: paired, with quality, flags - // Record: paired, with quality, headers, flags - // Expected: success (correctly specified, headers ignored for BQ) - let mut writer = BinseqWriterBuilder::new(Format::Bq) + fn test_bq_ignores_quality_flag() -> Result<()> { + // BQ format doesn't support quality scores; the setting is ignored + let writer = BinseqWriterBuilder::new(Format::Bq) .slen(32) - .xlen(32) - .paired(true) .quality(true) - .flags(true) .build(Cursor::new(Vec::new()))?; - - let record = full_paired_record(); - assert!(writer.push(record)?); - writer.finish()?; + assert!(!writer.has_quality()); Ok(()) }