From c4ac5a52c07c76b1b5feaafe60ce8c97137cbf04 Mon Sep 17 00:00:00 2001 From: noamteyssier <22600644+noamteyssier@users.noreply.github.com> Date: Tue, 25 Aug 2026 17:04:56 -0700 Subject: [PATCH 01/25] fix(error): compile test module only under cfg(test), prune dead variants The error test module was missing #[cfg(test)], so its test-only enum compiled into release builds. Also removes five never-constructed WriteError variants and CbqError::MissingSequenceOnSequencingRecord, folds the one-variant BuilderError into HeaderError::MissingSequenceLength, folds FastxEncodingError into WriteError, and trims tests that only exercised thiserror's derive macro. BREAKING: error::BuilderError and error::FastxEncodingError are removed. --- src/bq/header.rs | 4 +- src/error.rs | 308 +++------------------------------------------ src/utils/fastx.rs | 10 +- 3 files changed, 24 insertions(+), 298 deletions(-) diff --git a/src/bq/header.rs b/src/bq/header.rs index dd84593..1290217 100644 --- a/src/bq/header.rs +++ b/src/bq/header.rs @@ -8,7 +8,7 @@ use bitnuc_deprec::BitSize; use byteorder::{ByteOrder, LittleEndian}; use std::io::{Read, Write}; -use crate::error::{BuilderError, HeaderError, Result}; +use crate::error::{HeaderError, Result}; /// Current magic number: "BSEQ" in ASCII (in little-endian byte order) /// @@ -86,7 +86,7 @@ impl FileHeaderBuilder { slen: if let Some(slen) = self.slen { slen } else { - return Err(BuilderError::MissingSlen.into()); + return Err(HeaderError::MissingSequenceLength.into()); }, xlen: self.xlen.unwrap_or(0), bits: self.bitsize.unwrap_or_default(), diff --git a/src/error.rs b/src/error.rs index ef20cbb..27d4089 100644 --- a/src/error.rs +++ b/src/error.rs @@ -6,7 +6,6 @@ pub type Result = std::result::Result; /// The main error type for the binseq library, encompassing all possible error cases /// that can occur during binary sequence operations. #[derive(thiserror::Error, Debug)] -#[error(transparent)] pub enum Error { /// Errors related to file and block headers #[error("Error processing header: {0}")] @@ -24,10 +23,6 @@ pub enum Error { #[error("Error reading file: {0}")] ReadError(#[from] ReadError), - /// Errors that occur during build operations - #[error("Error building file: {0}")] - BuilderError(#[from] BuilderError), - /// Errors related to file indexing #[error("Error processing Index: {0}")] IndexError(#[from] IndexError), @@ -60,10 +55,6 @@ pub enum Error { /// Generic errors for other unexpected situations #[error("Generic error: {0}")] GenericError(#[from] Box), - - #[cfg(feature = "paraseq")] - #[error("Fastx encoding error: {0}")] - FastxEncodingError(#[from] FastxEncodingError), } /// Errors specific to processing and validating binary sequence headers @@ -98,6 +89,10 @@ pub enum HeaderError { /// * Second `usize` - The expected number of bytes according to the header #[error("Invalid number of bytes provided: {0}. Expected: {1}")] InvalidSize(usize, usize), + + /// A required sequence length was not provided when building the header + #[error("Missing sequence length")] + MissingSequenceLength, } /// Errors that can occur while reading binary sequence data @@ -162,12 +157,6 @@ pub enum ReadError { MissingIndexEndMagic, } -#[derive(thiserror::Error, Debug)] -pub enum BuilderError { - #[error("Missing sequence length")] - MissingSlen, -} - /// Errors that can occur while writing binary sequence data #[derive(thiserror::Error, Debug)] pub enum WriteError { @@ -213,26 +202,6 @@ pub enum WriteError { #[error("Missing header in writer builder")] MissingHeader, - /// When trying to write data without quality scores but the header specifies they should be present - #[error("Quality flag is set in header but trying to write without quality scores.")] - QualityFlagSet, - - /// When trying to write data without a pair but the header specifies paired records - #[error("Paired flag is set in header but trying to write without record pair.")] - PairedFlagSet, - - /// When trying to write quality scores but the header specifies they are not present - #[error("Quality flag not set in header but trying to write quality scores.")] - QualityFlagNotSet, - - /// When trying to write paired data but the header doesn't specify paired records - #[error("Paired flag not set in header but trying to write with record pair.")] - PairedFlagNotSet, - - /// When trying to write data without headers but the header specifies they should be present - #[error("Header flag is set in header but trying to write without headers.")] - HeaderFlagSet, - /// When a record is too large to fit in a block of the configured size /// /// The first parameter is the record size, the second is the maximum block size @@ -257,6 +226,16 @@ pub enum WriteError { /// When building a `SequencingRecord` without a primary sequence #[error("SequencingRecordBuilder requires a primary sequence (s_seq)")] MissingSequence, + + /// FASTX encoding was attempted on an empty input file + #[cfg(feature = "paraseq")] + #[error("Empty FASTX file")] + EmptyFastxFile, + + /// FASTX encoding was attempted without any input + #[cfg(feature = "paraseq")] + #[error("Builder not provided with any input")] + MissingInput, } /// Errors related to VBQ file indexing @@ -316,19 +295,6 @@ pub enum CbqError { #[error("Unable to cast bytes to Index - likely an alignment error")] IndexCastingError, - - #[error("SequenceRecordBuilder failed on build due to missing primary sequence (`s_seq`)")] - MissingSequenceOnSequencingRecord, -} - -#[cfg(feature = "paraseq")] -#[derive(thiserror::Error, Debug)] -pub enum FastxEncodingError { - #[error("Empty FASTX file")] - EmptyFastxFile, - - #[error("Builder not provided with any input")] - MissingInput, } #[derive(thiserror::Error, Debug)] @@ -353,12 +319,11 @@ where } } +#[cfg(test)] mod testing { - #[allow(unused)] use super::*; use thiserror::Error; - #[allow(unused)] #[derive(Error, Debug)] pub enum MyError { #[error("Custom error: {0}")] @@ -372,248 +337,9 @@ mod testing { assert!(matches!(binseq_error, Error::GenericError(_))); } - // ==================== HeaderError Tests ==================== - - #[test] - fn test_header_error_invalid_magic_number() { - let error = HeaderError::InvalidMagicNumber(0xDEAD_BEEF); - let error_str = format!("{error}"); - assert!(error_str.contains("0xdeadbeef") || error_str.contains("3735928559")); - } - - #[test] - fn test_header_error_invalid_format_version() { - let error = HeaderError::InvalidFormatVersion(99); - let error_str = format!("{error}"); - assert!(error_str.contains("99")); - } - - #[test] - fn test_header_error_invalid_bit_size() { - let error = HeaderError::InvalidBitSize(8); - let error_str = format!("{error}"); - assert!(error_str.contains('8')); - assert!(error_str.contains("[2,4]")); - } - - #[test] - fn test_header_error_invalid_size() { - let error = HeaderError::InvalidSize(100, 200); - let error_str = format!("{error}"); - assert!(error_str.contains("100")); - assert!(error_str.contains("200")); - } - - // ==================== ReadError Tests ==================== - - #[test] - fn test_read_error_out_of_range() { - let error = ReadError::OutOfRange { - requested_index: 150, - max_index: 100, - }; - let error_str = format!("{error}"); - assert!(error_str.contains("150")); - assert!(error_str.contains("100")); - } - - #[test] - fn test_read_error_file_truncation() { - let error = ReadError::FileTruncation(12345); - let error_str = format!("{error}"); - assert!(error_str.contains("12345")); - } - #[test] - fn test_read_error_partial_record() { - let error = ReadError::PartialRecord(42); - let error_str = format!("{error}"); - assert!(error_str.contains("42")); - } - - #[test] - fn test_read_error_invalid_block_magic_number() { - let error = ReadError::InvalidBlockMagicNumber(0x0BAD_C0DE, 1000); - let error_str = format!("{error}"); - assert!(error_str.contains("1000")); - } - - // ==================== WriteError Tests ==================== - - #[test] - fn test_write_error_configuration_mismatch() { - let error = WriteError::ConfigurationMismatch { - attribute: "paired", - expected: true, - actual: false, - }; - let error_str = format!("{error}"); - assert!(error_str.contains("paired")); - assert!(error_str.contains("true")); - assert!(error_str.contains("false")); - } - - #[test] - fn test_write_error_unexpected_sequence_length() { - let error = WriteError::UnexpectedSequenceLength { - expected: 100, - got: 150, - }; - let error_str = format!("{error}"); - assert!(error_str.contains("100")); - assert!(error_str.contains("150")); - } - - #[test] - fn test_write_error_invalid_nucleotide_sequence() { - let error = WriteError::InvalidNucleotideSequence("ACGTNX".to_string()); - let error_str = format!("{error}"); - assert!(error_str.contains("ACGTNX")); - } - - #[test] - fn test_write_error_record_size_exceeds_max() { - let error = WriteError::RecordSizeExceedsMaximumBlockSize(2000, 1024); - let error_str = format!("{error}"); - assert!(error_str.contains("2000")); - assert!(error_str.contains("1024")); - } - - #[test] - fn test_write_error_missing_sequence_length() { - let error = WriteError::MissingSequenceLength { - exp_primary: true, - exp_extended: false, - obs_primary: false, - obs_extended: false, - }; - let error_str = format!("{error}"); - assert!(error_str.contains("Missing required sequence length")); - } - - // ==================== CbqError Tests ==================== - - #[test] - fn test_cbq_error_exceeds_maximum_block_size() { - let error = CbqError::ExceedsMaximumBlockSize { - max_block_size: 1024, - record_size: 2048, - }; - let error_str = format!("{error}"); - assert!(error_str.contains("1024")); - assert!(error_str.contains("2048")); - } - - #[test] - fn test_cbq_error_block_full() { - let error = CbqError::BlockFull { - current_size: 900, - record_size: 200, - block_size: 1024, - }; - let error_str = format!("{error}"); - assert!(error_str.contains("900")); - assert!(error_str.contains("200")); - assert!(error_str.contains("1024")); - } - - #[test] - fn test_cbq_error_cannot_ingest_block() { - let error = CbqError::CannotIngestBlock { - self_block_size: 1024, - other_block_size: 2048, - }; - let error_str = format!("{error}"); - assert!(error_str.contains("1024")); - assert!(error_str.contains("2048")); - } - - // ==================== BuilderError Tests ==================== - - #[test] - fn test_builder_error_missing_slen() { - let error = BuilderError::MissingSlen; - let error_str = format!("{error}"); - assert!(error_str.contains("Missing sequence length")); - } - - // ==================== FormatError Tests ==================== - - #[test] - fn test_format_error_unrecognized_magic_bytes() { - let error = FormatError::UnrecognizedMagicBytes("test.xyz".to_string()); - let error_str = format!("{error}"); - assert!(error_str.contains("test.xyz")); - } - - // ==================== Error Conversion Tests ==================== - - #[test] - fn test_error_from_header_error() { - let header_error = HeaderError::InvalidMagicNumber(0x1234); - let error: Error = header_error.into(); + fn test_error_from_sub_error() { + let error: Error = HeaderError::InvalidMagicNumber(0x1234).into(); assert!(matches!(error, Error::HeaderError(_))); } - - #[test] - fn test_error_from_write_error() { - let write_error = WriteError::MissingHeader; - let error: Error = write_error.into(); - assert!(matches!(error, Error::WriteError(_))); - } - - #[test] - fn test_error_from_read_error() { - let read_error = ReadError::EndOfStream; - let error: Error = read_error.into(); - assert!(matches!(error, Error::ReadError(_))); - } - - #[test] - fn test_error_from_index_error() { - let index_error = IndexError::InvalidMagicNumber(0x5678); - let error: Error = index_error.into(); - assert!(matches!(error, Error::IndexError(_))); - } - - #[test] - fn test_error_from_cbq_error() { - let cbq_error = CbqError::InvalidBlockHeaderMagic; - let error: Error = cbq_error.into(); - assert!(matches!(error, Error::CbqError(_))); - } - - #[test] - fn test_error_from_builder_error() { - let builder_error = BuilderError::MissingSlen; - let error: Error = builder_error.into(); - assert!(matches!(error, Error::BuilderError(_))); - } - - #[test] - fn test_error_debug_output() { - let error = Error::WriteError(WriteError::MissingHeader); - let debug_str = format!("{error:?}"); - assert!(debug_str.contains("WriteError")); - } - - // ==================== Fastx Error Tests (conditional) ==================== - - #[cfg(feature = "paraseq")] - #[test] - fn test_fastx_error_empty_file() { - use super::FastxEncodingError; - let error = FastxEncodingError::EmptyFastxFile; - let error_str = format!("{error}"); - assert!(error_str.contains("Empty FASTX file")); - } - - #[cfg(feature = "paraseq")] - #[test] - fn test_fastx_error_missing_input() { - use super::FastxEncodingError; - let error = FastxEncodingError::MissingInput; - let error_str = format!("{error}"); - assert!(error_str.contains("not provided with any input")); - } } diff --git a/src/utils/fastx.rs b/src/utils/fastx.rs index 3d13007..05c9b18 100644 --- a/src/utils/fastx.rs +++ b/src/utils/fastx.rs @@ -17,7 +17,7 @@ use parking_lot::Mutex; use crate::{ BinseqWriter, BinseqWriterBuilder, IntoBinseqError, Result, SequencingRecordBuilder, - error::FastxEncodingError, + error::WriteError, }; type BoxedRead = Box; @@ -211,7 +211,7 @@ impl FastxEncoderBuilder { self.builder = self.builder.slen(slen as u32).xlen(xlen as u32); (reader1, Some(reader2)) } - None => return Err(FastxEncodingError::MissingInput.into()), + None => return Err(WriteError::MissingInput.into()), }; let writer = self.builder.build(self.output)?; @@ -286,13 +286,13 @@ fn detect_seq_len( let rec = r.map_err(IntoBinseqError::into_binseq_error)?; Ok(rec.seq().len()) }) else { - return Err(FastxEncodingError::EmptyFastxFile.into()); + return Err(WriteError::EmptyFastxFile.into()); }; let Some(Ok(xlen)) = rset_iter.next().map(|r| -> Result { let rec = r.map_err(IntoBinseqError::into_binseq_error)?; Ok(rec.seq().len()) }) else { - return Err(FastxEncodingError::EmptyFastxFile.into()); + return Err(WriteError::EmptyFastxFile.into()); }; (slen, xlen) } else { @@ -301,7 +301,7 @@ fn detect_seq_len( let rec = r.map_err(IntoBinseqError::into_binseq_error)?; Ok(rec.seq().len()) }) else { - return Err(FastxEncodingError::EmptyFastxFile.into()); + return Err(WriteError::EmptyFastxFile.into()); }; (slen, 0) }; From 23fb13fa4684f85ad76399ec26e9eb5ec8c7a992 Mon Sep 17 00:00:00 2001 From: noamteyssier <22600644+noamteyssier@users.noreply.github.com> Date: Tue, 25 Aug 2026 17:20:17 -0700 Subject: [PATCH 02/25] refactor(bq)!: drop StreamWriter and the deprecated write_record family StreamWriter was a pure delegating wrapper around Writer>. Callers may compose directly (see network_streaming example). The write_record/write_paired_record methods were already deprecated in favor of push(SequencingRecord). The write_flag and write_buffer free functions become one private little-endian helper. BREAKING: bq::StreamWriter, bq::StreamWriterBuilder, and the deprecated write_record/write_paired_record methods (bq and vbq) are removed. --- examples/network_streaming.rs | 19 +- examples/streaming.rs | 56 ----- src/bq/mod.rs | 15 +- src/bq/writer.rs | 438 +--------------------------------- src/vbq/writer.rs | 40 +--- 5 files changed, 25 insertions(+), 543 deletions(-) delete mode 100644 examples/streaming.rs diff --git a/examples/network_streaming.rs b/examples/network_streaming.rs index 71ca2f9..10d0415 100644 --- a/examples/network_streaming.rs +++ b/examples/network_streaming.rs @@ -2,8 +2,8 @@ use std::io::{BufReader, BufWriter}; use std::net::{TcpListener, TcpStream}; use std::thread; -use binseq::bq::{FileHeader, FileHeaderBuilder, StreamReader, StreamWriterBuilder}; -use binseq::{BinseqRecord, Policy, Result}; +use binseq::bq::{FileHeader, FileHeaderBuilder, StreamReader, WriterBuilder}; +use binseq::{BinseqRecord, Policy, Result, SequencingRecordBuilder}; fn server(header: FileHeader, sequence: &[u8]) -> Result<()> { // Create a listener on localhost:3000 @@ -14,19 +14,22 @@ fn server(header: FileHeader, sequence: &[u8]) -> Result<()> { let (stream, _) = listener.accept().expect("Failed to accept connection"); println!("Client connected"); - let stream = BufWriter::new(stream); + // Buffer the network stream for efficient writes + let stream = BufWriter::with_capacity(16384, stream); - // Create a stream writer with the network stream as destination - let mut writer = StreamWriterBuilder::default() + // Create a writer with the network stream as destination + let mut writer = WriterBuilder::default() .header(header) .policy(Policy::RandomDraw) - .buffer_capacity(16384) // Larger buffer for network I/O .build(stream)?; // Write sequences in a loop for i in 0..10 { - #[allow(deprecated)] - writer.write_record(Some(i), sequence)?; + let record = SequencingRecordBuilder::default() + .s_seq(sequence) + .flag(i) + .build()?; + writer.push(record)?; println!("Server: Sent record {i}"); // Simulate delay between records diff --git a/examples/streaming.rs b/examples/streaming.rs deleted file mode 100644 index 001ad47..0000000 --- a/examples/streaming.rs +++ /dev/null @@ -1,56 +0,0 @@ -use std::io::{BufReader, Cursor}; - -use binseq::bq::{FileHeaderBuilder, StreamReader, StreamWriterBuilder}; -use binseq::{BinseqRecord, Policy, Result}; - -fn main() -> Result<()> { - // Create a header for sequences of length 100 - let header = FileHeaderBuilder::new().slen(100).build()?; - - // Create some example sequence data - let sequence = b"ACGT".repeat(25); // 100 nucleotides - - // Create a stream writer with a memory buffer as destination - let mut writer = StreamWriterBuilder::default() - .header(header) - .policy(Policy::RandomDraw) // Use random nucleotides for invalid bases - .buffer_capacity(4096) // Use 4K buffer - .build(Cursor::new(Vec::new()))?; - - // Write the sequence with flag 0 - #[allow(deprecated)] - writer.write_record(Some(0), &sequence)?; - - // Write the sequence with flag 1 - #[allow(deprecated)] - writer.write_record(Some(1), &sequence)?; - - // Flush and get the buffer - let buffer = writer.into_inner()?; - let buffer_inner = buffer.into_inner(); - - println!("Wrote {} bytes to buffer", buffer_inner.len()); - - // Now read from the buffer using the streaming reader - let cursor = Cursor::new(buffer_inner); - let buf_reader = BufReader::new(cursor); - - // Create a stream reader - let mut reader = StreamReader::new(buf_reader); - - // Read and display the header - let header = reader.read_header()?; - println!("Read header: sequence length = {}", header.slen); - - // Read records one by one - let mut count = 0; - while let Some(record) = reader.next_record() { - let record = record?; - println!("Record {}: flag = {:?}", count, record.flag()); - count += 1; - } - - println!("Read {count} records"); - - Ok(()) -} diff --git a/src/bq/mod.rs b/src/bq/mod.rs index 47fa622..08bc097 100644 --- a/src/bq/mod.rs +++ b/src/bq/mod.rs @@ -109,18 +109,17 @@ //! //! ``` //! use binseq::{Policy, Result, BinseqRecord, SequencingRecordBuilder}; -//! use binseq::bq::{FileHeaderBuilder, StreamReader, StreamWriterBuilder}; -//! use std::io::{BufReader, Cursor}; +//! use binseq::bq::{FileHeaderBuilder, StreamReader, WriterBuilder}; +//! use std::io::{BufReader, BufWriter, Cursor}; //! //! fn main() -> Result<()> { //! // Create a header for sequences of length 100 //! let header = FileHeaderBuilder::new().slen(100).build()?; //! -//! // Create a stream writer -//! let mut writer = StreamWriterBuilder::default() +//! // Create a buffered writer over any `Write` destination +//! let mut writer = WriterBuilder::default() //! .header(header) -//! .buffer_capacity(8192) -//! .build(Cursor::new(Vec::new()))?; +//! .build(BufWriter::new(Cursor::new(Vec::new())))?; //! //! // Write sequences //! let sequence = b"ACGT".repeat(25); // 100 nucleotides @@ -131,7 +130,7 @@ //! writer.push(record)?; //! //! // Get the inner buffer -//! let buffer = writer.into_inner()?; +//! let buffer = writer.into_inner().into_inner().map_err(std::io::Error::from)?; //! let data = buffer.into_inner(); //! //! // Create a stream reader @@ -243,4 +242,4 @@ mod writer; pub use header::{FILE_MAGIC, FileHeader, FileHeaderBuilder, SIZE_HEADER}; pub use reader::{MmapReader, RefRecord, StreamReader}; -pub use writer::{Encoder, StreamWriter, StreamWriterBuilder, Writer, WriterBuilder}; +pub use writer::{Encoder, Writer, WriterBuilder}; diff --git a/src/bq/writer.rs b/src/bq/writer.rs index e457b13..cc1b7e0 100644 --- a/src/bq/writer.rs +++ b/src/bq/writer.rs @@ -7,9 +7,8 @@ //! - Efficient buffering and encoding //! - Headless mode for parallel writing -use std::io::{BufWriter, Write}; +use std::io::Write; -use byteorder::{LittleEndian, WriteBytesExt}; use rand::{SeedableRng, rngs::SmallRng}; use super::FileHeader; @@ -18,39 +17,10 @@ use crate::{ error::{Result, WriteError}, }; -/// Writes a single flag value to a writer in little-endian format -/// -/// # Arguments -/// -/// * `writer` - Any type that implements the `Write` trait -/// * `flag` - The 64-bit flag value to write -/// -/// # Returns -/// -/// * `Ok(())` - If the flag was successfully written -/// * `Err(Error)` - If writing to the writer failed -pub fn write_flag(writer: &mut W, flag: u64) -> Result<()> { - writer.write_u64::(flag)?; - Ok(()) -} - /// Writes a buffer of u64 values to a writer in little-endian format -/// -/// This function is used to write encoded sequence data to the output. -/// Each u64 in the buffer contains up to 32 nucleotides in 2-bit format. -/// -/// # Arguments -/// -/// * `writer` - Any type that implements the `Write` trait -/// * `ebuf` - The buffer of u64 values to write -/// -/// # Returns -/// -/// * `Ok(())` - If the buffer was successfully written -/// * `Err(Error)` - If writing to the writer failed -pub fn write_buffer(writer: &mut W, ebuf: &[u64]) -> Result<()> { +fn write_buffer(writer: &mut W, ebuf: &[u64]) -> Result<()> { ebuf.iter() - .try_for_each(|&x| writer.write_u64::(x))?; + .try_for_each(|&x| writer.write_all(&x.to_le_bytes()))?; Ok(()) } @@ -372,67 +342,6 @@ impl Writer { self.encoder.policy } - /// Writes a single record to the output - /// - /// This method encodes and writes a primary sequence along with an associated flag. - /// - /// # Arguments - /// - /// * `flag` - A 64-bit flag value associated with the sequence - /// * `primary` - The nucleotide sequence to write - /// - /// # Returns - /// - /// * `Ok(true)` if the record was written successfully - /// * `Ok(false)` if the record was not written because it was empty - /// * `Err(WriteError::FlagSet)` if the flag is set but no flag value is provided - #[deprecated] - pub fn write_record(&mut self, flag: Option, primary: &[u8]) -> Result { - let has_flag = self.encoder.header.flags; - if let Some(sbuffer) = self.encoder.encode_single(primary)? { - if has_flag { - write_flag(&mut self.inner, flag.unwrap_or(0))?; - } - write_buffer(&mut self.inner, sbuffer)?; - Ok(true) - } else { - Ok(false) - } - } - - /// Writes a paired record to the output - /// - /// This method writes a paired record to the output. It takes a flag, primary sequence, and extended sequence as input. - /// If the flag is set but no flag value is provided, it returns an error. - /// Otherwise, it writes the encoded single and extended sequences to the output and returns true. - /// - /// # Arguments - /// * `flag` - The flag value to write to the output - /// * `primary` - The primary sequence to encode and write to the output - /// * `extended` - The extended sequence to encode and write to the output - /// - /// # Returns - /// * `Result` - A result indicating whether the write was successful or not - #[deprecated] - pub fn write_paired_record( - &mut self, - flag: Option, - primary: &[u8], - extended: &[u8], - ) -> Result { - let has_flag = self.encoder.header.flags; - if let Some((sbuffer, xbuffer)) = self.encoder.encode_paired(primary, extended)? { - if has_flag { - write_flag(&mut self.inner, flag.unwrap_or(0))?; - } - write_buffer(&mut self.inner, sbuffer)?; - write_buffer(&mut self.inner, xbuffer)?; - Ok(true) - } else { - Ok(false) - } - } - /// Writes a record using the unified [`SequencingRecord`] API /// /// This method provides a consistent interface with VBQ and CBQ writers. @@ -472,7 +381,8 @@ impl Writer { pub fn push(&mut self, record: SequencingRecord) -> Result { let has_flag = self.encoder.header.flags; if has_flag { - write_flag(&mut self.inner, record.flag().unwrap_or(0))?; + self.inner + .write_all(&record.flag.unwrap_or(0).to_le_bytes())?; } // Check paired status - writer can require paired (record must have R2), @@ -549,21 +459,6 @@ impl Writer { Ok(()) } - /// Creates a new encoder with the same configuration as this writer - /// - /// This is useful when you need a separate encoder instance for parallel - /// processing or other scenarios where you need independent encoding. - /// The new encoder is initialized with a cleared state. - /// - /// # Returns - /// - /// A new `Encoder` instance with the same configuration but cleared buffers - pub fn new_encoder(&self) -> Encoder { - let mut encoder = self.encoder.clone(); - encoder.clear(); - encoder - } - /// Checks if this writer is in headless mode /// /// In headless mode, the writer does not write the header to the output. @@ -599,195 +494,6 @@ impl Writer { } } -/// A streaming writer for binary sequence data -/// -/// This writer buffers data before writing it to the underlying writer, -/// providing efficient streaming capabilities suitable for: -/// - Writing to network connections -/// - Processing very large datasets -/// - Pipeline processing -/// -/// The `StreamWriter` is a specialized version of `Writer` that -/// adds internal buffering and is optimized for streaming scenarios. -pub struct StreamWriter { - /// The underlying writer for processing sequences - writer: Writer>, -} - -impl StreamWriter { - /// Creates a new `StreamWriter` with the default buffer size - /// - /// This constructor initializes a `StreamWriter` with an 8K buffer - /// for efficient writing to the underlying writer. - /// - /// # Arguments - /// - /// * `inner` - The writer to write binary sequence data to - /// * `header` - The header defining sequence lengths and format - /// * `policy` - The policy for handling invalid nucleotides - /// * `headless` - Whether to skip writing the header - /// - /// # Returns - /// - /// * `Ok(StreamWriter)` - A new streaming writer - /// * `Err(Error)` - If initialization fails - pub fn new(inner: W, header: FileHeader, policy: Policy, headless: bool) -> Result { - Self::with_capacity(inner, 8192, header, policy, headless) - } - - /// Creates a new `StreamWriter` with a specified buffer capacity - /// - /// This constructor allows customizing the buffer size based on - /// expected usage patterns and performance requirements. - /// - /// # Arguments - /// - /// * `inner` - The writer to write binary sequence data to - /// * `capacity` - The size of the internal buffer in bytes - /// * `header` - The header defining sequence lengths and format - /// * `policy` - The policy for handling invalid nucleotides - /// * `headless` - Whether to skip writing the header - /// - /// # Returns - /// - /// * `Ok(StreamWriter)` - A new streaming writer with the specified buffer capacity - /// * `Err(Error)` - If initialization fails - pub fn with_capacity( - inner: W, - capacity: usize, - header: FileHeader, - policy: Policy, - headless: bool, - ) -> Result { - let buffered = BufWriter::with_capacity(capacity, inner); - let writer = Writer::new(buffered, header, policy, headless)?; - - Ok(Self { writer }) - } - - #[deprecated(note = "use `push` method with SequencingRecord instead")] - pub fn write_record(&mut self, flag: Option, primary: &[u8]) -> Result { - #[allow(deprecated)] - self.writer.write_record(flag, primary) - } - - #[deprecated(note = "use `push` method with SequencingRecord instead")] - pub fn write_paired_record( - &mut self, - flag: Option, - primary: &[u8], - extended: &[u8], - ) -> Result { - #[allow(deprecated)] - self.writer.write_paired_record(flag, primary, extended) - } - - /// Writes a record using the unified [`SequencingRecord`] API - pub fn push(&mut self, record: SequencingRecord) -> Result { - self.writer.push(record) - } - - /// Flushes any buffered data to the underlying writer - /// - /// # Returns - /// - /// * `Ok(())` - If the flush was successful - /// * `Err(Error)` - If flushing failed - pub fn flush(&mut self) -> Result<()> { - self.writer.flush() - } - - /// Consumes the streaming writer and returns the inner writer after flushing - /// - /// This method is useful when you need access to the underlying writer - /// after all writing is complete. - /// - /// # Returns - /// - /// * `Ok(W)` - The inner writer after flushing all data - /// * `Err(Error)` - If flushing failed - pub fn into_inner(self) -> Result { - // First unwrap the writer inner (BufWriter) - let bufw = self.writer.into_inner(); - // Now unwrap the BufWriter to get W - match bufw.into_inner() { - Ok(inner) => Ok(inner), - Err(e) => Err(std::io::Error::from(e).into()), - } - } -} - -/// Builder for `StreamWriter` instances -/// -/// This builder provides a convenient way to create and configure `StreamWriter` -/// instances with custom buffer sizes and other settings. -#[derive(Default)] -pub struct StreamWriterBuilder { - /// Required header defining sequence lengths and format - header: Option, - /// Optional policy for handling invalid nucleotides - policy: Option, - /// Optional headless mode for parallel writing scenarios - headless: Option, - /// Optional buffer capacity setting - buffer_capacity: Option, -} - -impl StreamWriterBuilder { - /// Sets the header for the writer - #[must_use] - pub fn header(mut self, header: FileHeader) -> Self { - self.header = Some(header); - self - } - - /// Sets the policy for handling invalid nucleotides - #[must_use] - pub fn policy(mut self, policy: Policy) -> Self { - self.policy = Some(policy); - self - } - - /// Sets headless mode (whether to skip writing the header) - #[must_use] - pub fn headless(mut self, headless: bool) -> Self { - self.headless = Some(headless); - self - } - - /// Sets the buffer capacity for the writer - #[must_use] - pub fn buffer_capacity(mut self, capacity: usize) -> Self { - self.buffer_capacity = Some(capacity); - self - } - - /// Builds a `StreamWriter` with the configured settings - /// - /// # Arguments - /// - /// * `inner` - The writer to write binary sequence data to - /// - /// # Returns - /// - /// * `Ok(StreamWriter)` - A new streaming writer with the specified configuration - /// * `Err(Error)` - If building the writer fails - pub fn build(self, inner: W) -> Result> { - let Some(header) = self.header else { - return Err(WriteError::MissingHeader.into()); - }; - - let capacity = self.buffer_capacity.unwrap_or(8192); - StreamWriter::with_capacity( - inner, - capacity, - header, - self.policy.unwrap_or_default(), - self.headless.unwrap_or(false), - ) - } -} - #[cfg(test)] mod testing { @@ -848,20 +554,6 @@ mod testing { Ok(()) } - #[test] - fn test_stream_writer() -> Result<()> { - let inner = Vec::new(); - let writer = StreamWriterBuilder::default() - .header(FileHeaderBuilder::new().slen(32).build()?) - .buffer_capacity(16384) - .build(inner)?; - - // Convert back to Vec to verify it works - let inner = writer.into_inner()?; - assert_eq!(inner.len(), SIZE_HEADER); - Ok(()) - } - // ==================== Encoder Tests ==================== #[test] @@ -935,47 +627,6 @@ mod testing { assert!(result.is_err()); } - // ==================== Deprecated Writer Methods ==================== - - #[test] - #[allow(deprecated)] - fn test_write_record_deprecated() -> Result<()> { - let mut writer = WriterBuilder::default() - .header(FileHeaderBuilder::new().slen(8).build()?) - .build(Vec::new())?; - let wrote = writer.write_record(None, b"ACGTACGT")?; - assert!(wrote); - Ok(()) - } - - #[test] - #[allow(deprecated)] - fn test_write_record_deprecated_skipped() -> Result<()> { - let mut writer = WriterBuilder::default() - .header(FileHeaderBuilder::new().slen(8).build()?) - .build(Vec::new())?; - let wrote = writer.write_record(None, b"NNNNNNNN")?; - assert!(!wrote); - Ok(()) - } - - #[test] - #[allow(deprecated)] - fn test_write_paired_record_deprecated() -> Result<()> { - let mut writer = WriterBuilder::default() - .header( - FileHeaderBuilder::new() - .slen(8) - .xlen(8) - .flags(true) - .build()?, - ) - .build(Vec::new())?; - let wrote = writer.write_paired_record(Some(5), b"ACGTACGT", b"TTGGCCAA")?; - assert!(wrote); - Ok(()) - } - // ==================== push() Tests ==================== #[test] @@ -1057,16 +708,6 @@ mod testing { // ==================== Writer Misc Tests ==================== - #[test] - fn test_new_encoder() -> Result<()> { - let writer = WriterBuilder::default() - .header(FileHeaderBuilder::new().slen(8).build()?) - .build(Vec::new())?; - let encoder = writer.new_encoder(); - assert!(encoder.sbuffer.is_empty()); - Ok(()) - } - #[test] fn test_writer_flush() -> Result<()> { let mut writer = WriterBuilder::default() @@ -1108,73 +749,4 @@ mod testing { assert!(matches!(writer.policy(), Policy::SetToA)); Ok(()) } - - // ==================== StreamWriter Tests ==================== - - #[test] - fn test_stream_writer_new() -> Result<()> { - let writer = StreamWriter::new( - Vec::new(), - FileHeaderBuilder::new().slen(8).build()?, - Policy::default(), - false, - )?; - let inner = writer.into_inner()?; - assert_eq!(inner.len(), SIZE_HEADER); - Ok(()) - } - - #[test] - #[allow(deprecated)] - fn test_stream_writer_deprecated_methods() -> Result<()> { - let mut writer = StreamWriter::new( - Vec::new(), - FileHeaderBuilder::new().slen(8).xlen(8).build()?, - Policy::default(), - false, - )?; - assert!(writer.write_record(None, b"ACGTACGT")?); - assert!(writer.write_paired_record(None, b"ACGTACGT", b"TTGGCCAA")?); - writer.flush()?; - Ok(()) - } - - #[test] - fn test_stream_writer_push() -> Result<()> { - let mut writer = StreamWriter::new( - Vec::new(), - FileHeaderBuilder::new().slen(8).build()?, - Policy::default(), - false, - )?; - let record = SequencingRecordBuilder::default() - .s_seq(b"ACGTACGT") - .build()?; - assert!(writer.push(record)?); - writer.flush()?; - let inner = writer.into_inner()?; - assert_eq!(inner.len(), SIZE_HEADER + 8); - Ok(()) - } - - // ==================== StreamWriterBuilder Tests ==================== - - #[test] - fn test_stream_writer_builder_missing_header() { - let result = StreamWriterBuilder::default().build(Vec::new()); - assert!(result.is_err()); - } - - #[test] - fn test_stream_writer_builder_with_policy_and_headless() -> Result<()> { - let inner = Vec::new(); - let writer = StreamWriterBuilder::default() - .header(FileHeaderBuilder::new().slen(8).build()?) - .policy(Policy::SetToA) - .headless(true) - .build(inner)?; - let inner = writer.into_inner()?; - assert!(inner.is_empty()); - Ok(()) - } } diff --git a/src/vbq/writer.rs b/src/vbq/writer.rs index 75e0d8b..3ebb9bd 100644 --- a/src/vbq/writer.rs +++ b/src/vbq/writer.rs @@ -433,47 +433,11 @@ impl Writer { self.header.headers } - #[deprecated(note = "use `push` method with SequencingRecord instead")] - pub fn write_record( - &mut self, - flag: Option, - header: Option<&[u8]>, - sequence: &[u8], - quality: Option<&[u8]>, - ) -> Result { - let record = SequencingRecord::new(sequence, quality, header, None, None, None, flag); - self.push(record) - } - - #[deprecated(note = "use `push` method with SequencingRecord instead")] - #[allow(clippy::too_many_arguments)] - pub fn write_paired_record( - &mut self, - flag: Option, - s_header: Option<&[u8]>, - s_sequence: &[u8], - s_qual: Option<&[u8]>, - x_header: Option<&[u8]>, - x_sequence: &[u8], - x_qual: Option<&[u8]>, - ) -> Result { - let record = SequencingRecord::new( - s_sequence, - s_qual, - s_header, - Some(x_sequence), - x_qual, - x_header, - flag, - ); - self.push(record) - } - /// Writes a record using the unified [`SequencingRecord`] API /// /// This method provides a consistent interface with BQ and CBQ writers. - /// It automatically routes to either `write_record` or `write_paired_record` - /// based on whether the record contains paired data. + /// It automatically routes to the single or paired write path based on + /// whether the record contains paired data. /// /// # Arguments /// From c3eea0709ba4a29464769042e7902ed3ca3dc7e0 Mon Sep 17 00:00:00 2001 From: noamteyssier <22600644+noamteyssier@users.noreply.github.com> Date: Tue, 25 Aug 2026 17:24:24 -0700 Subject: [PATCH 03/25] refactor(fastx): drain only completed blocks per batch The fastx encoder previously called ingest() on every batch, which for CBQ flushes the thread's incomplete block and compresses it under the global lock. Switch to ingest_completed() per batch (moves only pre-compressed bytes) with a final ingest() on thread completion, so blocks stay full across batches. This is what I discovered had large perf improvements with bqtools. This ports the pattern from examples/write.rs, which is now deleted as a redundant 292-line clone of encode_fastx (see examples/auto-write.rs). --- examples/write.rs | 292 --------------------------------------------- src/utils/fastx.rs | 18 +++ 2 files changed, 18 insertions(+), 292 deletions(-) delete mode 100644 examples/write.rs diff --git a/examples/write.rs b/examples/write.rs deleted file mode 100644 index f83a273..0000000 --- a/examples/write.rs +++ /dev/null @@ -1,292 +0,0 @@ -use std::{ - io::{BufWriter, Read}, - sync::Arc, -}; - -use anyhow::{Result, bail}; -use binseq::BitSize; -use binseq::{ - SequencingRecordBuilder, - write::{BinseqWriter, BinseqWriterBuilder, Format}, -}; -use clap::Parser; -use paraseq::{ - Record, fastx, - prelude::{IntoProcessError, PairedParallelProcessor, ParallelProcessor, ParallelReader}, -}; -use parking_lot::Mutex; - -type BoxedWriter = Box; - -#[derive(Parser)] -struct Args { - /// Input FASTX to encode into BINSEQ format - #[clap(required = true)] - input: String, - - /// Input FASTX to encode into BINSEQ format (R2) - #[clap(required = false)] - input2: Option, - - /// Output file path for BINSEQ format - #[clap(short = 'o', long)] - output: Option, - - /// Default prefix for writing BINSEQ: `.` - #[clap(short = 'p', long, default_value = "output")] - prefix: String, - - /// Format of the output BINSEQ file - /// - /// [bq: bq|BQ|b, vbq: vbq|VBQ|v, cbq: cbq|CBQ|c] - #[clap(short = 'f', long)] - format: Option, - - /// Exclude quality information in BINSEQ output - /// - /// (bq ignores quality always) - #[clap(short = 'Q', long)] - exclude_quality: bool, - - /// Exclude sequence headers in BINSEQ output - /// - /// (bq ignores headers always) - #[clap(short = 'H', long)] - exclude_headers: bool, - - /// Compression level for BINSEQ output (0: auto) - #[clap(long, default_value_t = 0)] - compression_level: i32, - - /// Default BITSIZE for BINSEQ output (2: 2bit, 4: 4bit) - #[clap(long, default_value_t = 2)] - bitsize: u8, - - /// Default BLOCKSIZE in KB for BINSEQ output (vbq,cbq) - #[clap(long, default_value_t = 128)] - blocksize: usize, - - /// Number of threads to use for parallel processing, 0: all available - #[clap(short = 'T', long, default_value = "0")] - threads: usize, -} -impl Args { - /// Determines the output format based on the file extension or the provided format - fn format(&self) -> Format { - if let Some(format) = self.format { - format - } else if let Some(output) = &self.output { - match output.split('.').next_back() { - Some("bq") => Format::Bq, - Some("vbq") => Format::Vbq, - Some("cbq") => Format::Cbq, - _ => Format::default(), - } - } else { - Format::default() - } - } - fn bitsize(&self) -> BitSize { - match self.bitsize { - 4 => BitSize::Four, - _ => BitSize::Two, - } - } - - /// Creates an output file handle - fn ohandle(&self) -> Result { - let path = if let Some(output) = &self.output { - output.clone() - } else { - format!("{}{}", self.prefix, self.format().extension()) - }; - let ofile = std::fs::File::create(path).map(BufWriter::new)?; - Ok(Box::new(ofile)) - } - - fn is_paired(&self) -> bool { - self.input2.is_some() - } -} - -/// Calculates the sequence length of the first record in the reader -fn get_seq_len(reader: &mut fastx::Reader) -> Result { - let mut rset = reader.new_record_set(); - rset.fill(reader)?; - - let slen = if let Some(record) = rset.iter().next() { - let record = record?; - record.seq().len() - } else { - bail!("No records found in reader"); - }; - - reader.reload(&mut rset)?; - - Ok(slen) -} - -#[derive(Clone)] -struct Encoder { - /// global writer - writer: Arc>>, - thread_writer: BinseqWriter>, -} -impl Encoder { - pub fn new(writer: BinseqWriter) -> Result { - let thread_writer = writer.new_headless_buffer()?; - Ok(Self { - writer: Arc::new(Mutex::new(writer)), - thread_writer, - }) - } - pub fn finish(&mut self) -> Result<()> { - self.writer.lock().finish()?; - Ok(()) - } -} -impl ParallelProcessor for Encoder { - fn process_record(&mut self, record: Rf) -> paraseq::Result<()> { - let seq = record.seq(); - let seq_record = SequencingRecordBuilder::default() - .s_header(record.id()) - .s_seq(&seq) - .opt_s_qual(record.qual()) - .build() - .map_err(IntoProcessError::into_process_error)?; - self.thread_writer - .push(seq_record) - .map_err(IntoProcessError::into_process_error)?; - Ok(()) - } - fn on_batch_complete(&mut self) -> paraseq::Result<()> { - // Drain only the already-compressed completed blocks under the lock. - // The incomplete block keeps accumulating on this thread, and its - // compression happens off-lock when it next fills up in `push`. - self.writer - .lock() - .ingest_completed(&mut self.thread_writer) - .map_err(IntoProcessError::into_process_error)?; - Ok(()) - } - - fn on_thread_complete(&mut self) -> paraseq::Result<()> { - // Flush this thread's residual incomplete block into the global writer. - self.writer - .lock() - .ingest(&mut self.thread_writer) - .map_err(IntoProcessError::into_process_error)?; - Ok(()) - } -} - -impl PairedParallelProcessor for Encoder { - fn process_record_pair(&mut self, record1: Rf, record2: Rf) -> paraseq::Result<()> { - let sseq = record1.seq(); - let xseq = record2.seq(); - let seq_record = SequencingRecordBuilder::default() - .s_header(record1.id()) - .s_seq(&sseq) - .opt_s_qual(record1.qual()) - .x_header(record2.id()) - .x_seq(&xseq) - .opt_x_qual(record2.qual()) - .build() - .map_err(IntoProcessError::into_process_error)?; - - self.thread_writer - .push(seq_record) - .map_err(IntoProcessError::into_process_error)?; - Ok(()) - } - - fn on_batch_complete(&mut self) -> paraseq::Result<()> { - // Drain only the already-compressed completed blocks under the lock. - // The incomplete block keeps accumulating on this thread, and its - // compression happens off-lock when it next fills up in `push`. - self.writer - .lock() - .ingest_completed(&mut self.thread_writer) - .map_err(IntoProcessError::into_process_error)?; - Ok(()) - } - - fn on_thread_complete(&mut self) -> paraseq::Result<()> { - // Flush this thread's residual incomplete block into the global writer. - self.writer - .lock() - .ingest(&mut self.thread_writer) - .map_err(IntoProcessError::into_process_error)?; - Ok(()) - } -} - -fn encode_paired(args: &Args) -> Result<()> { - let mut r1 = fastx::Reader::from_path(&args.input)?; - let mut r2 = fastx::Reader::from_path(args.input2.as_ref().expect("Missing input2"))?; - let ohandle = args.ohandle()?; - - // prepare writer - let writer = { - let format = args.format(); - let mut builder = BinseqWriterBuilder::new(format) - .headers(!args.exclude_headers) - .quality(!args.exclude_quality) - .compression_level(args.compression_level) - .bitsize(args.bitsize()) - .paired(true) - .block_size(args.blocksize * 1024); - - // BQ requires a fixed sequence length from init time - if matches!(format, Format::Bq) { - builder = builder.slen(get_seq_len(&mut r1)? as u32); - builder = builder.xlen(get_seq_len(&mut r2)? as u32); - } - - builder.build(ohandle)? - }; - - let mut encoder = Encoder::new(writer)?; - r1.process_parallel_paired(r2, &mut encoder, args.threads)?; - encoder.finish()?; - - Ok(()) -} - -fn encode_single(args: &Args) -> Result<()> { - let mut reader = fastx::Reader::from_path(&args.input)?; - let ohandle = args.ohandle()?; - - // prepare writer - let writer = { - let format = args.format(); - let mut builder = BinseqWriterBuilder::new(format) - .headers(!args.exclude_headers) - .quality(!args.exclude_quality) - .compression_level(args.compression_level) - .bitsize(args.bitsize()) - .block_size(args.blocksize * 1024); - - // BQ requires a fixed sequence length from init time - if matches!(format, Format::Bq) { - builder = builder.slen(get_seq_len(&mut reader)? as u32); - } - - builder.build(ohandle)? - }; - - let mut encoder = Encoder::new(writer)?; - reader.process_parallel(&mut encoder, args.threads)?; - encoder.finish()?; - - Ok(()) -} - -fn main() -> Result<()> { - let args = Args::parse(); - if args.is_paired() { - encode_paired(&args) - } else { - encode_single(&args) - } -} diff --git a/src/utils/fastx.rs b/src/utils/fastx.rs index 05c9b18..2a34919 100644 --- a/src/utils/fastx.rs +++ b/src/utils/fastx.rs @@ -355,6 +355,15 @@ impl ParallelProcessor for Encoder { } fn on_batch_complete(&mut self) -> paraseq::Result<()> { + // Only drain completed blocks mid-stream (keeps CBQ blocks full) + self.writer + .lock() + .ingest_completed(&mut self.thread_writer) + .map_err(IntoProcessError::into_process_error)?; + Ok(()) + } + + fn on_thread_complete(&mut self) -> paraseq::Result<()> { self.writer .lock() .ingest(&mut self.thread_writer) @@ -384,6 +393,15 @@ impl PairedParallelProcessor for Encoder { } fn on_batch_complete(&mut self) -> paraseq::Result<()> { + // Only drain completed blocks mid-stream (keeps CBQ blocks full) + self.writer + .lock() + .ingest_completed(&mut self.thread_writer) + .map_err(IntoProcessError::into_process_error)?; + Ok(()) + } + + fn on_thread_complete(&mut self) -> paraseq::Result<()> { self.writer .lock() .ingest(&mut self.thread_writer) From 40a13b8d7c3c9c7907bd30ec41e6b826bca9c469 Mon Sep 17 00:00:00 2001 From: noamteyssier <22600644+noamteyssier@users.noreply.github.com> Date: Tue, 25 Aug 2026 18:27:40 -0700 Subject: [PATCH 04/25] refactor: delete dead code across bq, vbq, write, and record MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removes API with no callers in-repo or in downstream consumers (bqtools, cyto): the vestigial BlockIndex::from_vbq file scanner, redundant FileHeader constructors and set_bitsize setters, the SequencingRecord getters (fields are now pub), the unused opt_* builder setters, and RefRecord::config/set_id. The from_*_header constructors shrink to struct-update syntax and the inherent BinseqReader::process_parallel_range becomes a one-line delegation to the ParallelReader impl — both kept because bqtools uses them. BREAKING: vbq::BlockIndex::from_vbq, FileHeader::new/new_extended/ set_bitsize (bq and vbq), vbq::FileHeader::with_capacity, and the SequencingRecord getters are removed; SequencingRecord fields are now public. --- src/bq/header.rs | 98 +----- src/bq/reader.rs | 19 -- src/cbq/core/block.rs | 4 +- src/parallel.rs | 26 +- src/record/sequencing_record.rs | 147 ++------- src/vbq/header.rs | 148 ++------- src/vbq/index.rs | 166 +--------- src/write.rs | 520 +++++--------------------------- 8 files changed, 138 insertions(+), 990 deletions(-) diff --git a/src/bq/header.rs b/src/bq/header.rs index 1290217..910fd3f 100644 --- a/src/bq/header.rs +++ b/src/bq/header.rs @@ -141,67 +141,6 @@ pub struct FileHeader { pub reserved: [u8; 17], } impl FileHeader { - /// Creates a new header with the specified sequence length - /// - /// This constructor initializes a standard header with the given sequence length, - /// setting the magic number and format version to their default values. - /// The extended sequence length (xlen) is set to 0. - /// - /// # Arguments - /// - /// * `bits` - The number of bits per nucleotide (currently 2 or 4) - /// * `slen` - The length of sequences in the file - /// * `flags` - The flags for the header - /// - /// # Returns - /// - /// A new `FileHeader` instance - #[must_use] - pub fn new(bits: BitSize, slen: u32, flags: bool) -> Self { - Self { - magic: MAGIC, - format: FORMAT, - slen, - xlen: 0, - bits, - flags, - reserved: RESERVED, - } - } - - /// Creates a new header with both primary and extended sequence lengths - /// - /// This constructor initializes a header for files that contain both primary - /// and secondary sequence data, such as quality scores or annotations. - /// - /// # Arguments - /// - /// * `bits` - The number of bits per nucleotide (currently 2 or 4) - /// * `slen` - The length of primary sequences in the file - /// * `xlen` - The length of secondary/extended sequences in the file - /// * `flags` - The flags for the header - /// - /// # Returns - /// - /// A new `FileHeader` instance with extended sequence information - #[must_use] - pub fn new_extended(bits: BitSize, slen: u32, xlen: u32, flags: bool) -> Self { - Self { - magic: MAGIC, - format: FORMAT, - slen, - xlen, - bits, - flags, - reserved: RESERVED, - } - } - - /// Sets the bitsize of the header - pub fn set_bitsize(&mut self, bits: BitSize) { - self.bits = bits; - } - /// Checks if the file is paired #[must_use] pub fn is_paired(&self) -> bool { @@ -373,38 +312,11 @@ mod tests { assert!(result.is_err()); } - // ==================== FileHeader Constructor Tests ==================== - - #[test] - fn test_header_new() { - let header = FileHeader::new(BitSize::Two, 100, true); - assert_eq!(header.slen, 100); - assert_eq!(header.xlen, 0); - assert!(header.flags); - assert!(!header.is_paired()); - } - - #[test] - fn test_header_new_extended() { - let header = FileHeader::new_extended(BitSize::Four, 100, 50, false); - assert_eq!(header.slen, 100); - assert_eq!(header.xlen, 50); - assert_eq!(header.bits, BitSize::Four); - assert!(header.is_paired()); - } - - #[test] - fn test_set_bitsize() { - let mut header = FileHeader::new(BitSize::Two, 100, false); - header.set_bitsize(BitSize::Four); - assert_eq!(header.bits, BitSize::Four); - } - // ==================== from_bytes Tests ==================== #[test] fn test_from_bytes_invalid_format_version() { - let header = FileHeader::new(BitSize::Two, 32, false); + let header = FileHeaderBuilder::new().slen(32).build().unwrap(); let mut buffer = [0u8; SIZE_HEADER]; let mut cursor = std::io::Cursor::new(&mut buffer[..]); header.write_bytes(&mut cursor).unwrap(); @@ -415,7 +327,7 @@ mod tests { #[test] fn test_from_bytes_four_bit_size() { - let header = FileHeader::new(BitSize::Four, 32, false); + let header = FileHeaderBuilder::new().bitsize(BitSize::Four).slen(32).build().unwrap(); let mut buffer = [0u8; SIZE_HEADER]; let mut cursor = std::io::Cursor::new(&mut buffer[..]); header.write_bytes(&mut cursor).unwrap(); @@ -425,7 +337,7 @@ mod tests { #[test] fn test_from_bytes_invalid_bitsize() { - let header = FileHeader::new(BitSize::Two, 32, false); + let header = FileHeaderBuilder::new().slen(32).build().unwrap(); let mut buffer = [0u8; SIZE_HEADER]; let mut cursor = std::io::Cursor::new(&mut buffer[..]); header.write_bytes(&mut cursor).unwrap(); @@ -452,7 +364,7 @@ mod tests { #[test] fn test_from_buffer_valid() { - let header = FileHeader::new(BitSize::Two, 32, false); + let header = FileHeaderBuilder::new().slen(32).build().unwrap(); let mut buffer = Vec::new(); header.write_bytes(&mut buffer).unwrap(); buffer.extend_from_slice(&[0u8; 16]); // trailing data beyond header @@ -464,7 +376,7 @@ mod tests { #[test] fn test_from_reader_valid() { - let header = FileHeader::new_extended(BitSize::Two, 32, 16, true); + let header = FileHeaderBuilder::new().slen(32).xlen(16).flags(true).build().unwrap(); let mut buffer = Vec::new(); header.write_bytes(&mut buffer).unwrap(); let mut cursor = std::io::Cursor::new(buffer); diff --git a/src/bq/reader.rs b/src/bq/reader.rs index 679b210..012498d 100644 --- a/src/bq/reader.rs +++ b/src/bq/reader.rs @@ -72,18 +72,6 @@ impl<'a> RefRecord<'a> { header_len: 0, } } - /// Returns the record's configuration - /// - /// The configuration defines the layout and size of the record's components. - #[must_use] - pub fn config(&self) -> RecordConfig { - self.config - } - - pub fn set_id(&mut self, id: &[u8]) { - self.header_len = id.len(); - self.header_buf[..self.header_len].copy_from_slice(id); - } } impl BinseqRecord for RefRecord<'_> { @@ -312,13 +300,6 @@ impl RecordConfig { ) } - /// Returns whether this record contains extended sequence data - /// - /// A record is considered paired if it has a non-zero extended sequence length. - pub fn paired(&self) -> bool { - self.xlen > 0 - } - /// Returns the primary sequence length in base pairs /// /// This method returns the length of the primary sequence in base pairs. diff --git a/src/cbq/core/block.rs b/src/cbq/core/block.rs index 1045fd7..669231b 100644 --- a/src/cbq/core/block.rs +++ b/src/cbq/core/block.rs @@ -220,7 +220,7 @@ impl ColumnarBlock { /// Note: this does not check if quality scores are different lengths from sequence fn add_quality(&mut self, record: &SequencingRecord) -> Result<()> { if self.header.has_qualities() { - let Some(squal) = record.s_qual() else { + let Some(squal) = record.s_qual else { return Err(WriteError::ConfigurationMismatch { attribute: "s_qual", expected: true, @@ -231,7 +231,7 @@ impl ColumnarBlock { self.qual.extend_from_slice(squal); if self.header.is_paired() { - let Some(xqual) = record.x_qual() else { + let Some(xqual) = record.x_qual else { return Err(WriteError::ConfigurationMismatch { attribute: "x_qual", expected: true, diff --git a/src/parallel.rs b/src/parallel.rs index eedd96c..b969da5 100644 --- a/src/parallel.rs +++ b/src/parallel.rs @@ -88,32 +88,15 @@ impl BinseqReader { /// Process records in parallel within a specified range /// - /// This method allows parallel processing of a subset of records within the file, - /// defined by a start and end index. The range is distributed across the specified - /// number of threads. - /// - /// # Arguments - /// - /// * `processor` - The processor to use for each record - /// * `num_threads` - The number of threads to spawn - /// * `start` - The starting record index (inclusive) - /// * `end` - The ending record index (exclusive) - /// - /// # Returns - /// - /// * `Ok(())` - If all records were processed successfully - /// * `Err(Error)` - If an error occurred during processing + /// Inherent convenience for [`ParallelReader::process_parallel_range`], callable + /// without importing the trait. pub fn process_parallel_range( self, processor: P, num_threads: usize, range: Range, ) -> Result<()> { - match self { - Self::Bq(reader) => reader.process_parallel_range(processor, num_threads, range), - Self::Vbq(reader) => reader.process_parallel_range(processor, num_threads, range), - Self::Cbq(reader) => reader.process_parallel_range(processor, num_threads, range), - } + ::process_parallel_range(self, processor, num_threads, range) } } impl ParallelReader for BinseqReader { @@ -224,14 +207,12 @@ pub trait ParallelProcessor: Send + Clone { /// Called when a thread finishes processing its batch /// Default implementation does nothing - #[allow(unused_variables)] fn on_batch_complete(&mut self) -> Result<()> { Ok(()) } /// Called when a thread finished processing all its batches /// Default implementation does nothing - #[allow(unused_variables)] fn on_thread_complete(&mut self) -> Result<()> { Ok(()) } @@ -239,7 +220,6 @@ pub trait ParallelProcessor: Send + Clone { /// Set the thread ID for this processor /// /// Each thread should call this method with its own unique ID. - #[allow(unused_variables)] fn set_tid(&mut self, _tid: usize) { // Default implementation does nothing } diff --git a/src/record/sequencing_record.rs b/src/record/sequencing_record.rs index b5da435..ba3a876 100644 --- a/src/record/sequencing_record.rs +++ b/src/record/sequencing_record.rs @@ -20,13 +20,13 @@ use crate::{BitSize, Result, error::WriteError}; /// ``` #[derive(Clone, Copy, Default)] pub struct SequencingRecord<'a> { - pub(crate) s_seq: &'a [u8], - pub(crate) s_qual: Option<&'a [u8]>, - pub(crate) s_header: Option<&'a [u8]>, - pub(crate) x_seq: Option<&'a [u8]>, - pub(crate) x_qual: Option<&'a [u8]>, - pub(crate) x_header: Option<&'a [u8]>, - pub(crate) flag: Option, + pub s_seq: &'a [u8], + pub s_qual: Option<&'a [u8]>, + pub s_header: Option<&'a [u8]>, + pub x_seq: Option<&'a [u8]>, + pub x_qual: Option<&'a [u8]>, + pub x_header: Option<&'a [u8]>, + pub flag: Option, } impl<'a> SequencingRecord<'a> { @@ -52,55 +52,6 @@ impl<'a> SequencingRecord<'a> { } } - /// Returns the primary sequence - #[inline] - #[must_use] - pub fn s_seq(&self) -> &'a [u8] { - self.s_seq - } - - /// Returns the primary quality scores if present - #[inline] - #[must_use] - pub fn s_qual(&self) -> Option<&'a [u8]> { - self.s_qual - } - - /// Returns the primary header if present - #[inline] - #[must_use] - pub fn s_header(&self) -> Option<&'a [u8]> { - self.s_header - } - - /// Returns the extended/paired sequence if present - #[inline] - #[must_use] - pub fn x_seq(&self) -> Option<&'a [u8]> { - self.x_seq - } - - /// Returns the extended quality scores if present - #[inline] - #[must_use] - pub fn x_qual(&self) -> Option<&'a [u8]> { - self.x_qual - } - - /// Returns the extended header if present - #[inline] - #[must_use] - pub fn x_header(&self) -> Option<&'a [u8]> { - self.x_header - } - - /// Returns the flag if present - #[inline] - #[must_use] - pub fn flag(&self) -> Option { - self.flag - } - /// Returns the configured size of this record for CBQ format. /// /// CBQ uses columnar storage so there are no per-record length prefixes. @@ -318,13 +269,6 @@ impl<'a> SequencingRecordBuilder<'a> { self } - /// Sets the primary header from an Option - #[must_use] - pub fn opt_s_header(mut self, s_header: Option<&'a [u8]>) -> Self { - self.s_header = s_header; - self - } - /// Sets the extended/paired sequence #[must_use] pub fn x_seq(mut self, x_seq: &'a [u8]) -> Self { @@ -332,13 +276,6 @@ impl<'a> SequencingRecordBuilder<'a> { self } - /// Sets the extended/paired sequence from an Option - #[must_use] - pub fn opt_x_seq(mut self, x_seq: Option<&'a [u8]>) -> Self { - self.x_seq = x_seq; - self - } - /// Sets the extended quality scores #[must_use] pub fn x_qual(mut self, x_qual: &'a [u8]) -> Self { @@ -360,13 +297,6 @@ impl<'a> SequencingRecordBuilder<'a> { self } - /// Sets the extended header from an Option - #[must_use] - pub fn opt_x_header(mut self, x_header: Option<&'a [u8]>) -> Self { - self.x_header = x_header; - self - } - /// Sets the flag value #[must_use] pub fn flag(mut self, flag: u64) -> Self { @@ -374,13 +304,6 @@ impl<'a> SequencingRecordBuilder<'a> { self } - /// Sets the flag value from an Option - #[must_use] - pub fn opt_flag(mut self, flag: Option) -> Self { - self.flag = flag; - self - } - /// Builds the `SequencingRecord` /// /// # Errors @@ -419,25 +342,25 @@ mod tests { Some(b"x_hdr"), Some(7), ); - assert_eq!(record.s_seq(), b"ACGT"); - assert_eq!(record.s_qual(), Some(b"IIII".as_slice())); - assert_eq!(record.s_header(), Some(b"s_hdr".as_slice())); - assert_eq!(record.x_seq(), Some(b"TGCA".as_slice())); - assert_eq!(record.x_qual(), Some(b"FFFF".as_slice())); - assert_eq!(record.x_header(), Some(b"x_hdr".as_slice())); - assert_eq!(record.flag(), Some(7)); + assert_eq!(record.s_seq, b"ACGT"); + assert_eq!(record.s_qual, Some(b"IIII".as_slice())); + assert_eq!(record.s_header, Some(b"s_hdr".as_slice())); + assert_eq!(record.x_seq, Some(b"TGCA".as_slice())); + assert_eq!(record.x_qual, Some(b"FFFF".as_slice())); + assert_eq!(record.x_header, Some(b"x_hdr".as_slice())); + assert_eq!(record.flag, Some(7)); } #[test] fn test_new_constructor_minimal() { let record = SequencingRecord::new(b"ACGT", None, None, None, None, None, None); - assert_eq!(record.s_seq(), b"ACGT"); - assert_eq!(record.s_qual(), None); - assert_eq!(record.s_header(), None); - assert_eq!(record.x_seq(), None); - assert_eq!(record.x_qual(), None); - assert_eq!(record.x_header(), None); - assert_eq!(record.flag(), None); + assert_eq!(record.s_seq, b"ACGT"); + assert_eq!(record.s_qual, None); + assert_eq!(record.s_header, None); + assert_eq!(record.x_seq, None); + assert_eq!(record.x_qual, None); + assert_eq!(record.x_header, None); + assert_eq!(record.flag, None); assert!(!record.is_paired()); assert!(!record.has_flags()); assert!(!record.has_headers()); @@ -447,35 +370,15 @@ mod tests { // ==================== SequencingRecordBuilder opt_* setters ==================== #[test] - fn test_builder_opt_setters_some() { - let record = SequencingRecordBuilder::default() - .s_seq(b"ACGT") - .opt_s_header(Some(b"s_hdr")) - .opt_x_seq(Some(b"TGCA")) - .opt_x_qual(Some(b"FFFF")) - .opt_flag(Some(9)) - .build() - .unwrap(); - assert_eq!(record.s_header(), Some(b"s_hdr".as_slice())); - assert_eq!(record.x_seq(), Some(b"TGCA".as_slice())); - assert_eq!(record.x_qual(), Some(b"FFFF".as_slice())); - assert_eq!(record.flag(), Some(9)); - } - - #[test] - fn test_builder_opt_setters_none() { + fn test_builder_opt_setters() { let record = SequencingRecordBuilder::default() .s_seq(b"ACGT") - .opt_s_header(None) - .opt_x_seq(None) + .opt_s_qual(Some(b"FFFF")) .opt_x_qual(None) - .opt_flag(None) .build() .unwrap(); - assert_eq!(record.s_header(), None); - assert_eq!(record.x_seq(), None); - assert_eq!(record.x_qual(), None); - assert_eq!(record.flag(), None); + assert_eq!(record.s_qual, Some(b"FFFF".as_slice())); + assert_eq!(record.x_qual, None); } #[test] diff --git a/src/vbq/header.rs b/src/vbq/header.rs index 8d5fbf9..e1e3a4c 100644 --- a/src/vbq/header.rs +++ b/src/vbq/header.rs @@ -119,15 +119,16 @@ impl FileHeaderBuilder { } #[must_use] pub fn build(self) -> FileHeader { - FileHeader::with_capacity( - self.block.unwrap_or(BLOCK_SIZE), - self.qual.unwrap_or(false), - self.compressed.unwrap_or(false), - self.paired.unwrap_or(false), - self.bitsize.unwrap_or_default(), - self.headers.unwrap_or(false), - self.flags.unwrap_or(false), - ) + FileHeader { + block: self.block.unwrap_or(BLOCK_SIZE), + qual: self.qual.unwrap_or(false), + compressed: self.compressed.unwrap_or(false), + paired: self.paired.unwrap_or(false), + bits: self.bitsize.unwrap_or_default(), + headers: self.headers.unwrap_or(false), + flags: self.flags.unwrap_or(false), + ..FileHeader::default() + } } } @@ -214,103 +215,21 @@ impl Default for FileHeader { /// - Does not include sequence headers /// - Uses 2-bit nucleotide encoding fn default() -> Self { - Self::with_capacity( - BLOCK_SIZE, - false, - false, - false, - BitSize::default(), - false, - false, - ) - } -} -impl FileHeader { - /// Creates a new VBQ header with the default block size - /// - /// # Parameters - /// - /// * `qual` - Whether to include quality scores with sequences - /// * `compressed` - Whether to use ZSTD compression for blocks - /// * `paired` - Whether records contain paired sequences - /// * `bitsize` - Number of bits per nucleotide (2 or 4) - /// * `headers` - Whether to include sequence headers with records - /// - /// # Example - /// - /// ```rust - /// use binseq::vbq::FileHeaderBuilder; - /// - /// // Create header with quality scores and compression, without paired sequences - /// let header = FileHeaderBuilder::new() - /// .qual(true) - /// .compressed(true) - /// .build(); - /// ``` - #[must_use] - pub fn new( - qual: bool, - compressed: bool, - paired: bool, - bitsize: BitSize, - headers: bool, - flags: bool, - ) -> Self { - Self::with_capacity( - BLOCK_SIZE, qual, compressed, paired, bitsize, headers, flags, - ) - } - - /// Creates a new VBQ header with a custom block size - /// - /// # Parameters - /// - /// * `block` - Custom block size in bytes (virtual/uncompressed size) - /// * `qual` - Whether to include quality scores with sequences - /// * `compressed` - Whether to use ZSTD compression for blocks - /// * `paired` - Whether records contain paired sequences - /// - /// # Example - /// - /// ```rust - /// use binseq::vbq::FileHeaderBuilder; - /// - /// // Create header with a 256KB block size, with quality scores and compression - /// let header = FileHeaderBuilder::new() - /// .block(256 * 1024) - /// .qual(true) - /// .compressed(true) - /// .build(); - /// ``` - #[must_use] - pub fn with_capacity( - block: u64, - qual: bool, - compressed: bool, - paired: bool, - bitsize: BitSize, - headers: bool, - flags: bool, - ) -> Self { Self { magic: MAGIC, format: FORMAT, - block, - qual, - compressed, - paired, - headers, - flags, - bits: bitsize, + block: BLOCK_SIZE, + qual: false, + compressed: false, + paired: false, + headers: false, + flags: false, + bits: BitSize::default(), reserved: RESERVED_BYTES, } } - - /// Sets the encoding bitsize for the header. - pub fn set_bitsize(&mut self, bits: BitSize) { - self.bits = bits; - } - +} +impl FileHeader { /// Creates a header from a 32-byte buffer /// /// This function parses a raw byte buffer into a `FileHeader` structure, @@ -592,19 +511,6 @@ mod tests { // ==================== FileHeader Constructor Tests ==================== - #[test] - fn test_file_header_new() { - let header = FileHeader::new(true, true, true, BitSize::Four, true, true); - assert_eq!(header.block, BLOCK_SIZE); - assert!(header.qual); - assert!(header.compressed); - assert!(header.paired); - assert_eq!(header.bits, BitSize::Four); - assert!(header.headers); - assert!(header.flags); - assert!(header.is_paired()); - } - #[test] fn test_file_header_default() { let header = FileHeader::default(); @@ -612,18 +518,16 @@ mod tests { assert!(!header.is_paired()); } - #[test] - fn test_set_bitsize() { - let mut header = FileHeader::default(); - header.set_bitsize(BitSize::Four); - assert_eq!(header.bits, BitSize::Four); - } - // ==================== FileHeader from_bytes/from_reader Tests ==================== #[test] fn test_file_header_roundtrip() { - let header = FileHeader::new(true, false, true, BitSize::Two, true, true); + let header = FileHeaderBuilder::new() + .qual(true) + .paired(true) + .headers(true) + .flags(true) + .build(); let mut buffer = Vec::new(); header.write_bytes(&mut buffer).unwrap(); let mut cursor = std::io::Cursor::new(buffer); @@ -633,7 +537,7 @@ mod tests { #[test] fn test_file_header_from_bytes_four_bit() { - let header = FileHeader::new(false, false, false, BitSize::Four, false, false); + let header = FileHeaderBuilder::new().bitsize(BitSize::Four).build(); let mut buffer = [0u8; SIZE_HEADER]; { let mut cursor = std::io::Cursor::new(&mut buffer[..]); diff --git a/src/vbq/index.rs b/src/vbq/index.rs index 7b2c8e1..7f7e0fa 100644 --- a/src/vbq/index.rs +++ b/src/vbq/index.rs @@ -32,19 +32,11 @@ //! - Cumulative record counts changed from `u32` to `u64` //! - Support for files with more than 4 billion records -use std::{ - fs::File, - io::{Cursor, Read, Write}, - path::Path, -}; +use std::io::{Cursor, Read, Write}; use byteorder::{ByteOrder, LittleEndian}; use zstd::{Decoder, Encoder}; -use super::{ - BlockHeader, FileHeader, - header::{SIZE_BLOCK_HEADER, SIZE_HEADER}, -}; use crate::error::{IndexError, Result}; /// Size of `BlockRange` in bytes @@ -374,22 +366,18 @@ impl IndexHeader { /// the file. /// /// The index is embedded at the end of VBQ files and can be loaded using -/// `MmapReader::load_index()` or created by scanning a VBQ file using -/// `BlockIndex::from_vbq()`. Once loaded, it provides information about block +/// `MmapReader::load_index()`. Once loaded, it provides information about block /// locations, sizes, and record counts. /// /// # Examples /// /// ```rust,no_run -/// use binseq::vbq::{BlockIndex, MmapReader}; +/// use binseq::vbq::MmapReader; /// use std::path::Path; /// -/// // Create an index from a VBQ file -/// let vbq_path = Path::new("example.vbq"); -/// let index = BlockIndex::from_vbq(vbq_path).unwrap(); -/// -/// // Use the index with a reader for parallel processing -/// let reader = MmapReader::new(vbq_path).unwrap(); +/// // Load the embedded index from a VBQ file +/// let reader = MmapReader::new(Path::new("example.vbq")).unwrap(); +/// let index = reader.load_index().unwrap(); /// println!("File contains {} blocks", index.n_blocks()); /// ``` #[derive(Debug, Clone)] @@ -480,85 +468,6 @@ impl BlockIndex { self.ranges.push(range); } - /// Creates a new index by scanning a VBQ file - /// - /// This method memory-maps the specified VBQ file and scans it block by block - /// to create an index. This is primarily used internally when embedding the index - /// into VBQ files during the write process. - /// - /// # Parameters - /// - /// * `path` - Path to the VBQ file to index - /// - /// # Returns - /// - /// * `Ok(Self)` - A new `BlockIndex` containing information about all blocks in the file - /// * `Err(_)` - If an error occurred during file opening, validation, or scanning - /// - /// # Examples - /// - /// ```rust,no_run - /// use binseq::vbq::BlockIndex; - /// use std::path::Path; - /// - /// // Create an index from a VBQ file - /// let index = BlockIndex::from_vbq(Path::new("example.vbq")).unwrap(); - /// - /// // Get statistics about the file - /// println!("File contains {} blocks", index.n_blocks()); - /// - /// // Analyze the record distribution - /// if let Some(last_range) = index.ranges().last() { - /// println!("Total records: {}", last_range.cumulative_records); - /// println!("Average records per block: {}", - /// last_range.cumulative_records as f64 / index.n_blocks() as f64); - /// } - /// ``` - /// - /// # Notes - /// - /// This method uses memory mapping for efficiency, which allows the operating system - /// to load only the needed portions of the file into memory as they are accessed. - pub fn from_vbq>(path: P) -> Result { - let file = File::open(path)?; - let mmap = unsafe { memmap2::Mmap::map(&file)? }; - let file_size = mmap.len(); - - // Read header from mapped memory (unused but checks for validity) - let _header = { - let mut header_bytes = [0u8; SIZE_HEADER]; - header_bytes.copy_from_slice(&mmap[..SIZE_HEADER]); - FileHeader::from_bytes(&header_bytes)? - }; - - // Initialize position after the header - let mut pos = SIZE_HEADER; - - // Initialize the collection - let index_header = IndexHeader::new(file_size as u64); - let mut index = BlockIndex::new(index_header); - - // Find all block headers - let mut record_total = 0; - while pos < mmap.len() { - let block_header = { - let mut header_bytes = [0u8; SIZE_BLOCK_HEADER]; - header_bytes.copy_from_slice(&mmap[pos..pos + SIZE_BLOCK_HEADER]); - BlockHeader::from_bytes(&header_bytes)? - }; - index.add_range(BlockRange::new( - pos as u64, - block_header.size, - block_header.records, - record_total, - )); - pos += SIZE_BLOCK_HEADER + block_header.size as usize; - record_total += u64::from(block_header.records); - } - - Ok(index) - } - pub fn from_bytes(bytes: &[u8]) -> Result { let index_header = IndexHeader::from_bytes(bytes)?; let buffer = { @@ -634,69 +543,6 @@ impl BlockIndex { mod tests { use super::*; - /// Writes a minimal raw VBQ file (file header + block headers/data, with - /// **no** embedded index) to `path`, suitable for `BlockIndex::from_vbq`. - /// - /// `from_vbq` scans block-by-block from just after the file header to - /// EOF, so it requires the file to end exactly at the last block's data - /// -- unlike files produced by `vbq::Writer`, which always append an - /// embedded index on `finish()` that `from_vbq` cannot parse as a block. - fn write_raw_vbq_file(path: &str, blocks: &[(u64, u32)]) { - let mut buffer = Vec::new(); - FileHeader::default().write_bytes(&mut buffer).unwrap(); - for &(size, records) in blocks { - BlockHeader::new(size, records) - .write_bytes(&mut buffer) - .unwrap(); - buffer.extend(std::iter::repeat_n(0u8, size as usize)); - } - std::fs::write(path, buffer).unwrap(); - } - - // ==================== BlockIndex::from_vbq Tests ==================== - - #[test] - fn test_from_vbq() { - let path = "test_index_from_vbq.vbq"; - write_raw_vbq_file(path, &[(64, 3), (128, 5)]); - - let index = BlockIndex::from_vbq(path).unwrap(); - std::fs::remove_file(path).unwrap(); - - assert_eq!(index.n_blocks(), 2); - assert_eq!(index.num_records(), 8); - assert_eq!(index.ranges()[0].cumulative_records, 0); - assert_eq!(index.ranges()[1].cumulative_records, 3); - } - - #[test] - fn test_from_vbq_nonexistent_file() { - let result = BlockIndex::from_vbq("./data/does_not_exist.vbq"); - assert!(result.is_err()); - } - - #[test] - fn test_n_blocks() { - let path = "test_index_n_blocks.vbq"; - write_raw_vbq_file(path, &[(32, 1)]); - - let index = BlockIndex::from_vbq(path).unwrap(); - std::fs::remove_file(path).unwrap(); - - assert_eq!(index.n_blocks(), index.ranges().len()); - } - - #[test] - fn test_pprint() { - let path = "test_index_pprint.vbq"; - write_raw_vbq_file(path, &[(32, 1)]); - - // Just verify it doesn't panic - let index = BlockIndex::from_vbq(path).unwrap(); - std::fs::remove_file(path).unwrap(); - index.pprint(); - } - #[test] fn test_num_records_empty_index() { let index = BlockIndex::new(IndexHeader::new(0)); diff --git a/src/write.rs b/src/write.rs index 30dd1c2..89dd1b0 100644 --- a/src/write.rs +++ b/src/write.rs @@ -268,13 +268,7 @@ impl BinseqWriterBuilder { bitsize: Some(header.bits), paired: header.is_paired(), flags: header.flags, - compression: false, - headers: false, - quality: false, - compression_level: None, - block_size: None, - headless: false, - policy: None, + ..Self::new(Format::Bq) } } @@ -283,8 +277,6 @@ impl BinseqWriterBuilder { pub fn from_vbq_header(header: vbq::FileHeader) -> Self { Self { format: Format::Vbq, - slen: None, - xlen: None, flags: header.flags, quality: header.qual, paired: header.paired, @@ -292,9 +284,7 @@ impl BinseqWriterBuilder { headers: header.headers, compression: header.compressed, block_size: Some(header.block as usize), - policy: None, - compression_level: None, - headless: false, + ..Self::new(Format::Vbq) } } @@ -309,12 +299,7 @@ impl BinseqWriterBuilder { paired: header.is_paired(), block_size: Some(header.block_size as usize), compression_level: Some(header.compression_level as i32), - compression: false, - slen: None, - xlen: None, - bitsize: None, - policy: None, - headless: false, + ..Self::new(Format::Cbq) } } @@ -608,10 +593,8 @@ impl BinseqWriter { /// - There's an I/O error during ingestion pub fn ingest_completed(&mut self, other: &mut BinseqWriter>) -> Result<()> { match (self, other) { - (Self::Bq(dst), BinseqWriter::Bq(src)) => dst.ingest(src), - (Self::Vbq(dst), BinseqWriter::Vbq(src)) => dst.ingest(src), (Self::Cbq(dst), BinseqWriter::Cbq(src)) => dst.ingest_completed(src), - _ => Err(WriteError::FormatMismatch.into()), + (dst, src) => dst.ingest(src), } } } @@ -995,444 +978,83 @@ mod tests { .unwrap() } - // ==================== VBQ Tests ==================== - - #[test] - fn test_vbq_single_minimal_writer_minimal_record() -> Result<()> { - // Writer: single-end, no quality, no headers, no flags - // Record: single-end, no quality, no headers, no flags - // Expected: success (correctly specified) - let mut writer = BinseqWriterBuilder::new(Format::Vbq) - .paired(false) - .quality(false) - .headers(false) - .flags(false) - .build(Cursor::new(Vec::new()))?; - - let record = minimal_single_record(); - assert!(writer.push(record)?); - writer.finish()?; - Ok(()) - } - - #[test] - fn test_vbq_single_minimal_writer_full_record() -> Result<()> { - // Writer: single-end, no quality, no headers, no flags - // Record: single-end, with quality, headers, flags - // Expected: success (over-specified - extra data ignored) - let mut writer = BinseqWriterBuilder::new(Format::Vbq) - .paired(false) - .quality(false) - .headers(false) - .flags(false) - .build(Cursor::new(Vec::new()))?; + // ==================== Writer x Record Specification Matrix ==================== - let record = full_single_record(); - assert!(writer.push(record)?); - writer.finish()?; - Ok(()) - } - - #[test] - fn test_vbq_single_full_writer_minimal_record() -> Result<()> { - // Writer: single-end, with quality, headers, flags - // Record: single-end, no quality, no headers, no flags - // Expected: error (under-specified) - let mut writer = BinseqWriterBuilder::new(Format::Vbq) - .paired(false) - .quality(true) - .headers(true) - .flags(true) - .build(Cursor::new(Vec::new()))?; - - let record = minimal_single_record(); - let result = writer.push(record); - assert!(result.is_err()); - Ok(()) - } - - #[test] - fn test_vbq_single_full_writer_full_record() -> Result<()> { - // Writer: single-end, with quality, headers, flags - // Record: single-end, with quality, headers, flags - // Expected: success (correctly specified) - let mut writer = BinseqWriterBuilder::new(Format::Vbq) - .paired(false) - .quality(true) - .headers(true) - .flags(true) - .build(Cursor::new(Vec::new()))?; - - let record = full_single_record(); - assert!(writer.push(record)?); - writer.finish()?; - Ok(()) - } - - #[test] - fn test_vbq_paired_writer_single_record() -> Result<()> { - // Writer: paired - // Record: single-end - // Expected: error (under-specified - missing R2) - let mut writer = BinseqWriterBuilder::new(Format::Vbq) - .paired(true) - .quality(false) - .headers(false) - .flags(false) - .build(Cursor::new(Vec::new()))?; - - let record = minimal_single_record(); - let result = writer.push(record); - assert!(result.is_err()); - Ok(()) - } - - #[test] - fn test_vbq_single_writer_paired_record() -> Result<()> { - // Writer: single-end - // Record: paired - // Expected: success (over-specified - R2 ignored) - let mut writer = BinseqWriterBuilder::new(Format::Vbq) - .paired(false) - .quality(false) - .headers(false) - .flags(false) - .build(Cursor::new(Vec::new()))?; - - let record = minimal_paired_record(); - assert!(writer.push(record)?); - writer.finish()?; - Ok(()) - } - - #[test] - fn test_vbq_paired_minimal_writer_paired_full_record() -> Result<()> { - // Writer: paired, no quality, no headers, no flags - // Record: paired, with quality, headers, flags - // Expected: success (over-specified) - let mut writer = BinseqWriterBuilder::new(Format::Vbq) - .paired(true) - .quality(false) - .headers(false) - .flags(false) - .build(Cursor::new(Vec::new()))?; - - let record = full_paired_record(); - assert!(writer.push(record)?); - writer.finish()?; - Ok(()) - } - - #[test] - fn test_vbq_paired_full_writer_paired_full_record() -> Result<()> { - // Writer: paired, with quality, headers, flags - // Record: paired, with quality, headers, flags - // Expected: success (correctly specified) - let mut writer = BinseqWriterBuilder::new(Format::Vbq) - .paired(true) - .quality(true) - .headers(true) - .flags(true) - .build(Cursor::new(Vec::new()))?; - - let record = full_paired_record(); - assert!(writer.push(record)?); - writer.finish()?; - Ok(()) - } - - // ==================== CBQ Tests ==================== - - #[test] - fn test_cbq_single_minimal_writer_minimal_record() -> Result<()> { - // Writer: single-end, no quality, no headers, no flags - // Record: single-end, no quality, no headers, no flags - // Expected: success (correctly specified) - let mut writer = BinseqWriterBuilder::new(Format::Cbq) - .paired(false) - .quality(false) - .headers(false) - .flags(false) - .build(Cursor::new(Vec::new()))?; - - let record = minimal_single_record(); - assert!(writer.push(record)?); - writer.finish()?; - Ok(()) - } - - #[test] - fn test_cbq_single_minimal_writer_full_record() -> Result<()> { - // Writer: single-end, no quality, no headers, no flags - // Record: single-end, with quality, headers, flags - // Expected: success (over-specified - extra data ignored) - let mut writer = BinseqWriterBuilder::new(Format::Cbq) - .paired(false) - .quality(false) - .headers(false) - .flags(false) - .build(Cursor::new(Vec::new()))?; - - let record = full_single_record(); - assert!(writer.push(record)?); - writer.finish()?; - Ok(()) - } - - #[test] - fn test_cbq_single_full_writer_minimal_record() -> Result<()> { - // Writer: single-end, with quality, headers, flags - // Record: single-end, no quality, no headers, no flags - // Expected: error (under-specified) - let mut writer = BinseqWriterBuilder::new(Format::Cbq) - .paired(false) - .quality(true) - .headers(true) - .flags(true) - .build(Cursor::new(Vec::new()))?; - - let record = minimal_single_record(); - let result = writer.push(record); - assert!(result.is_err()); - Ok(()) - } + type RecordFn = fn() -> SequencingRecord<'static>; #[test] - fn test_cbq_single_full_writer_full_record() -> Result<()> { - // Writer: single-end, with quality, headers, flags - // Record: single-end, with quality, headers, flags - // Expected: success (correctly specified) - let mut writer = BinseqWriterBuilder::new(Format::Cbq) - .paired(false) - .quality(true) - .headers(true) - .flags(true) - .build(Cursor::new(Vec::new()))?; - - let record = full_single_record(); - assert!(writer.push(record)?); - writer.finish()?; - Ok(()) - } - - #[test] - fn test_cbq_paired_writer_single_record() -> Result<()> { - // Writer: paired - // Record: single-end - // Expected: error (under-specified - missing R2) - let mut writer = BinseqWriterBuilder::new(Format::Cbq) - .paired(true) - .quality(false) - .headers(false) - .flags(false) - .build(Cursor::new(Vec::new()))?; - - let record = minimal_single_record(); - let result = writer.push(record); - assert!(result.is_err()); - Ok(()) - } - - #[test] - fn test_cbq_single_writer_paired_record() -> Result<()> { - // Writer: single-end - // Record: paired - // Expected: success (over-specified - R2 ignored) - let mut writer = BinseqWriterBuilder::new(Format::Cbq) - .paired(false) - .quality(false) - .headers(false) - .flags(false) - .build(Cursor::new(Vec::new()))?; - - let record = minimal_paired_record(); - assert!(writer.push(record)?); - writer.finish()?; - Ok(()) - } - - #[test] - fn test_cbq_paired_minimal_writer_paired_full_record() -> Result<()> { - // Writer: paired, no quality, no headers, no flags - // Record: paired, with quality, headers, flags - // Expected: success (over-specified) - let mut writer = BinseqWriterBuilder::new(Format::Cbq) - .paired(true) - .quality(false) - .headers(false) - .flags(false) - .build(Cursor::new(Vec::new()))?; - - let record = full_paired_record(); - assert!(writer.push(record)?); - writer.finish()?; - Ok(()) - } - - #[test] - fn test_cbq_paired_full_writer_paired_full_record() -> Result<()> { - // Writer: paired, with quality, headers, flags - // Record: paired, with quality, headers, flags - // Expected: success (correctly specified) - let mut writer = BinseqWriterBuilder::new(Format::Cbq) - .paired(true) - .quality(true) - .headers(true) - .flags(true) - .build(Cursor::new(Vec::new()))?; - - let record = full_paired_record(); - assert!(writer.push(record)?); - writer.finish()?; - Ok(()) - } - - // ==================== BQ Tests ==================== - // Note: BQ format has fixed-length sequences and doesn't support headers - - #[test] - fn test_bq_single_minimal_writer_minimal_record() -> Result<()> { - // Writer: single-end, no quality, no flags - // Record: single-end, no quality, no flags - // Expected: success (correctly specified) - let mut writer = BinseqWriterBuilder::new(Format::Bq) - .slen(32) - .paired(false) - .quality(false) - .flags(false) - .build(Cursor::new(Vec::new()))?; - - let record = minimal_single_record(); - assert!(writer.push(record)?); - writer.finish()?; - Ok(()) - } - - #[test] - fn test_bq_single_minimal_writer_full_record() -> Result<()> { - // Writer: single-end, no quality, no flags - // Record: single-end, with quality, headers, flags - // Expected: success (over-specified - extra data ignored) - let mut writer = BinseqWriterBuilder::new(Format::Bq) - .slen(32) - .paired(false) - .quality(false) - .flags(false) - .build(Cursor::new(Vec::new()))?; - - let record = full_single_record(); - assert!(writer.push(record)?); - writer.finish()?; - Ok(()) - } - - #[test] - fn test_bq_single_with_quality_writer_minimal_record() -> Result<()> { - // Writer: single-end, with quality (note: BQ ignores quality setting) - // Record: single-end, no quality - // Expected: success (BQ format doesn't support quality scores, setting is ignored) - let mut writer = BinseqWriterBuilder::new(Format::Bq) - .slen(32) - .paired(false) - .quality(true) // This is ignored for BQ format - .build(Cursor::new(Vec::new()))?; - - // BQ always reports has_quality as false - assert!(!writer.has_quality()); - - let record = minimal_single_record(); - assert!(writer.push(record)?); - writer.finish()?; - Ok(()) - } - - #[test] - fn test_bq_single_with_quality_writer_full_record() -> Result<()> { - // Writer: single-end, with quality - // Record: single-end, with quality - // Expected: success (correctly specified) - let mut writer = BinseqWriterBuilder::new(Format::Bq) - .slen(32) - .paired(false) - .quality(true) - .build(Cursor::new(Vec::new()))?; - - let record = full_single_record(); - assert!(writer.push(record)?); - writer.finish()?; - Ok(()) - } - - #[test] - fn test_bq_paired_writer_single_record() -> Result<()> { - // Writer: paired - // Record: single-end - // Expected: error (under-specified - missing R2) - let mut writer = BinseqWriterBuilder::new(Format::Bq) - .slen(32) - .xlen(32) - .paired(true) - .quality(false) - .build(Cursor::new(Vec::new()))?; - - let record = minimal_single_record(); - let result = writer.push(record); - assert!(result.is_err()); - Ok(()) - } - - #[test] - fn test_bq_single_writer_paired_record() -> Result<()> { - // Writer: single-end - // Record: paired - // Expected: success (over-specified - R2 ignored) - let mut writer = BinseqWriterBuilder::new(Format::Bq) - .slen(32) - .paired(false) - .quality(false) - .build(Cursor::new(Vec::new()))?; - - let record = minimal_paired_record(); - assert!(writer.push(record)?); - writer.finish()?; - Ok(()) - } - - #[test] - fn test_bq_paired_minimal_writer_paired_full_record() -> Result<()> { - // Writer: paired, no quality, no flags - // Record: paired, with quality, headers, flags - // Expected: success (over-specified) - let mut writer = BinseqWriterBuilder::new(Format::Bq) - .slen(32) - .xlen(32) - .paired(true) - .quality(false) - .flags(false) - .build(Cursor::new(Vec::new()))?; - - let record = full_paired_record(); - assert!(writer.push(record)?); - writer.finish()?; + fn test_record_specification_matrix() -> Result<()> { + // (format, paired, quality, headers, flags, record, expect_ok) + // + // - Under-specified: record is missing data the writer needs -> error + // - Over-specified: record has extra data the writer ignores -> success + // - Correctly-specified: record matches writer config exactly -> success + // BQ ignores quality/headers settings entirely. + #[rustfmt::skip] + let cases: &[(Format, bool, bool, bool, bool, RecordFn, bool)] = &[ + (Format::Vbq, false, false, false, false, minimal_single_record, true), + (Format::Vbq, false, false, false, false, full_single_record, true), + (Format::Vbq, false, true, true, true, minimal_single_record, false), + (Format::Vbq, false, true, true, true, full_single_record, true), + (Format::Vbq, true, false, false, false, minimal_single_record, false), + (Format::Vbq, false, false, false, false, minimal_paired_record, true), + (Format::Vbq, true, false, false, false, full_paired_record, true), + (Format::Vbq, true, true, true, true, full_paired_record, true), + (Format::Cbq, false, false, false, false, minimal_single_record, true), + (Format::Cbq, false, false, false, false, full_single_record, true), + (Format::Cbq, false, true, true, true, minimal_single_record, false), + (Format::Cbq, false, true, true, true, full_single_record, true), + (Format::Cbq, true, false, false, false, minimal_single_record, false), + (Format::Cbq, false, false, false, false, minimal_paired_record, true), + (Format::Cbq, true, false, false, false, full_paired_record, true), + (Format::Cbq, true, true, true, true, full_paired_record, true), + (Format::Bq, false, false, false, false, minimal_single_record, true), + (Format::Bq, false, false, false, false, full_single_record, true), + (Format::Bq, false, true, false, false, minimal_single_record, true), + (Format::Bq, false, true, false, false, full_single_record, true), + (Format::Bq, true, false, false, false, minimal_single_record, false), + (Format::Bq, false, false, false, false, minimal_paired_record, true), + (Format::Bq, true, false, false, false, full_paired_record, true), + (Format::Bq, true, true, false, true, full_paired_record, true), + ]; + + for &(format, paired, quality, headers, flags, record, expect_ok) in cases { + let mut builder = BinseqWriterBuilder::new(format) + .paired(paired) + .quality(quality) + .headers(headers) + .flags(flags); + // BQ is fixed-length: sequence lengths come from the header + if matches!(format, Format::Bq) { + builder = builder.slen(32); + if paired { + builder = builder.xlen(32); + } + } + let mut writer = builder.build(Cursor::new(Vec::new()))?; + + let case = format!( + "{format:?} paired={paired} quality={quality} headers={headers} flags={flags}" + ); + let result = writer.push(record()); + if expect_ok { + assert!(result.unwrap_or_else(|e| panic!("{case}: {e}")), "{case}"); + writer.finish()?; + } else { + assert!(result.is_err(), "{case}: expected error"); + } + } Ok(()) } #[test] - fn test_bq_paired_full_writer_paired_full_record() -> Result<()> { - // Writer: paired, with quality, flags - // Record: paired, with quality, headers, flags - // Expected: success (correctly specified, headers ignored for BQ) - let mut writer = BinseqWriterBuilder::new(Format::Bq) + fn test_bq_ignores_quality_flag() -> Result<()> { + // BQ format doesn't support quality scores; the setting is ignored + let writer = BinseqWriterBuilder::new(Format::Bq) .slen(32) - .xlen(32) - .paired(true) .quality(true) - .flags(true) .build(Cursor::new(Vec::new()))?; - - let record = full_paired_record(); - assert!(writer.push(record)?); - writer.finish()?; + assert!(!writer.has_quality()); Ok(()) } From 374d6e0467377ea2eb40ce40ad573ee0fbcd36a1 Mon Sep 17 00:00:00 2001 From: noamteyssier <22600644+noamteyssier@users.noreply.github.com> Date: Tue, 25 Aug 2026 18:39:58 -0700 Subject: [PATCH 05/25] refactor(cbq): delete dead code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removes the unused usage()/bytes_written() writer accessors, the with_compression_level builder setter (only the with_optional_* variant is called), the write-only num_npos block field, and a stale dead_code allow. The BlockIter struct collapses into iter().copied(), and average_block_size (used by bqtools info) is rewritten over windows(2) — same value, but single-block files now report 0.0 instead of NaN. --- src/cbq/core/block.rs | 6 ----- src/cbq/core/block_header.rs | 1 - src/cbq/core/header.rs | 5 ---- src/cbq/core/index.rs | 46 +++++++++--------------------------- src/cbq/read.rs | 33 -------------------------- src/cbq/write.rs | 11 --------- 6 files changed, 11 insertions(+), 91 deletions(-) diff --git a/src/cbq/core/block.rs b/src/cbq/core/block.rs index 669231b..6fa3e51 100644 --- a/src/cbq/core/block.rs +++ b/src/cbq/core/block.rs @@ -70,8 +70,6 @@ pub struct ColumnarBlock { /// Total nucleotides in this block pub(crate) nuclen: usize, - /// Number of npos positions - pub(crate) num_npos: usize, /// Current size of this block (virtual) current_size: usize, @@ -113,7 +111,6 @@ impl ColumnarBlock { self.num_sequences = 0; self.num_records = 0; self.current_size = 0; - self.num_npos = 0; self.len_nef = 0; } @@ -369,7 +366,6 @@ impl ColumnarBlock { /// Find all positions of 'N' in the sequence fn fill_npos(&mut self) -> Result<()> { bitnuc::ambiguous_bases(&self.seq, &mut self.npos); - self.num_npos = self.npos.len(); // build Elias-Fano encoding for N positions if self.npos.is_empty() { @@ -462,7 +458,6 @@ impl ColumnarBlock { copy_decode(self.z_npos.as_slice(), &mut self.ef_bytes)?; let ef = EliasFano::deserialize_from(self.ef_bytes.as_slice())?; - self.num_npos = ef.len(); self.ef = Some(ef); } @@ -630,7 +625,6 @@ impl ColumnarBlock { // reinitialize the EliasFano encoding let ef = EliasFano::deserialize_from(self.ef_bytes.as_slice())?; - self.num_npos = ef.len(); self.ef = Some(ef); } diff --git a/src/cbq/core/block_header.rs b/src/cbq/core/block_header.rs index a63e9e7..3f86895 100644 --- a/src/cbq/core/block_header.rs +++ b/src/cbq/core/block_header.rs @@ -59,7 +59,6 @@ impl BlockHeader { } /// Calculate the length of the block in bytes. - #[allow(dead_code)] #[must_use] pub fn block_len(&self) -> usize { (self.len_z_seq_len diff --git a/src/cbq/core/header.rs b/src/cbq/core/header.rs index 287d197..92c6145 100644 --- a/src/cbq/core/header.rs +++ b/src/cbq/core/header.rs @@ -139,11 +139,6 @@ pub struct FileHeaderBuilder { } impl FileHeaderBuilder { - pub fn with_compression_level(&mut self, compression_level: usize) -> &mut Self { - self.compression_level = Some(compression_level); - self - } - pub fn with_optional_compression_level( &mut self, compression_level: Option, diff --git a/src/cbq/core/index.rs b/src/cbq/core/index.rs index e61657f..29ed26b 100644 --- a/src/cbq/core/index.rs +++ b/src/cbq/core/index.rs @@ -150,31 +150,25 @@ impl Index { self.ranges.len() } - #[must_use] - pub fn iter_blocks(&self) -> BlockIter<'_> { - BlockIter { - index: self, - pos: 0, - } + pub fn iter_blocks(&self) -> impl Iterator + '_ { + self.ranges.iter().copied() } + /// Returns the average block size in bytes (0.0 for indexes with fewer than two blocks) #[must_use] pub fn average_block_size(&self) -> f64 { - let mut block_iter = self.iter_blocks(); - let Some(mut last_block) = block_iter.next() else { + if self.ranges.len() < 2 { return 0.0; - }; - let mut total_size = 0.0; - let mut count = 0; - for block in block_iter { - let last_block_size = block.offset - last_block.offset; - total_size += last_block_size as f64; - count += 1; - last_block = block; } - total_size / f64::from(count) + let total: u64 = self + .ranges + .windows(2) + .map(|pair| pair[1].offset - pair[0].offset) + .sum(); + total as f64 / (self.ranges.len() - 1) as f64 } + /// Prints a debug summary of each block range to stdout pub fn pprint(&self) { for block in self.iter_blocks() { println!("{block:?}"); @@ -182,24 +176,6 @@ impl Index { } } -pub struct BlockIter<'a> { - index: &'a Index, - pos: usize, -} -impl Iterator for BlockIter<'_> { - type Item = BlockRange; - - fn next(&mut self) -> Option { - if self.pos >= self.index.num_blocks() { - None - } else { - let block = self.index.ranges[self.pos]; - self.pos += 1; - Some(block) - } - } -} - /// A struct representing a block range in a CBQ file and stored in the [`Index`](crate::cbq::Index) /// /// This is stored identically in memory and on disk. diff --git a/src/cbq/read.rs b/src/cbq/read.rs index 9d16cf2..ec3f0b4 100644 --- a/src/cbq/read.rs +++ b/src/cbq/read.rs @@ -582,39 +582,6 @@ mod tests { assert!(result.is_err(), "Should fail on invalid file format"); } - // ==================== Block Header Iterator Tests ==================== - - #[test] - fn test_iter_block_headers() { - let reader = MmapReader::new(TEST_CBQ_FILE).unwrap(); - - let headers: Vec<_> = reader - .iter_block_headers() - .take(5) - .collect::>>() - .unwrap(); - - assert!(!headers.is_empty(), "Should have at least one block header"); - - for header in headers { - assert!(header.num_records > 0, "Block should have records"); - } - } - - #[test] - fn test_iter_block_headers_count() { - let reader = MmapReader::new(TEST_CBQ_FILE).unwrap(); - - let header_count = reader - .iter_block_headers() - .collect::>>() - .unwrap() - .len(); - - let num_blocks = reader.num_blocks(); - assert_eq!(header_count, num_blocks, "Should iterate all block headers"); - } - // ==================== Empty Range Tests ==================== #[test] diff --git a/src/cbq/write.rs b/src/cbq/write.rs index e94df6c..90cb821 100644 --- a/src/cbq/write.rs +++ b/src/cbq/write.rs @@ -84,11 +84,6 @@ impl ColumnarBlockWriter { self.block.header } - /// Calculate the usage of the block as a percentage - pub fn usage(&self) -> f64 { - self.block.usage() - } - /// Push a record to the writer /// /// Returns `Ok(true)` if the record was written successfully. @@ -211,12 +206,6 @@ impl ColumnarBlockWriter> { pub fn clear_incomplete_data(&mut self) { self.block.clear(); } - - /// Returns the number of bytes written to the inner data structure - #[must_use] - pub fn bytes_written(&self) -> usize { - self.inner.len() - } } #[cfg(test)] From 75d88bb4e701e2a1088a946d33ccad9bf6078732 Mon Sep 17 00:00:00 2001 From: noamteyssier <22600644+noamteyssier@users.noreply.github.com> Date: Tue, 25 Aug 2026 18:46:46 -0700 Subject: [PATCH 06/25] dep: replace byteorder with std to_le_bytes / from_le_bytes Every use was fixed-offset LE read/write. Two helpers (read_u64_le / read_u32_le) cover the reads. writes become copy_from_slice/write_all over to_le_bytes. --- Cargo.toml | 1 - src/bq/header.rs | 31 +++++++++++++++++++++---------- src/utils/mod.rs | 10 ++++++++++ src/vbq/header.rs | 26 ++++++++++++++------------ src/vbq/index.rs | 29 +++++++++++++++-------------- src/vbq/reader.rs | 21 +++++++++++---------- src/vbq/writer.rs | 20 +++++++++----------- 7 files changed, 80 insertions(+), 58 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index bb976a0..f22c897 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,6 @@ auto_impl = "1.3.0" bitnuc-deprec = { package ="bitnuc", version = "0.4.1" } bitnuc = { version = "0.5.1" } bytemuck = { version = "1.25.1", features = ["derive", "extern_crate_alloc"] } -byteorder = "1.5.0" itoa = "1.0.18" memchr = "2.8.3" memmap2 = "0.9.11" diff --git a/src/bq/header.rs b/src/bq/header.rs index 910fd3f..f55d3ab 100644 --- a/src/bq/header.rs +++ b/src/bq/header.rs @@ -5,10 +5,12 @@ //! sequence length, and other information necessary for proper interpretation of the data. use bitnuc_deprec::BitSize; -use byteorder::{ByteOrder, LittleEndian}; use std::io::{Read, Write}; -use crate::error::{HeaderError, Result}; +use crate::{ + error::{HeaderError, Result}, + utils::read_u32_le, +}; /// Current magic number: "BSEQ" in ASCII (in little-endian byte order) /// @@ -168,7 +170,7 @@ impl FileHeader { /// * The format version is unsupported /// * The reserved bytes are invalid pub fn from_bytes(buffer: &[u8; SIZE_HEADER]) -> Result { - let magic = LittleEndian::read_u32(&buffer[0..4]); + let magic = read_u32_le(&buffer[0..4]); if magic != MAGIC { return Err(HeaderError::InvalidMagicNumber(magic).into()); } @@ -176,8 +178,8 @@ impl FileHeader { if format != FORMAT { return Err(HeaderError::InvalidFormatVersion(format).into()); } - let slen = LittleEndian::read_u32(&buffer[5..9]); - let xlen = LittleEndian::read_u32(&buffer[9..13]); + let slen = read_u32_le(&buffer[5..9]); + let xlen = read_u32_le(&buffer[9..13]); let bits = match buffer[13] { 0 | 2 | 42 => BitSize::Two, 4 => BitSize::Four, @@ -246,10 +248,10 @@ impl FileHeader { /// Returns an error if writing to the writer fails (typically an I/O error). pub fn write_bytes(&self, writer: &mut W) -> Result<()> { let mut buffer = [0u8; SIZE_HEADER]; - LittleEndian::write_u32(&mut buffer[0..4], self.magic); + buffer[0..4].copy_from_slice(&self.magic.to_le_bytes()); buffer[4] = self.format; - LittleEndian::write_u32(&mut buffer[5..9], self.slen); - LittleEndian::write_u32(&mut buffer[9..13], self.xlen); + buffer[5..9].copy_from_slice(&self.slen.to_le_bytes()); + buffer[9..13].copy_from_slice(&self.xlen.to_le_bytes()); buffer[13] = self.bits.into(); buffer[14] = self.flags.into(); buffer[15..32].copy_from_slice(&self.reserved); @@ -327,7 +329,11 @@ mod tests { #[test] fn test_from_bytes_four_bit_size() { - let header = FileHeaderBuilder::new().bitsize(BitSize::Four).slen(32).build().unwrap(); + let header = FileHeaderBuilder::new() + .bitsize(BitSize::Four) + .slen(32) + .build() + .unwrap(); let mut buffer = [0u8; SIZE_HEADER]; let mut cursor = std::io::Cursor::new(&mut buffer[..]); header.write_bytes(&mut cursor).unwrap(); @@ -376,7 +382,12 @@ mod tests { #[test] fn test_from_reader_valid() { - let header = FileHeaderBuilder::new().slen(32).xlen(16).flags(true).build().unwrap(); + let header = FileHeaderBuilder::new() + .slen(32) + .xlen(16) + .flags(true) + .build() + .unwrap(); let mut buffer = Vec::new(); header.write_bytes(&mut buffer).unwrap(); let mut cursor = std::io::Cursor::new(buffer); diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 031c492..bdd0233 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -5,3 +5,13 @@ pub mod fastx; #[cfg(feature = "paraseq")] pub use fastx::FastxEncoderBuilder; + +/// Read a little-endian u64 from the start of a byte slice +pub(crate) fn read_u64_le(b: &[u8]) -> u64 { + u64::from_le_bytes(b[..8].try_into().unwrap()) +} + +/// Read a little-endian u32 from the start of a byte slice +pub(crate) fn read_u32_le(b: &[u8]) -> u32 { + u32::from_le_bytes(b[..4].try_into().unwrap()) +} diff --git a/src/vbq/header.rs b/src/vbq/header.rs index e1e3a4c..bec355a 100644 --- a/src/vbq/header.rs +++ b/src/vbq/header.rs @@ -15,9 +15,11 @@ use std::io::{Read, Write}; use bitnuc_deprec::BitSize; -use byteorder::{ByteOrder, LittleEndian}; -use crate::error::{HeaderError, ReadError, Result}; +use crate::{ + error::{HeaderError, ReadError, Result}, + utils::{read_u32_le, read_u64_le}, +}; /// Magic number for file identification: "VSEQ" in ASCII (0x51455356) /// @@ -249,7 +251,7 @@ impl FileHeader { /// * `HeaderError::InvalidFormatVersion` - If the format version is unsupported /// * `HeaderError::InvalidReservedBytes` - If the reserved bytes section is invalid pub fn from_bytes(buffer: &[u8; SIZE_HEADER]) -> Result { - let magic = LittleEndian::read_u32(&buffer[0..4]); + let magic = read_u32_le(&buffer[0..4]); if magic != MAGIC { return Err(HeaderError::InvalidMagicNumber(magic).into()); } @@ -257,7 +259,7 @@ impl FileHeader { if format != FORMAT { return Err(HeaderError::InvalidFormatVersion(format).into()); } - let block = LittleEndian::read_u64(&buffer[5..13]); + let block = read_u64_le(&buffer[5..13]); let qual = buffer[13] != 0; let compressed = buffer[14] != 0; let paired = buffer[15] != 0; @@ -306,9 +308,9 @@ impl FileHeader { /// * IO errors if writing to the writer fails pub fn write_bytes(&self, writer: &mut W) -> Result<()> { let mut buffer = [0u8; SIZE_HEADER]; - LittleEndian::write_u32(&mut buffer[0..4], self.magic); + buffer[0..4].copy_from_slice(&self.magic.to_le_bytes()); buffer[4] = self.format; - LittleEndian::write_u64(&mut buffer[5..13], self.block); + buffer[5..13].copy_from_slice(&self.block.to_le_bytes()); buffer[13] = self.qual.into(); buffer[14] = self.compressed.into(); buffer[15] = self.paired.into(); @@ -442,9 +444,9 @@ impl BlockHeader { /// * IO errors if writing to the writer fails pub fn write_bytes(&self, writer: &mut W) -> Result<()> { let mut buffer = [0u8; SIZE_BLOCK_HEADER]; - LittleEndian::write_u64(&mut buffer[0..8], self.magic); - LittleEndian::write_u64(&mut buffer[8..16], self.size); - LittleEndian::write_u32(&mut buffer[16..20], self.records); + buffer[0..8].copy_from_slice(&self.magic.to_le_bytes()); + buffer[8..16].copy_from_slice(&self.size.to_le_bytes()); + buffer[16..20].copy_from_slice(&self.records.to_le_bytes()); buffer[20..].copy_from_slice(&self.reserved); writer.write_all(&buffer)?; Ok(()) @@ -467,12 +469,12 @@ impl BlockHeader { /// /// * `ReadError::InvalidBlockMagicNumber` - If the magic number doesn't match "BLOCKSEQ" pub fn from_bytes(buffer: &[u8; SIZE_BLOCK_HEADER]) -> Result { - let magic = LittleEndian::read_u64(&buffer[0..8]); + let magic = read_u64_le(&buffer[0..8]); if magic != BLOCK_MAGIC { return Err(ReadError::InvalidBlockMagicNumber(magic, 0).into()); } - let size = LittleEndian::read_u64(&buffer[8..16]); - let records = LittleEndian::read_u32(&buffer[16..20]); + let size = read_u64_le(&buffer[8..16]); + let records = read_u32_le(&buffer[16..20]); Ok(Self::new(size, records)) } diff --git a/src/vbq/index.rs b/src/vbq/index.rs index 7f7e0fa..2b90105 100644 --- a/src/vbq/index.rs +++ b/src/vbq/index.rs @@ -34,9 +34,10 @@ use std::io::{Cursor, Read, Write}; -use byteorder::{ByteOrder, LittleEndian}; use zstd::{Decoder, Encoder}; +use crate::utils::{read_u32_le, read_u64_le}; + use crate::error::{IndexError, Result}; /// Size of `BlockRange` in bytes @@ -174,10 +175,10 @@ impl BlockRange { /// * `Err(_)` - If an error occurred during writing pub fn write_bytes(&self, writer: &mut W) -> Result<()> { let mut buf = [0; SIZE_BLOCK_RANGE]; - LittleEndian::write_u64(&mut buf[0..8], self.start_offset); - LittleEndian::write_u64(&mut buf[8..16], self.len); - LittleEndian::write_u32(&mut buf[16..20], self.block_records); - LittleEndian::write_u64(&mut buf[20..28], self.cumulative_records); + buf[0..8].copy_from_slice(&self.start_offset.to_le_bytes()); + buf[8..16].copy_from_slice(&self.len.to_le_bytes()); + buf[16..20].copy_from_slice(&self.block_records.to_le_bytes()); + buf[20..28].copy_from_slice(&self.cumulative_records.to_le_bytes()); buf[28..].copy_from_slice(&self.reservation); writer.write_all(&buf)?; Ok(()) @@ -207,10 +208,10 @@ impl BlockRange { #[must_use] pub fn from_exact(buffer: &[u8; SIZE_BLOCK_RANGE]) -> Self { Self { - start_offset: LittleEndian::read_u64(&buffer[0..8]), - len: LittleEndian::read_u64(&buffer[8..16]), - block_records: LittleEndian::read_u32(&buffer[16..20]), - cumulative_records: LittleEndian::read_u64(&buffer[20..28]), + start_offset: read_u64_le(&buffer[0..8]), + len: read_u64_le(&buffer[8..16]), + block_records: read_u32_le(&buffer[16..20]), + cumulative_records: read_u64_le(&buffer[20..28]), reservation: INDEX_RESERVATION, } } @@ -307,8 +308,8 @@ impl IndexHeader { pub fn from_reader(reader: &mut R) -> Result { let mut buffer = [0; INDEX_HEADER_SIZE]; reader.read_exact(&mut buffer)?; - let magic = LittleEndian::read_u64(&buffer[0..8]); - let bytes = LittleEndian::read_u64(&buffer[8..16]); + let magic = read_u64_le(&buffer[0..8]); + let bytes = read_u64_le(&buffer[8..16]); let Ok(reserved) = buffer[16..INDEX_HEADER_SIZE].try_into() else { return Err(IndexError::InvalidReservedBytes.into()); }; @@ -350,8 +351,8 @@ impl IndexHeader { /// - Bytes 16-31: reserved for future extensions pub fn write_bytes(&self, writer: &mut W) -> Result<()> { let mut buffer = [0; INDEX_HEADER_SIZE]; - LittleEndian::write_u64(&mut buffer[0..8], self.magic); - LittleEndian::write_u64(&mut buffer[8..16], self.bytes); + buffer[0..8].copy_from_slice(&self.magic.to_le_bytes()); + buffer[8..16].copy_from_slice(&self.bytes.to_le_bytes()); buffer[16..].copy_from_slice(&self.reserved); writer.write_all(&buffer)?; Ok(()) @@ -555,7 +556,7 @@ mod tests { #[test] fn test_index_header_from_reader_invalid_magic() { let mut buffer = [0u8; INDEX_HEADER_SIZE]; - LittleEndian::write_u64(&mut buffer[0..8], 0xDEAD_BEEF); + buffer[0..8].copy_from_slice(&0xDEAD_BEEF_u64.to_le_bytes()); let result = IndexHeader::from_reader(&mut Cursor::new(buffer)); assert!(result.is_err()); } diff --git a/src/vbq/reader.rs b/src/vbq/reader.rs index ef5b931..fd92e85 100644 --- a/src/vbq/reader.rs +++ b/src/vbq/reader.rs @@ -55,8 +55,9 @@ use std::path::Path; use std::sync::Arc; use bitnuc_deprec::BitSize; -use byteorder::{ByteOrder, LittleEndian}; use memmap2::Mmap; + +use crate::utils::read_u64_le; use zstd::zstd_safe; use super::{ @@ -370,7 +371,7 @@ impl RecordBlock { // Read flag let flag = if has_flags { - let flag = LittleEndian::read_u64(&bytes[pos..pos + 8]); + let flag = read_u64_le(&bytes[pos..pos + 8]); pos += 8; Some(flag) } else { @@ -378,9 +379,9 @@ impl RecordBlock { }; // Read lengths - let slen = LittleEndian::read_u64(&bytes[pos..pos + 8]); + let slen = read_u64_le(&bytes[pos..pos + 8]); pos += 8; - let xlen = LittleEndian::read_u64(&bytes[pos..pos + 8]); + let xlen = read_u64_le(&bytes[pos..pos + 8]); pos += 8; // Check for end of records @@ -395,7 +396,7 @@ impl RecordBlock { // Primary sequence - store span into sequences Vec let s_seq_span = Span::new(self.sequences.len(), s_seq_words); for _ in 0..s_seq_words { - let val = LittleEndian::read_u64(&bytes[pos..pos + 8]); + let val = read_u64_le(&bytes[pos..pos + 8]); self.sequences.push(val); pos += 8; } @@ -411,7 +412,7 @@ impl RecordBlock { // Primary header - store span into rbuf let s_header_span = if has_header { - let header_len = LittleEndian::read_u64(&bytes[pos..pos + 8]) as usize; + let header_len = read_u64_le(&bytes[pos..pos + 8]) as usize; pos += 8; let span = Span::new(pos, header_len); pos += header_len; @@ -423,7 +424,7 @@ impl RecordBlock { // Extended sequence - store span into sequences Vec let x_seq_span = Span::new(self.sequences.len(), x_seq_words); for _ in 0..x_seq_words { - let val = LittleEndian::read_u64(&bytes[pos..pos + 8]); + let val = read_u64_le(&bytes[pos..pos + 8]); self.sequences.push(val); pos += 8; } @@ -439,7 +440,7 @@ impl RecordBlock { // Extended header - store span into rbuf let x_header_span = if has_header && xlen > 0 { - let header_len = LittleEndian::read_u64(&bytes[pos..pos + 8]) as usize; + let header_len = read_u64_le(&bytes[pos..pos + 8]) as usize; pos += 8; let span = Span::new(pos, header_len); pos += header_len; @@ -1077,13 +1078,13 @@ impl MmapReader { let start_pos_index_size = start_pos_magic - 8; // Validate the magic number - let magic = LittleEndian::read_u64(&self.mmap[start_pos_magic..]); + let magic = read_u64_le(&self.mmap[start_pos_magic..]); if magic != INDEX_END_MAGIC { return Err(ReadError::MissingIndexEndMagic.into()); } // Get the index size - let index_size = LittleEndian::read_u64(&self.mmap[start_pos_index_size..start_pos_magic]); + let index_size = read_u64_le(&self.mmap[start_pos_index_size..start_pos_magic]); // Determine the start position of the index bytes let start_pos_index = start_pos_index_size - index_size as usize; diff --git a/src/vbq/writer.rs b/src/vbq/writer.rs index 3ebb9bd..68dc116 100644 --- a/src/vbq/writer.rs +++ b/src/vbq/writer.rs @@ -65,7 +65,6 @@ use std::io::Write; use bitnuc_deprec::BitSize; -use byteorder::{LittleEndian, WriteBytesExt}; use rand::SeedableRng; use rand::rngs::SmallRng; use zstd::stream::copy_encode; @@ -747,10 +746,10 @@ impl Writer { self.inner.write_all(&buffer)?; // Write the number of bytes written to the index - self.inner.write_u64::(n_bytes)?; + self.inner.write_all(&n_bytes.to_le_bytes())?; // Write the index footer magic - self.inner.write_u64::(INDEX_END_MAGIC)?; + self.inner.write_all(&INDEX_END_MAGIC.to_le_bytes())?; Ok(()) } @@ -902,20 +901,20 @@ impl BlockWriter { } fn write_flag(&mut self, flag: u64) -> Result<()> { - self.ubuf.write_u64::(flag)?; + self.ubuf.write_all(&flag.to_le_bytes())?; self.pos += 8; Ok(()) } fn write_length(&mut self, length: u64) -> Result<()> { - self.ubuf.write_u64::(length)?; + self.ubuf.write_all(&length.to_le_bytes())?; self.pos += 8; Ok(()) } fn write_buffer(&mut self, ebuf: &[u64]) -> Result<()> { ebuf.iter() - .try_for_each(|&x| self.ubuf.write_u64::(x))?; + .try_for_each(|&x| self.ubuf.write_all(&x.to_le_bytes()))?; self.pos += 8 * ebuf.len(); Ok(()) } @@ -1170,6 +1169,7 @@ impl Encoder { #[cfg(test)] mod tests { + use crate::utils::read_u64_le; use super::*; use crate::SequencingRecordBuilder; use crate::vbq::{FileHeaderBuilder, header::SIZE_HEADER}; @@ -1483,7 +1483,6 @@ mod tests { #[test] fn test_index_always_written_on_finish() -> super::Result<()> { use crate::vbq::index::INDEX_END_MAGIC; - use byteorder::{ByteOrder, LittleEndian}; // Create a writer with some records let header = FileHeaderBuilder::new().build(); @@ -1507,7 +1506,7 @@ mod tests { // Verify the file ends with the index magic number assert!(bytes.len() >= 8, "File is too short to contain index"); let magic_offset = bytes.len() - 8; - let magic = LittleEndian::read_u64(&bytes[magic_offset..]); + let magic = read_u64_le(&bytes[magic_offset..]); assert_eq!( magic, INDEX_END_MAGIC, "Index magic number not found at end of file" @@ -1516,7 +1515,7 @@ mod tests { // Verify we can read the index size assert!(bytes.len() >= 16, "File is too short to contain index size"); let size_offset = bytes.len() - 16; - let index_size = LittleEndian::read_u64(&bytes[size_offset..size_offset + 8]); + let index_size = read_u64_le(&bytes[size_offset..size_offset + 8]); assert!(index_size > 0, "Index size should be greater than 0"); // Verify the index size makes sense (should be less than total file size) @@ -1531,7 +1530,6 @@ mod tests { #[test] fn test_finish_idempotent() -> super::Result<()> { use crate::vbq::index::INDEX_END_MAGIC; - use byteorder::{ByteOrder, LittleEndian}; // Create a writer let header = FileHeaderBuilder::new().build(); @@ -1563,7 +1561,7 @@ mod tests { // Verify only one index magic number at the end let bytes = &writer.inner; let magic_offset = bytes.len() - 8; - let magic = LittleEndian::read_u64(&bytes[magic_offset..]); + let magic = read_u64_le(&bytes[magic_offset..]); assert_eq!(magic, INDEX_END_MAGIC); Ok(()) From ad775a21222c5a50aa7c8833338bff8fdc3b0bbf Mon Sep 17 00:00:00 2001 From: noamteyssier <22600644+noamteyssier@users.noreply.github.com> Date: Tue, 25 Aug 2026 18:51:13 -0700 Subject: [PATCH 07/25] dep: use std available_parallelism instead of num_cpus --- Cargo.toml | 1 - src/bq/reader.rs | 6 +----- src/cbq/read.rs | 6 +----- src/parallel.rs | 12 ++++++++++++ src/vbq/reader.rs | 6 +----- 5 files changed, 15 insertions(+), 16 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index f22c897..00fbc14 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,6 @@ bytemuck = { version = "1.25.1", features = ["derive", "extern_crate_alloc"] } itoa = "1.0.18" memchr = "2.8.3" memmap2 = "0.9.11" -num_cpus = "1.17.0" paraseq = { version = "0.4.14", optional = true } parking_lot = {version = "0.12.5", optional = true } rand = { version = "0.9.5", features = ["small_rng"] } diff --git a/src/bq/reader.rs b/src/bq/reader.rs index 012498d..2f0e6bd 100644 --- a/src/bq/reader.rs +++ b/src/bq/reader.rs @@ -868,11 +868,7 @@ impl ParallelReader for MmapReader { range: Range, ) -> Result<()> { // Calculate the number of threads to use - let num_threads = if num_threads == 0 { - num_cpus::get() - } else { - num_threads.min(num_cpus::get()) - }; + let num_threads = crate::parallel::clamp_threads(num_threads); // Validate range let num_records = self.num_records(); diff --git a/src/cbq/read.rs b/src/cbq/read.rs index ec3f0b4..763bcc8 100644 --- a/src/cbq/read.rs +++ b/src/cbq/read.rs @@ -231,11 +231,7 @@ impl ParallelReader for MmapReader { num_threads: usize, range: std::ops::Range, ) -> crate::Result<()> { - let num_threads = if num_threads == 0 { - num_cpus::get() - } else { - num_threads.min(num_cpus::get()) - }; + let num_threads = crate::parallel::clamp_threads(num_threads); // validate range let total_records = self.num_records(); diff --git a/src/parallel.rs b/src/parallel.rs index b969da5..eaf3a88 100644 --- a/src/parallel.rs +++ b/src/parallel.rs @@ -123,6 +123,18 @@ impl ParallelReader for BinseqReader { } } +/// Clamp a requested thread count to the available parallelism. +/// +/// A request of 0 means "use all available cores". +pub(crate) fn clamp_threads(num_threads: usize) -> usize { + let available = std::thread::available_parallelism().map_or(1, std::num::NonZero::get); + if num_threads == 0 { + available + } else { + num_threads.min(available) + } +} + /// Trait for BINSEQ readers that can process records in parallel /// /// This is implemented by the **reader** not by the **processor**. diff --git a/src/vbq/reader.rs b/src/vbq/reader.rs index fd92e85..80fc8f5 100644 --- a/src/vbq/reader.rs +++ b/src/vbq/reader.rs @@ -1237,11 +1237,7 @@ impl ParallelReader for MmapReader { range: Range, ) -> Result<()> { // Calculate the number of threads to use - let num_threads = if num_threads == 0 { - num_cpus::get() - } else { - num_threads.min(num_cpus::get()) - }; + let num_threads = crate::parallel::clamp_threads(num_threads); // Generate or load the index first let index = self.load_index()?; From 5bcd1b80536f0051aa165fc441d9b6b76c0bcea6 Mon Sep 17 00:00:00 2001 From: noamteyssier <22600644+noamteyssier@users.noreply.github.com> Date: Tue, 25 Aug 2026 18:55:19 -0700 Subject: [PATCH 08/25] dep: drop auto_impl; move memchr to dev-dependencies auto_impl was a proc-macro dep serving one attribute. Generated &R / &mut R impls have no callers here or downstream. memchr is only used by examples/grep.rs, so belongs in dev-dep. BREAKING: &R and &mut R no longer implement BinseqRecord. --- Cargo.toml | 3 +-- src/record/binseq_record.rs | 2 -- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 00fbc14..9b39d01 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,12 +12,10 @@ keywords = ["binary", "nucleotide", "sequencing", "genomics", "fastq"] [dependencies] anyhow = {version = "1.0.103", optional = true} -auto_impl = "1.3.0" bitnuc-deprec = { package ="bitnuc", version = "0.4.1" } bitnuc = { version = "0.5.1" } bytemuck = { version = "1.25.1", features = ["derive", "extern_crate_alloc"] } itoa = "1.0.18" -memchr = "2.8.3" memmap2 = "0.9.11" paraseq = { version = "0.4.14", optional = true } parking_lot = {version = "0.12.5", optional = true } @@ -27,6 +25,7 @@ thiserror = "2.0.20" zstd = { version = "0.13.3", features = ["zstdmt"] } [dev-dependencies] +memchr = "2.8.3" anyhow = "1.0.103" parking_lot = "0.12.5" clap = { version = "4.6.2", features = ["derive"] } diff --git a/src/record/binseq_record.rs b/src/record/binseq_record.rs index 7641799..9ed9ef6 100644 --- a/src/record/binseq_record.rs +++ b/src/record/binseq_record.rs @@ -1,4 +1,3 @@ -use auto_impl::auto_impl; use bitnuc_deprec::BitSize; use crate::Result; @@ -11,7 +10,6 @@ use crate::Result; /// Implemented by [`bq::RefRecord`](crate::bq::RefRecord) and [`vbq::RefRecord`](crate::vbq::RefRecord). /// /// Used to interact with [`ParallelProcessor`](crate::ParallelProcessor) for easy parallel processing. -#[auto_impl(&, &mut)] pub trait BinseqRecord { /// Returns the bitsize of the record (number of bits per nucleotide) fn bitsize(&self) -> BitSize; From 8e4c8ef76d3a52d4aa1ae84db23fcb21b3d5914c Mon Sep 17 00:00:00 2001 From: noamteyssier <22600644+noamteyssier@users.noreply.github.com> Date: Tue, 25 Aug 2026 19:04:08 -0700 Subject: [PATCH 09/25] refactor!: share one nucleotide Encoder between bq and vbq writers The two Encoders were identical except that bq's embedded a FileHeader for fixed-length validation. One crate-private Encoder (parameterized only by BitSize) replaces both. bq::Writer now stores its header directly and validates sequence lengths in push() before encoding. BREAKING: bq::Encoder is no longer part of the public API. --- src/bq/mod.rs | 2 +- src/bq/writer.rs | 291 ++++++++-------------------------------------- src/encoder.rs | 145 +++++++++++++++++++++++ src/lib.rs | 3 + src/vbq/writer.rs | 101 +--------------- 5 files changed, 203 insertions(+), 339 deletions(-) create mode 100644 src/encoder.rs diff --git a/src/bq/mod.rs b/src/bq/mod.rs index 08bc097..3811e48 100644 --- a/src/bq/mod.rs +++ b/src/bq/mod.rs @@ -242,4 +242,4 @@ mod writer; pub use header::{FILE_MAGIC, FileHeader, FileHeaderBuilder, SIZE_HEADER}; pub use reader::{MmapReader, RefRecord, StreamReader}; -pub use writer::{Encoder, Writer, WriterBuilder}; +pub use writer::{Writer, WriterBuilder}; diff --git a/src/bq/writer.rs b/src/bq/writer.rs index cc1b7e0..641073f 100644 --- a/src/bq/writer.rs +++ b/src/bq/writer.rs @@ -9,11 +9,10 @@ use std::io::Write; -use rand::{SeedableRng, rngs::SmallRng}; - use super::FileHeader; use crate::{ - Policy, RNG_SEED, SequencingRecord, + Policy, SequencingRecord, + encoder::Encoder, error::{Result, WriteError}, }; @@ -24,177 +23,6 @@ fn write_buffer(writer: &mut W, ebuf: &[u64]) -> Result<()> { Ok(()) } -/// Encodes nucleotide sequences into a compact 2-bit binary format -/// -/// The `Encoder` handles the conversion of nucleotide sequences (A, C, G, T) -/// into a compact binary representation where each nucleotide is stored using -/// 2 bits. It also handles invalid nucleotides according to a configurable policy. -/// -/// The encoder maintains internal buffers to avoid repeated allocations during -/// encoding operations. These buffers are reused across multiple encode calls -/// and are cleared automatically when needed. -#[derive(Clone)] -pub struct Encoder { - /// Header containing sequence length and format information - header: FileHeader, - - /// Buffers for storing encoded nucleotides in 2-bit format - /// Each u64 can store 32 nucleotides (64 bits / 2 bits per nucleotide) - sbuffer: Vec, // Primary sequence buffer - xbuffer: Vec, // Extended sequence buffer - - /// Temporary buffers for handling invalid nucleotides - /// These store the processed sequences after policy application - s_ibuf: Vec, // Primary sequence invalid buffer - x_ibuf: Vec, // Extended sequence invalid buffer - - /// Policy for handling invalid nucleotides during encoding - policy: Policy, - - /// Random number generator for the `RandomDraw` policy - /// Seeded with `RNG_SEED` for reproducibility - rng: SmallRng, -} -impl Encoder { - /// Creates a new encoder with default invalid nucleotide policy - /// - /// # Arguments - /// - /// * `header` - The header defining sequence lengths and format - /// - /// # Examples - /// - /// ``` - /// # use binseq::bq::{FileHeaderBuilder, Encoder}; - /// let header = FileHeaderBuilder::new().slen(100).build().unwrap(); - /// let encoder = Encoder::new(header); - /// ``` - #[must_use] - pub fn new(header: FileHeader) -> Self { - Self::with_policy(header, Policy::default()) - } - - /// Creates a new encoder with a specific invalid nucleotide policy - /// - /// # Arguments - /// - /// * `header` - The header defining sequence lengths and format - /// * `policy` - The policy for handling invalid nucleotides - /// - /// # Examples - /// - /// ``` - /// # use binseq::bq::{FileHeaderBuilder, Encoder}; - /// # use binseq::Policy; - /// let header = FileHeaderBuilder::new().slen(100).build().unwrap(); - /// let encoder = Encoder::with_policy(header, Policy::SetToA); - /// ``` - #[must_use] - pub fn with_policy(header: FileHeader, policy: Policy) -> Self { - Self { - header, - policy, - sbuffer: Vec::default(), - xbuffer: Vec::default(), - s_ibuf: Vec::default(), - x_ibuf: Vec::default(), - rng: SmallRng::seed_from_u64(RNG_SEED), - } - } - - /// Returns whether the header is paired-end. - #[must_use] - pub fn is_paired(&self) -> bool { - self.header.is_paired() - } - - /// Encodes a single sequence as 2-bit. - /// - /// Will return `None` if the sequence is invalid and the policy does not allow correction. - pub fn encode_single(&mut self, primary: &[u8]) -> Result> { - if primary.len() != self.header.slen as usize { - return Err(WriteError::UnexpectedSequenceLength { - expected: self.header.slen, - got: primary.len(), - } - .into()); - } - - // Fill the buffer with the 2-bit representation of the nucleotides - self.clear(); - if self.header.bits.encode(primary, &mut self.sbuffer).is_err() { - self.clear(); - if self - .policy - .handle(primary, &mut self.s_ibuf, &mut self.rng)? - { - self.header.bits.encode(&self.s_ibuf, &mut self.sbuffer)?; - } else { - return Ok(None); - } - } - - Ok(Some(&self.sbuffer)) - } - - /// Encodes a pair of sequences as 2-bit. - /// - /// Will return `None` if either sequence is invalid and the policy does not allow correction. - pub fn encode_paired( - &mut self, - primary: &[u8], - extended: &[u8], - ) -> Result> { - if primary.len() != self.header.slen as usize { - return Err(WriteError::UnexpectedSequenceLength { - expected: self.header.slen, - got: primary.len(), - } - .into()); - } - if extended.len() != self.header.xlen as usize { - return Err(WriteError::UnexpectedSequenceLength { - expected: self.header.xlen, - got: extended.len(), - } - .into()); - } - - self.clear(); - if self.header.bits.encode(primary, &mut self.sbuffer).is_err() - || self - .header - .bits - .encode(extended, &mut self.xbuffer) - .is_err() - { - self.clear(); - if self - .policy - .handle(primary, &mut self.s_ibuf, &mut self.rng)? - && self - .policy - .handle(extended, &mut self.x_ibuf, &mut self.rng)? - { - self.header.bits.encode(&self.s_ibuf, &mut self.sbuffer)?; - self.header.bits.encode(&self.x_ibuf, &mut self.xbuffer)?; - } else { - return Ok(None); - } - } - - Ok(Some((&self.sbuffer, &self.xbuffer))) - } - - /// Clear all buffers and reset the encoder. - pub fn clear(&mut self) { - self.sbuffer.clear(); - self.xbuffer.clear(); - self.s_ibuf.clear(); - self.x_ibuf.clear(); - } -} - /// Builder for creating configured `Writer` instances /// /// This builder provides a flexible way to create writers with various @@ -275,6 +103,9 @@ pub struct Writer { /// The underlying writer for output inner: W, + /// Header defining sequence lengths and format + header: FileHeader, + /// Encoder for converting sequences to binary format encoder: Encoder, @@ -322,24 +153,25 @@ impl Writer { } Ok(Self { inner, - encoder: Encoder::with_policy(header, policy), + header, + encoder: Encoder::with_policy(header.bits, policy), headless, }) } /// Returns whether the header is paired-end. pub fn is_paired(&self) -> bool { - self.encoder.is_paired() + self.header.is_paired() } /// Returns the header of the writer pub fn header(&self) -> FileHeader { - self.encoder.header + self.header } /// Returns the N-policy of the writer pub fn policy(&self) -> Policy { - self.encoder.policy + self.encoder.policy() } /// Writes a record using the unified [`SequencingRecord`] API @@ -379,7 +211,7 @@ impl Writer { /// # } /// ``` pub fn push(&mut self, record: SequencingRecord) -> Result { - let has_flag = self.encoder.header.flags; + let has_flag = self.header.flags; if has_flag { self.inner .write_all(&record.flag.unwrap_or(0).to_le_bytes())?; @@ -387,16 +219,35 @@ impl Writer { // Check paired status - writer can require paired (record must have R2), // but if writer is single-end, we simply ignore any R2 data in the record. - if self.encoder.header.is_paired() && !record.is_paired() { + if self.header.is_paired() && !record.is_paired() { return Err(WriteError::ConfigurationMismatch { attribute: "paired", - expected: self.encoder.header.is_paired(), + expected: self.header.is_paired(), actual: record.is_paired(), } .into()); } - if self.encoder.header.is_paired() { + // BQ records are fixed-length: validate against the header + if record.s_seq.len() != self.header.slen as usize { + return Err(WriteError::UnexpectedSequenceLength { + expected: self.header.slen, + got: record.s_seq.len(), + } + .into()); + } + if self.header.is_paired() { + let xlen = record.x_seq.unwrap_or_default().len(); + if xlen != self.header.xlen as usize { + return Err(WriteError::UnexpectedSequenceLength { + expected: self.header.xlen, + got: xlen, + } + .into()); + } + } + + if self.header.is_paired() { if let Some((sbuffer, xbuffer)) = self .encoder .encode_paired(record.s_seq, record.x_seq.unwrap_or_default())? @@ -554,69 +405,29 @@ mod testing { Ok(()) } - // ==================== Encoder Tests ==================== + // ==================== Length Validation Tests ==================== #[test] - fn test_encoder_new() { - let header = FileHeaderBuilder::new().slen(8).build().unwrap(); - let encoder = Encoder::new(header); - assert!(matches!(encoder.policy, Policy::IgnoreSequence)); - } - - #[test] - fn test_encoder_encode_single_wrong_length() { - let header = FileHeaderBuilder::new().slen(8).build().unwrap(); - let mut encoder = Encoder::new(header); - let result = encoder.encode_single(b"ACGT"); - assert!(result.is_err()); - } - - #[test] - fn test_encoder_encode_single_invalid_ignored() { - let header = FileHeaderBuilder::new().slen(8).build().unwrap(); - let mut encoder = Encoder::with_policy(header, Policy::IgnoreSequence); - let result = encoder.encode_single(b"ACGTNNNN").unwrap(); - assert!(result.is_none()); - } - - #[test] - fn test_encoder_encode_single_invalid_corrected() { - let header = FileHeaderBuilder::new().slen(8).build().unwrap(); - let mut encoder = Encoder::with_policy(header, Policy::SetToA); - let result = encoder.encode_single(b"ACGTNNNN").unwrap(); - assert!(result.is_some()); - } - - #[test] - fn test_encoder_encode_paired_wrong_primary_length() { - let header = FileHeaderBuilder::new().slen(8).xlen(8).build().unwrap(); - let mut encoder = Encoder::new(header); - let result = encoder.encode_paired(b"ACGT", b"ACGTACGT"); - assert!(result.is_err()); - } - - #[test] - fn test_encoder_encode_paired_wrong_extended_length() { - let header = FileHeaderBuilder::new().slen(8).xlen(8).build().unwrap(); - let mut encoder = Encoder::new(header); - let result = encoder.encode_paired(b"ACGTACGT", b"ACGT"); - assert!(result.is_err()); - } - - #[test] - fn test_encoder_encode_paired_invalid_ignored() { - let header = FileHeaderBuilder::new().slen(8).xlen(8).build().unwrap(); - let mut encoder = Encoder::with_policy(header, Policy::IgnoreSequence); - let result = encoder.encode_paired(b"ACGTNNNN", b"ACGTACGT").unwrap(); - assert!(result.is_none()); + fn test_push_wrong_length() -> Result<()> { + let mut writer = WriterBuilder::default() + .header(FileHeaderBuilder::new().slen(8).build()?) + .build(Vec::new())?; + let record = SequencingRecordBuilder::default().s_seq(b"ACGT").build()?; + assert!(writer.push(record).is_err()); + Ok(()) } #[test] - fn test_encoder_encode_paired_invalid_corrected() { - let header = FileHeaderBuilder::new().slen(8).xlen(8).build().unwrap(); - let mut encoder = Encoder::with_policy(header, Policy::SetToA); - let result = encoder.encode_paired(b"ACGTNNNN", b"NNNNACGT").unwrap(); - assert!(result.is_some()); + fn test_push_wrong_extended_length() -> Result<()> { + let mut writer = WriterBuilder::default() + .header(FileHeaderBuilder::new().slen(8).xlen(8).build()?) + .build(Vec::new())?; + let record = SequencingRecordBuilder::default() + .s_seq(b"ACGTACGT") + .x_seq(b"ACGT") + .build()?; + assert!(writer.push(record).is_err()); + Ok(()) } // ==================== WriterBuilder Tests ==================== diff --git a/src/encoder.rs b/src/encoder.rs new file mode 100644 index 0000000..23ff7da --- /dev/null +++ b/src/encoder.rs @@ -0,0 +1,145 @@ +//! Shared nucleotide encoder used by the BQ and VBQ writers. + +use bitnuc_deprec::BitSize; +use rand::{SeedableRng, rngs::SmallRng}; + +use crate::{Policy, RNG_SEED, error::Result}; + +/// Encodes nucleotide sequences into a compact multi-bit binary format. +/// +/// The encoder maintains internal buffers to avoid repeated allocations and +/// handles invalid nucleotides according to a configurable [`Policy`]. +#[derive(Clone)] +pub(crate) struct Encoder { + /// Bitsize of the nucleotides + bitsize: BitSize, + + /// Reusable buffers for encoded nucleotides + sbuffer: Vec, + xbuffer: Vec, + + /// Reusable buffers for invalid nucleotide sequences + s_ibuf: Vec, + x_ibuf: Vec, + + /// Invalid Nucleotide Policy + policy: Policy, + + /// Random Number Generator (seeded with `RNG_SEED` for reproducibility) + rng: SmallRng, +} + +impl Encoder { + /// Initialize a new encoder with the given policy. + pub fn with_policy(bitsize: BitSize, policy: Policy) -> Self { + Self { + bitsize, + policy, + sbuffer: Vec::default(), + xbuffer: Vec::default(), + s_ibuf: Vec::default(), + x_ibuf: Vec::default(), + rng: SmallRng::seed_from_u64(RNG_SEED), + } + } + + /// Returns the invalid-nucleotide policy of this encoder. + pub fn policy(&self) -> Policy { + self.policy + } + + /// Encodes a single sequence. + /// + /// Will return `None` if the sequence is invalid and the policy does not allow correction. + pub fn encode_single(&mut self, primary: &[u8]) -> Result> { + self.clear(); + if self.bitsize.encode(primary, &mut self.sbuffer).is_err() { + self.clear(); + if self + .policy + .handle(primary, &mut self.s_ibuf, &mut self.rng)? + { + self.bitsize.encode(&self.s_ibuf, &mut self.sbuffer)?; + } else { + return Ok(None); + } + } + Ok(Some(&self.sbuffer)) + } + + /// Encodes a pair of sequences. + /// + /// Will return `None` if either sequence is invalid and the policy does not allow correction. + pub fn encode_paired( + &mut self, + primary: &[u8], + extended: &[u8], + ) -> Result> { + self.clear(); + if self.bitsize.encode(primary, &mut self.sbuffer).is_err() + || self.bitsize.encode(extended, &mut self.xbuffer).is_err() + { + self.clear(); + if self + .policy + .handle(primary, &mut self.s_ibuf, &mut self.rng)? + && self + .policy + .handle(extended, &mut self.x_ibuf, &mut self.rng)? + { + self.bitsize.encode(&self.s_ibuf, &mut self.sbuffer)?; + self.bitsize.encode(&self.x_ibuf, &mut self.xbuffer)?; + } else { + return Ok(None); + } + } + Ok(Some((&self.sbuffer, &self.xbuffer))) + } + + /// Clear all buffers and reset the encoder. + pub fn clear(&mut self) { + self.sbuffer.clear(); + self.xbuffer.clear(); + self.s_ibuf.clear(); + self.x_ibuf.clear(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_encode_single_invalid_ignored() { + let mut encoder = Encoder::with_policy(BitSize::Two, Policy::IgnoreSequence); + assert!(encoder.encode_single(b"ACGTNNNN").unwrap().is_none()); + } + + #[test] + fn test_encode_single_invalid_corrected() { + let mut encoder = Encoder::with_policy(BitSize::Two, Policy::SetToA); + assert!(encoder.encode_single(b"ACGTNNNN").unwrap().is_some()); + } + + #[test] + fn test_encode_paired_invalid_ignored() { + let mut encoder = Encoder::with_policy(BitSize::Two, Policy::IgnoreSequence); + assert!( + encoder + .encode_paired(b"ACGTNNNN", b"ACGTACGT") + .unwrap() + .is_none() + ); + } + + #[test] + fn test_encode_paired_invalid_corrected() { + let mut encoder = Encoder::with_policy(BitSize::Two, Policy::SetToA); + assert!( + encoder + .encode_paired(b"ACGTNNNN", b"NNNNACGT") + .unwrap() + .is_some() + ); + } +} diff --git a/src/lib.rs b/src/lib.rs index 7a3ccb6..b83e21c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -81,6 +81,9 @@ /// BQ - fixed length records, no quality scores pub mod bq; +/// Shared nucleotide encoder for BQ/VBQ writers +mod encoder; + /// Error definitions pub mod error; diff --git a/src/vbq/writer.rs b/src/vbq/writer.rs index 68dc116..3365ffb 100644 --- a/src/vbq/writer.rs +++ b/src/vbq/writer.rs @@ -64,15 +64,13 @@ use std::io::Write; -use bitnuc_deprec::BitSize; -use rand::SeedableRng; -use rand::rngs::SmallRng; use zstd::stream::copy_encode; use super::header::{BlockHeader, FileHeader}; use crate::SequencingRecord; +use crate::encoder::Encoder; use crate::error::{Result, WriteError}; -use crate::policy::{Policy, RNG_SEED}; +use crate::policy::Policy; use crate::vbq::header::{SIZE_BLOCK_HEADER, SIZE_HEADER}; use crate::vbq::index::{INDEX_END_MAGIC, IndexHeader}; use crate::vbq::{BlockIndex, BlockRange}; @@ -392,7 +390,7 @@ impl Writer { /// Returns the N-policy of the writer pub fn policy(&self) -> Policy { - self.encoder.policy + self.encoder.policy() } /// Checks if the writer is configured for quality scores @@ -1074,99 +1072,6 @@ impl BlockWriter { } } -/// Encapsulates the logic for encoding sequences into a binary format. -#[derive(Clone)] -pub struct Encoder { - /// Bitsize of the nucleotides - bitsize: BitSize, - - /// Reusable buffers for all nucleotides (written as 2-bit after conversion) - sbuffer: Vec, - xbuffer: Vec, - - /// Reusable buffers for invalid nucleotide sequences - s_ibuf: Vec, - x_ibuf: Vec, - - /// Invalid Nucleotide Policy - policy: Policy, - - /// Random Number Generator - rng: SmallRng, -} - -impl Encoder { - /// Initialize a new encoder with the given policy. - pub fn with_policy(bitsize: BitSize, policy: Policy) -> Self { - Self { - bitsize, - policy, - sbuffer: Vec::default(), - xbuffer: Vec::default(), - s_ibuf: Vec::default(), - x_ibuf: Vec::default(), - rng: SmallRng::seed_from_u64(RNG_SEED), - } - } - - /// Encodes a single sequence as 2-bit. - /// - /// Will return `None` if the sequence is invalid and the policy does not allow correction. - pub fn encode_single(&mut self, primary: &[u8]) -> Result> { - // Fill the buffer with the bit representation of the nucleotides - self.clear(); - if self.bitsize.encode(primary, &mut self.sbuffer).is_err() { - self.clear(); - if self - .policy - .handle(primary, &mut self.s_ibuf, &mut self.rng)? - { - self.bitsize.encode(&self.s_ibuf, &mut self.sbuffer)?; - } else { - return Ok(None); - } - } - Ok(Some(&self.sbuffer)) - } - - /// Encodes a pair of sequences as 2-bit. - /// - /// Will return `None` if either sequence is invalid and the policy does not allow correction. - pub fn encode_paired( - &mut self, - primary: &[u8], - extended: &[u8], - ) -> Result> { - self.clear(); - if self.bitsize.encode(primary, &mut self.sbuffer).is_err() - || self.bitsize.encode(extended, &mut self.xbuffer).is_err() - { - self.clear(); - if self - .policy - .handle(primary, &mut self.s_ibuf, &mut self.rng)? - && self - .policy - .handle(extended, &mut self.x_ibuf, &mut self.rng)? - { - self.bitsize.encode(&self.s_ibuf, &mut self.sbuffer)?; - self.bitsize.encode(&self.x_ibuf, &mut self.xbuffer)?; - } else { - return Ok(None); - } - } - Ok(Some((&self.sbuffer, &self.xbuffer))) - } - - /// Clear all buffers and reset the encoder. - pub fn clear(&mut self) { - self.sbuffer.clear(); - self.xbuffer.clear(); - self.s_ibuf.clear(); - self.x_ibuf.clear(); - } -} - #[cfg(test)] mod tests { use crate::utils::read_u64_le; From 946ad1cd9eb340f3d593b39eccc1e7e36933e7ee Mon Sep 17 00:00:00 2001 From: noamteyssier <22600644+noamteyssier@users.noreply.github.com> Date: Tue, 25 Aug 2026 19:09:31 -0700 Subject: [PATCH 10/25] refactor: simplify internals across readers, writers, and indexes - BatchRecord wraps RefRecord instead of re-implementing it - cbq thread partitioning uses chunks(div_ceil); - the cbq Span struct becomes Range; - vbq's flush_block becomes a method and push's twin branches merge (a full block now flushes before encoding, so a policy-skipped record may close a block earlier. files remain identical in format; - vbq index constructor pairs collapse and IndexHeader stops storing constant fields; - the fastx encode wrappers inline into one match. --- src/bq/reader.rs | 82 ++++++++++++--------------------- src/cbq/core/block.rs | 42 ++++++++--------- src/cbq/core/utils.rs | 24 ++-------- src/cbq/read.rs | 47 ++----------------- src/utils/fastx.rs | 90 ++++++++---------------------------- src/vbq/index.rs | 104 +++++++++--------------------------------- src/vbq/reader.rs | 49 ++++++++------------ src/vbq/writer.rs | 103 +++++++++++++++-------------------------- 8 files changed, 154 insertions(+), 387 deletions(-) diff --git a/src/bq/reader.rs b/src/bq/reader.rs index 2f0e6bd..19b9692 100644 --- a/src/bq/reader.rs +++ b/src/bq/reader.rs @@ -128,64 +128,38 @@ impl BinseqRecord for RefRecord<'_> { /// A reference to a record in the map with a precomputed decoded buffer slice pub struct BatchRecord<'a> { - /// Unprocessed buffer slice (with flags) - buffer: &'a [u64], + /// The underlying record view (encoded buffer, config, header) + inner: RefRecord<'a>, /// Decoded buffer slice dbuf: &'a [u8], - /// Record ID - id: u64, - /// The configuration that defines the layout and size of record components - config: RecordConfig, - /// A reusable pre-initialized quality score buffer - qbuf: &'a [u8], - /// Cached index string for the sequence header - header_buf: [u8; 20], - /// Length of the header in bytes - header_len: usize, } impl BinseqRecord for BatchRecord<'_> { fn bitsize(&self) -> BitSize { - self.config.bitsize + self.inner.bitsize() } fn index(&self) -> u64 { - self.id + self.inner.index() } - /// Clear the buffer and fill it with the sequence header fn sheader(&self) -> &[u8] { - &self.header_buf[..self.header_len] + self.inner.sheader() } - - /// Clear the buffer and fill it with the extended header fn xheader(&self) -> &[u8] { - self.sheader() + self.inner.xheader() } - fn flag(&self) -> Option { - if self.config.flags { - Some(self.buffer[0]) - } else { - None - } + self.inner.flag() } fn slen(&self) -> u64 { - self.config.slen + self.inner.slen() } fn xlen(&self) -> u64 { - self.config.xlen + self.inner.xlen() } fn sbuf(&self) -> &[u64] { - if self.config.flags { - &self.buffer[1..=(self.config.schunk as usize)] - } else { - &self.buffer[..(self.config.schunk as usize)] - } + self.inner.sbuf() } fn xbuf(&self) -> &[u64] { - if self.config.flags { - &self.buffer[1 + self.config.schunk as usize..] - } else { - &self.buffer[self.config.schunk as usize..] - } + self.inner.xbuf() } fn decode_s(&self, dbuf: &mut Vec) -> Result<()> { dbuf.extend_from_slice(self.sseq()); @@ -197,10 +171,11 @@ impl BinseqRecord for BatchRecord<'_> { } /// Override this method since we can make use of block information fn sseq(&self) -> &[u8] { - let scalar = self.config.scalar(); + let config = &self.inner.config; + let scalar = config.scalar(); let mut lbound = 0; - let mut rbound = self.config.slen(); - if self.config.flags { + let mut rbound = config.slen(); + if config.flags { lbound += scalar; rbound += scalar; } @@ -208,20 +183,21 @@ impl BinseqRecord for BatchRecord<'_> { } /// Override this method since we can make use of block information fn xseq(&self) -> &[u8] { - let scalar = self.config.scalar(); - let mut lbound = scalar * self.config.schunk(); - let mut rbound = lbound + self.config.xlen(); - if self.config.flags { + let config = &self.inner.config; + let scalar = config.scalar(); + let mut lbound = scalar * config.schunk(); + let mut rbound = lbound + config.xlen(); + if config.flags { lbound += scalar; rbound += scalar; } &self.dbuf[lbound..rbound] } fn squal(&self) -> &[u8] { - &self.qbuf[..self.config.slen()] + self.inner.squal() } fn xqual(&self) -> &[u8] { - &self.qbuf[..self.config.xlen()] + self.inner.xqual() } } @@ -949,13 +925,15 @@ impl ParallelReader for MmapReader { // initialize the record let record = BatchRecord { - buffer: &ebuf[ebuf_start..(ebuf_start + rsize_u64)], + inner: RefRecord { + buffer: &ebuf[ebuf_start..(ebuf_start + rsize_u64)], + qbuf: &qbuf, + id: idx as u64, + config: reader.config, + header_buf, + header_len, + }, dbuf: &dbuf[dbuf_start..(dbuf_start + dbuf_rsize)], - qbuf: &qbuf, - id: idx as u64, - config: reader.config, - header_buf, - header_len, }; // process the record diff --git a/src/cbq/core/block.rs b/src/cbq/core/block.rs index 6fa3e51..185e75c 100644 --- a/src/cbq/core/block.rs +++ b/src/cbq/core/block.rs @@ -10,7 +10,7 @@ use crate::cbq::core::utils::sized_compress; use crate::error::{CbqError, WriteError}; use crate::{BinseqRecord, BitSize, DEFAULT_QUALITY_SCORE, Result}; -use super::utils::{Span, calculate_offsets, extension_read, resize_uninit, slice_and_increment}; +use super::utils::{calculate_offsets, extension_read, resize_uninit, slice_and_increment, span}; use super::{BlockHeader, BlockRange, FileHeader}; use crate::SequencingRecord; @@ -767,10 +767,9 @@ impl<'a> Iterator for RefRecordIter<'a> { self.index }; - let sseq_span = - Span::new_u64(self.block.l_seq_offsets[seq_idx], self.block.l_seq[seq_idx]); + let sseq_span = span(self.block.l_seq_offsets[seq_idx], self.block.l_seq[seq_idx]); let sheader_span = if self.has_headers { - Some(Span::new_u64( + Some(span( self.block.l_header_offsets[seq_idx], self.block.l_headers[seq_idx], )) @@ -778,7 +777,7 @@ impl<'a> Iterator for RefRecordIter<'a> { None }; let xseq_span = if self.is_paired { - Some(Span::new_u64( + Some(span( self.block.l_seq_offsets[seq_idx + 1], self.block.l_seq[seq_idx + 1], )) @@ -786,7 +785,7 @@ impl<'a> Iterator for RefRecordIter<'a> { None }; let xheader_span = if self.is_paired && self.has_headers { - Some(Span::new_u64( + Some(span( self.block.l_header_offsets[seq_idx + 1], self.block.l_headers[seq_idx + 1], )) @@ -841,7 +840,7 @@ impl RefRecordIndex { } /// A reference to a record in a [`ColumnarBlock`](crate::cbq::ColumnarBlock) that implements the [`BinseqRecord`](crate::BinseqRecord) trait -#[derive(Clone, Copy)] +#[derive(Clone)] pub struct RefRecord<'a> { /// A reference to the block containing this record block: &'a ColumnarBlock, @@ -856,16 +855,16 @@ pub struct RefRecord<'a> { global_index: usize, /// Span of the primary sequence within the block - sseq_span: Span, + sseq_span: std::ops::Range, /// Span of the extended sequence within the block - xseq_span: Option, + xseq_span: Option>, /// Span of the primary header within the block - sheader_span: Option, + sheader_span: Option>, /// Span of the extended header within the block - xheader_span: Option, + xheader_span: Option>, /// A buffer to the name of this record when not storing headers rr_index: RefRecordIndex, @@ -888,16 +887,16 @@ impl BinseqRecord for RefRecord<'_> { } fn sheader(&self) -> &[u8] { - if let Some(span) = self.sheader_span { - &self.block.headers[span.range()] + if let Some(span) = &self.sheader_span { + &self.block.headers[span.clone()] } else { self.rr_index.as_bytes() } } fn xheader(&self) -> &[u8] { - if let Some(span) = self.xheader_span { - &self.block.headers[span.range()] + if let Some(span) = &self.xheader_span { + &self.block.headers[span.clone()] } else { self.rr_index.as_bytes() } @@ -916,7 +915,7 @@ impl BinseqRecord for RefRecord<'_> { } fn xlen(&self) -> u64 { - self.xseq_span.map_or(0, |span| span.len() as u64) + self.xseq_span.as_ref().map_or(0, |span| span.len() as u64) } fn decode_s(&self, buf: &mut Vec) -> crate::Result<()> { @@ -930,12 +929,13 @@ impl BinseqRecord for RefRecord<'_> { } fn sseq(&self) -> &[u8] { - &self.block.seq[self.sseq_span.range()] + &self.block.seq[self.sseq_span.clone()] } fn xseq(&self) -> &[u8] { self.xseq_span - .map_or(&[], |span| &self.block.seq[span.range()]) + .as_ref() + .map_or(&[], |span| &self.block.seq[span.clone()]) } fn has_quality(&self) -> bool { @@ -944,7 +944,7 @@ impl BinseqRecord for RefRecord<'_> { fn squal(&self) -> &[u8] { if self.has_quality() { - &self.block.qual[self.sseq_span.range()] + &self.block.qual[self.sseq_span.clone()] } else { &self.qbuf[..self.slen() as usize] } @@ -952,9 +952,9 @@ impl BinseqRecord for RefRecord<'_> { fn xqual(&self) -> &[u8] { if self.has_quality() - && let Some(span) = self.xseq_span + && let Some(span) = &self.xseq_span { - &self.block.qual[span.range()] + &self.block.qual[span.clone()] } else { &self.qbuf[..self.xlen() as usize] } diff --git a/src/cbq/core/utils.rs b/src/cbq/core/utils.rs index eb7503b..16fbc44 100644 --- a/src/cbq/core/utils.rs +++ b/src/cbq/core/utils.rs @@ -80,25 +80,7 @@ pub(crate) fn calculate_offsets(values: &[u64], offsets: &mut Vec) { } } -#[derive(Clone, Copy, Debug)] -pub struct Span { - offset: usize, - length: usize, -} -impl Span { - pub fn new(offset: usize, length: usize) -> Self { - Span { offset, length } - } - - pub fn new_u64(offset: u64, length: u64) -> Self { - Span::new(offset as usize, length as usize) - } - - pub fn range(&self) -> std::ops::Range { - self.offset..self.offset + self.length - } - - pub fn len(&self) -> usize { - self.length - } +/// Build a `Range` from a u64 offset and length. +pub(crate) fn span(offset: u64, length: u64) -> std::ops::Range { + offset as usize..(offset + length) as usize } diff --git a/src/cbq/read.rs b/src/cbq/read.rs index 763bcc8..9d864ce 100644 --- a/src/cbq/read.rs +++ b/src/cbq/read.rs @@ -254,53 +254,14 @@ impl ParallelReader for MmapReader { return Ok(()); // nothing to do } - // Distribute blocks evenly across threads, giving extra blocks to first threads - let base_blocks_per_thread = num_blocks / num_threads; - let extra_blocks = num_blocks % num_threads; - + // Distribute blocks evenly across threads let mut handles = Vec::new(); - for thread_id in 0..num_threads { - // Threads 0..extra_blocks get one extra block - let blocks_for_this_thread = if thread_id < extra_blocks { - base_blocks_per_thread + 1 - } else { - base_blocks_per_thread - }; - - // Calculate cumulative start position - let start_block_idx = if thread_id < extra_blocks { - thread_id * (base_blocks_per_thread + 1) - } else { - extra_blocks * (base_blocks_per_thread + 1) - + (thread_id - extra_blocks) * base_blocks_per_thread - }; - let end_block_idx = start_block_idx + blocks_for_this_thread; - - // Skip threads with no work (happens when num_threads > num_blocks) - if blocks_for_this_thread == 0 { - continue; - } - + for t_block_ranges in relevant_blocks.chunks(num_blocks.div_ceil(num_threads)) { + let t_block_ranges = t_block_ranges.to_vec(); + let range = range.clone(); let mut t_reader = self.clone(); let mut t_proc = processor.clone(); - // pull all block ranges for this thread - let t_block_ranges = relevant_blocks - .iter() - .skip(start_block_idx) - .take(end_block_idx - start_block_idx) - .copied() - .collect::>(); - - // eprintln!( - // "Thread {} block range: {}-{}. First block Cumulative Records: {}. Last block Cumulative Records: {}", - // thread_id, - // start_block_idx, - // end_block_idx, - // t_block_ranges[0].cumulative_records, - // t_block_ranges.last().unwrap().cumulative_records - // ); - let thread_handle = thread::spawn(move || -> crate::Result<()> { for b_range in t_block_ranges { t_reader.load_block(b_range)?; diff --git a/src/utils/fastx.rs b/src/utils/fastx.rs index 2a34919..1143855 100644 --- a/src/utils/fastx.rs +++ b/src/utils/fastx.rs @@ -215,96 +215,44 @@ impl FastxEncoderBuilder { }; let writer = self.builder.build(self.output)?; - if writer.is_paired() { - if let Some(r2) = r2 { - encode_paired(writer, r1, r2, self.threads)?; - } else { - encode_interleaved(writer, r1, self.threads)?; - } - } else { - encode_single_file(writer, r1, self.threads)?; + let paired = writer.is_paired(); + let mut encoder = Encoder::new(writer)?; + match (paired, r2) { + (true, Some(r2)) => r1.process_parallel_paired(r2, &mut encoder, self.threads), + (true, None) => r1.process_parallel_interleaved(&mut encoder, self.threads), + (false, _) => r1.process_parallel(&mut encoder, self.threads), } + .map_err(IntoBinseqError::into_binseq_error)?; + encoder.finish()?; Ok(()) } } -/// Encode single-end reads from a file -fn encode_single_file( - writer: BinseqWriter, - reader: fastx::Reader, - threads: usize, -) -> Result<()> { - let mut encoder = Encoder::new(writer)?; - reader - .process_parallel(&mut encoder, threads) - .map_err(IntoBinseqError::into_binseq_error)?; - encoder.finish()?; - Ok(()) -} - -/// Encode paired-end reads from interleaved file -fn encode_interleaved( - writer: BinseqWriter, - reader: fastx::Reader, - threads: usize, -) -> Result<()> { - let mut encoder = Encoder::new(writer)?; - reader - .process_parallel_interleaved(&mut encoder, threads) - .map_err(IntoBinseqError::into_binseq_error)?; - encoder.finish()?; - Ok(()) -} - -/// Encode paired-end reads from files -fn encode_paired( - writer: BinseqWriter, - r1: fastx::Reader, - r2: fastx::Reader, - threads: usize, -) -> Result<()> { - let mut encoder = Encoder::new(writer)?; - r1.process_parallel_paired(r2, &mut encoder, threads) - .map_err(IntoBinseqError::into_binseq_error)?; - encoder.finish()?; - Ok(()) -} - fn detect_seq_len( reader: &mut fastx::Reader, interleaved: bool, ) -> Result<(usize, usize)> { - // Initialze the record set + // Initialize the record set let mut rset = reader.new_record_set(); rset.fill(reader) .map_err(IntoBinseqError::into_binseq_error)?; - let (slen, xlen) = if interleaved { + let (slen, xlen) = { let mut rset_iter = rset.iter(); - let Some(Ok(slen)) = rset_iter.next().map(|r| -> Result { - let rec = r.map_err(IntoBinseqError::into_binseq_error)?; - Ok(rec.seq().len()) - }) else { - return Err(WriteError::EmptyFastxFile.into()); - }; - let Some(Ok(xlen)) = rset_iter.next().map(|r| -> Result { - let rec = r.map_err(IntoBinseqError::into_binseq_error)?; + let mut next_len = || -> Result { + let rec = rset_iter + .next() + .ok_or(WriteError::EmptyFastxFile)? + .map_err(IntoBinseqError::into_binseq_error)?; Ok(rec.seq().len()) - }) else { - return Err(WriteError::EmptyFastxFile.into()); }; + + let slen = next_len()?; + let xlen = if interleaved { next_len()? } else { 0 }; (slen, xlen) - } else { - let mut rset_iter = rset.iter(); - let Some(Ok(slen)) = rset_iter.next().map(|r| -> Result { - let rec = r.map_err(IntoBinseqError::into_binseq_error)?; - Ok(rec.seq().len()) - }) else { - return Err(WriteError::EmptyFastxFile.into()); - }; - (slen, 0) }; + reader .reload(&mut rset) .map_err(IntoBinseqError::into_binseq_error)?; diff --git a/src/vbq/index.rs b/src/vbq/index.rs index 2b90105..c530dca 100644 --- a/src/vbq/index.rs +++ b/src/vbq/index.rs @@ -34,9 +34,8 @@ use std::io::{Cursor, Read, Write}; -use zstd::{Decoder, Encoder}; - use crate::utils::{read_u32_le, read_u64_le}; +use zstd::{Decoder, Encoder}; use crate::error::{IndexError, Result}; @@ -184,18 +183,7 @@ impl BlockRange { Ok(()) } - /// Deserializes a `BlockRange` from a fixed-size buffer - /// - /// This method deserializes a `BlockRange` from a 32-byte buffer in the format - /// used by `write_bytes`. It's typically used when reading an index file. - /// - /// # Parameters - /// - /// * `buffer` - A fixed-size buffer containing a serialized `BlockRange` - /// - /// # Returns - /// - /// A new `BlockRange` with the values read from the buffer + /// Deserializes a `BlockRange` from a slice of bytes /// /// # Format /// @@ -205,8 +193,12 @@ impl BlockRange { /// - Bytes 16-19: `block_records` (u32, little endian) /// - Bytes 20-27: `cumulative_records` (u64, little endian) /// - Bytes 28-31: reservation (ignored, default value used) + /// + /// # Panics + /// + /// Panics if the buffer is less than 28 bytes long. #[must_use] - pub fn from_exact(buffer: &[u8; SIZE_BLOCK_RANGE]) -> Self { + pub fn from_bytes(buffer: &[u8]) -> Self { Self { start_offset: read_u64_le(&buffer[0..8]), len: read_u64_le(&buffer[8..16]), @@ -215,30 +207,6 @@ impl BlockRange { reservation: INDEX_RESERVATION, } } - - /// Deserializes a `BlockRange` from a slice of bytes - /// - /// This is a convenience method that copies the first 32 bytes from the provided slice - /// into a fixed-size buffer and then calls `from_exact`. It's useful when reading from - /// a larger buffer that contains multiple serialized `BlockRange` instances. - /// - /// # Parameters - /// - /// * `buffer` - A slice containing at least 32 bytes with a serialized `BlockRange` - /// - /// # Returns - /// - /// A new `BlockRange` with the values read from the buffer - /// - /// # Panics - /// - /// This method will panic if the buffer is less than 32 bytes long. - #[must_use] - pub fn from_bytes(buffer: &[u8]) -> Self { - let mut buf = [0; SIZE_BLOCK_RANGE]; - buf.copy_from_slice(buffer); - Self::from_exact(&buf) - } } /// Header for a VBQ index file @@ -250,22 +218,11 @@ impl BlockRange { /// The header has a fixed size of 32 bytes to ensure compatibility across versions. #[derive(Debug, Clone, Copy)] pub struct IndexHeader { - /// Magic number to designate the index file ("VBQINDEX" in ASCII) - /// - /// This is used to verify that a file is indeed a VBQ index file. - /// (8 bytes in serialized form) - magic: u64, - /// Total size of the indexed VBQ file in bytes /// - /// This is used to verify that the index matches the file it references. - /// (8 bytes in serialized form) + /// The serialized form also carries the `INDEX_MAGIC` magic number (8 bytes) + /// and 16 reserved bytes. bytes: u64, - - /// Reserved bytes for future extensions - /// - /// (16 bytes in serialized form) - reserved: [u8; INDEX_HEADER_SIZE - 16], } impl IndexHeader { /// Creates a new index header for a VBQ file of the specified size @@ -278,11 +235,7 @@ impl IndexHeader { /// /// A new `IndexHeader` instance with the appropriate magic number and size pub fn new(bytes: u64) -> Self { - Self { - magic: INDEX_MAGIC, - bytes, - reserved: [42; INDEX_HEADER_SIZE - 16], - } + Self { bytes } } /// Reads an index header from the provided reader /// @@ -305,30 +258,16 @@ impl IndexHeader { /// - Bytes 0-7: magic number (u64, little endian, must be `INDEX_MAGIC`) /// - Bytes 8-15: file size in bytes (u64, little endian) /// - Bytes 16-31: reserved for future extensions - pub fn from_reader(reader: &mut R) -> Result { - let mut buffer = [0; INDEX_HEADER_SIZE]; - reader.read_exact(&mut buffer)?; + pub fn from_bytes(buffer: &[u8]) -> Result { let magic = read_u64_le(&buffer[0..8]); - let bytes = read_u64_le(&buffer[8..16]); - let Ok(reserved) = buffer[16..INDEX_HEADER_SIZE].try_into() else { - return Err(IndexError::InvalidReservedBytes.into()); - }; if magic != INDEX_MAGIC { return Err(IndexError::InvalidMagicNumber(magic).into()); } Ok(Self { - magic, - bytes, - reserved, + bytes: read_u64_le(&buffer[8..16]), }) } - pub fn from_bytes(bytes: &[u8]) -> Result { - let mut buffer = [0; INDEX_HEADER_SIZE]; - buffer.copy_from_slice(&bytes[..INDEX_HEADER_SIZE]); - Self::from_reader(&mut Cursor::new(buffer)) - } - /// Serializes the index header to a binary format and writes it to the provided writer /// /// This method serializes the `IndexHeader` to a fixed-size 32-byte structure and @@ -349,11 +288,10 @@ impl IndexHeader { /// - Bytes 0-7: magic number (u64, little endian) /// - Bytes 8-15: file size in bytes (u64, little endian) /// - Bytes 16-31: reserved for future extensions - pub fn write_bytes(&self, writer: &mut W) -> Result<()> { - let mut buffer = [0; INDEX_HEADER_SIZE]; - buffer[0..8].copy_from_slice(&self.magic.to_le_bytes()); + pub fn write_bytes(self, writer: &mut W) -> Result<()> { + let mut buffer = [42; INDEX_HEADER_SIZE]; + buffer[0..8].copy_from_slice(&INDEX_MAGIC.to_le_bytes()); buffer[8..16].copy_from_slice(&self.bytes.to_le_bytes()); - buffer[16..].copy_from_slice(&self.reserved); writer.write_all(&buffer)?; Ok(()) } @@ -520,13 +458,14 @@ impl BlockIndex { &self.ranges } + /// Prints a tab-separated summary of each block range to stdout pub fn pprint(&self) { - self.ranges.iter().for_each(|range| { + for range in &self.ranges { println!( "{}\t{}\t{}\t{}", range.start_offset, range.len, range.block_records, range.cumulative_records ); - }); + } } /// Returns the total number of records in the dataset @@ -554,10 +493,10 @@ mod tests { // ==================== IndexHeader Tests ==================== #[test] - fn test_index_header_from_reader_invalid_magic() { + fn test_index_header_from_bytes_invalid_magic() { let mut buffer = [0u8; INDEX_HEADER_SIZE]; buffer[0..8].copy_from_slice(&0xDEAD_BEEF_u64.to_le_bytes()); - let result = IndexHeader::from_reader(&mut Cursor::new(buffer)); + let result = IndexHeader::from_bytes(&buffer); assert!(result.is_err()); } @@ -566,9 +505,8 @@ mod tests { let header = IndexHeader::new(12345); let mut buffer = Vec::new(); header.write_bytes(&mut buffer).unwrap(); - let parsed = IndexHeader::from_reader(&mut Cursor::new(buffer)).unwrap(); + let parsed = IndexHeader::from_bytes(&buffer).unwrap(); assert_eq!(parsed.bytes, 12345); - assert_eq!(parsed.magic, INDEX_MAGIC); } // ==================== BlockIndex round-trip through write_bytes/from_bytes ==================== diff --git a/src/vbq/reader.rs b/src/vbq/reader.rs index 80fc8f5..c298eb3 100644 --- a/src/vbq/reader.rs +++ b/src/vbq/reader.rs @@ -83,13 +83,19 @@ use crate::{ /// # Returns /// /// The number of 64-bit words required to encode the sequence -fn encoded_sequence_len(len: u64, bitsize: BitSize) -> usize { +/// Number of bases packed into each u64 word for the given bitsize +fn bases_per_word(bitsize: BitSize) -> usize { match bitsize { - BitSize::Two => len.div_ceil(32) as usize, - BitSize::Four => len.div_ceil(16) as usize, + BitSize::Two => 32, + BitSize::Four => 16, } } +/// Number of u64 words needed to store `len` bases at the given bitsize +fn encoded_sequence_len(len: u64, bitsize: BitSize) -> usize { + len.div_ceil(bases_per_word(bitsize) as u64) as usize +} + /// Represents a span (offset, length) into a buffer #[derive(Clone, Copy, Debug, Default)] pub struct Span { @@ -497,26 +503,23 @@ impl RecordBlock { Ok(()) } - /// Get decoded primary sequence for a record by index - #[must_use] - pub fn get_decoded_s(&self, record_idx: usize) -> Option<&[u8]> { - let meta = self.records.get(record_idx)?; + /// Slice the decoded buffer for a record given a word span and base length + fn decoded(&self, word_offset: usize, len: usize) -> Option<&[u8]> { if self.dbuf.is_empty() { return None; } - - let bases_per_word = match self.bitsize { - BitSize::Two => 32, - BitSize::Four => 16, - }; - // Calculate offset in decoded buffer (accounting for padding) - let offset = meta.s_seq_span.offset * bases_per_word; - let len = meta.slen as usize; - + let offset = word_offset * bases_per_word(self.bitsize); Some(&self.dbuf[offset..offset + len]) } + /// Get decoded primary sequence for a record by index + #[must_use] + pub fn get_decoded_s(&self, record_idx: usize) -> Option<&[u8]> { + let meta = self.records.get(record_idx)?; + self.decoded(meta.s_seq_span.offset, meta.slen as usize) + } + /// Get decoded extended sequence for a record by index #[must_use] pub fn get_decoded_x(&self, record_idx: usize) -> Option<&[u8]> { @@ -524,19 +527,7 @@ impl RecordBlock { if meta.xlen == 0 { return Some(&[]); } - if self.dbuf.is_empty() { - return None; - } - - let bases_per_word = match self.bitsize { - BitSize::Two => 32, - BitSize::Four => 16, - }; - - let offset = meta.x_seq_span.offset * bases_per_word; - let len = meta.xlen as usize; - - Some(&self.dbuf[offset..offset + len]) + self.decoded(meta.x_seq_span.offset, meta.xlen as usize) } } diff --git a/src/vbq/writer.rs b/src/vbq/writer.rs index 3365ffb..8553c77 100644 --- a/src/vbq/writer.rs +++ b/src/vbq/writer.rs @@ -474,6 +474,20 @@ impl Writer { /// writer.push(record).unwrap(); /// writer.finish().unwrap(); /// ``` + /// Flush the current block and record its range in the index + fn flush_block(&mut self) -> Result<()> { + let block_header = self.cblock.flush(&mut self.inner)?; + self.ranges.push(BlockRange::new( + self.bytes_written as u64, + block_header.size, + block_header.records, + self.records_written as u64, + )); + self.bytes_written += block_header.size_with_header(); + self.records_written += block_header.records as usize; + Ok(()) + } + pub fn push(&mut self, record: SequencingRecord) -> Result { // Check paired status - writer can require paired (record must have R2), // but if writer is single-end, we simply ignore any R2 data in the record. @@ -513,46 +527,27 @@ impl Writer { self.header.bits, ); - if self.header.is_paired() { - // encode the sequences - if let Some((sbuffer, xbuffer)) = self - .encoder + // flush the current block first if this record would overflow it + if self.cblock.exceeds_block_size(record_size)? { + self.flush_block()?; + } + + // encode the sequence(s); a `None` means the record was skipped by policy + let encoded = if self.header.is_paired() { + self.encoder .encode_paired(record.s_seq, record.x_seq.unwrap_or_default())? - { - if self.cblock.exceeds_block_size(record_size)? { - impl_flush_block( - &mut self.inner, - &mut self.cblock, - &mut self.ranges, - &mut self.bytes_written, - &mut self.records_written, - )?; - } - - self.cblock.write_record(&record, sbuffer, Some(xbuffer))?; - Ok(true) - } else { - Ok(false) - } + .map(|(sbuffer, xbuffer)| (sbuffer, Some(xbuffer))) } else { - // encode the sequence - if let Some(sbuffer) = self.encoder.encode_single(record.s_seq)? { - if self.cblock.exceeds_block_size(record_size)? { - impl_flush_block( - &mut self.inner, - &mut self.cblock, - &mut self.ranges, - &mut self.bytes_written, - &mut self.records_written, - )?; - } - - self.cblock.write_record(&record, sbuffer, None)?; - Ok(true) - } else { - Ok(false) - } - } + self.encoder + .encode_single(record.s_seq)? + .map(|sbuffer| (sbuffer, None)) + }; + let Some((sbuffer, xbuffer)) = encoded else { + return Ok(false); + }; + + self.cblock.write_record(&record, sbuffer, xbuffer)?; + Ok(true) } /// Finishes writing and flushes all data to the underlying writer @@ -593,13 +588,7 @@ impl Writer { /// ``` pub fn finish(&mut self) -> Result<()> { // Flush any remaining data in the current block - impl_flush_block( - &mut self.inner, - &mut self.cblock, - &mut self.ranges, - &mut self.bytes_written, - &mut self.records_written, - )?; + self.flush_block()?; self.inner.flush()?; // Always write the index - this is critical for VBQ file validity @@ -725,7 +714,7 @@ impl Writer { Ok(()) } - pub fn write_index(&mut self) -> Result<()> { + fn write_index(&mut self) -> Result<()> { // Build the index let index_header = IndexHeader::new(self.bytes_written as u64); let block_index = BlockIndex { @@ -753,26 +742,6 @@ impl Writer { } } -fn impl_flush_block( - writer: &mut W, - cblock: &mut BlockWriter, - ranges: &mut Vec, - bytes_written: &mut usize, - records_written: &mut usize, -) -> Result<()> { - let block_header = cblock.flush(writer)?; - let range = BlockRange::new( - *bytes_written as u64, - block_header.size, - block_header.records, - *records_written as u64, - ); - ranges.push(range); - *bytes_written += block_header.size_with_header(); - *records_written += block_header.records as usize; - Ok(()) -} - impl Drop for Writer { fn drop(&mut self) { self.finish().expect("Writer: Failed to finish writing"); @@ -1074,9 +1043,9 @@ impl BlockWriter { #[cfg(test)] mod tests { - use crate::utils::read_u64_le; use super::*; use crate::SequencingRecordBuilder; + use crate::utils::read_u64_le; use crate::vbq::{FileHeaderBuilder, header::SIZE_HEADER}; #[test] From 98e22b1231c105848cacaee6a07401bcb322a086 Mon Sep 17 00:00:00 2001 From: noamteyssier <22600644+noamteyssier@users.noreply.github.com> Date: Tue, 25 Aug 2026 19:14:48 -0700 Subject: [PATCH 11/25] test(policy): reduce the number of repeated tests and collapse into single table --- src/policy.rs | 285 ++++++++------------------------------------------ 1 file changed, 44 insertions(+), 241 deletions(-) diff --git a/src/policy.rs b/src/policy.rs index c09d326..3e9a515 100644 --- a/src/policy.rs +++ b/src/policy.rs @@ -84,13 +84,7 @@ impl Policy { for &n in sequence { ibuf.push(match n { b'A' | b'C' | b'G' | b'T' => n, - _ => match rng.random_range(0..4) { - 0 => b'A', - 1 => b'C', - 2 => b'G', - 3 => b'T', - _ => unreachable!(), - }, + _ => b"ACGT"[rng.random_range(0..4)], }); } } @@ -176,37 +170,28 @@ mod tests { use rand::SeedableRng; use rand::rngs::StdRng; - // ==================== Basic Policy Tests ==================== + fn run(policy: Policy, seq: &[u8]) -> (Result, Vec) { + let mut output = Vec::new(); + let mut rng = StdRng::seed_from_u64(RNG_SEED); + let result = policy.handle(seq, &mut output, &mut rng); + (result, output) + } #[test] fn test_default_policy() { - let policy = Policy::default(); - assert!(matches!(policy, Policy::IgnoreSequence)); + assert!(matches!(Policy::default(), Policy::IgnoreSequence)); } #[test] fn test_ignore_sequence_policy() { - let policy = Policy::IgnoreSequence; - let sequence = b"ACGTNX"; - let mut output = Vec::new(); - let mut rng = StdRng::seed_from_u64(RNG_SEED); - - let should_process = policy.handle(sequence, &mut output, &mut rng).unwrap(); - - assert!(!should_process); // Should return false to skip this sequence - assert!(output.is_empty()); // Output buffer should be empty + let (result, output) = run(Policy::IgnoreSequence, b"ACGTNX"); + assert!(!result.unwrap()); // Should return false to skip this sequence + assert!(output.is_empty()); } #[test] fn test_break_on_invalid_policy() { - let policy = Policy::BreakOnInvalid; - let sequence = b"ACGTNX"; - let mut output = Vec::new(); - let mut rng = StdRng::seed_from_u64(RNG_SEED); - - let result = policy.handle(sequence, &mut output, &mut rng); - - assert!(result.is_err()); + let (result, _) = run(Policy::BreakOnInvalid, b"ACGTNX"); assert!(matches!( result.unwrap_err(), crate::error::Error::WriteError(WriteError::InvalidNucleotideSequence(_)) @@ -214,237 +199,55 @@ mod tests { } #[test] - fn test_break_on_invalid_with_valid_sequence() { - let policy = Policy::BreakOnInvalid; - let sequence = b"ACGT"; - let mut output = Vec::new(); - let mut rng = StdRng::seed_from_u64(RNG_SEED); - - let result = policy.handle(sequence, &mut output, &mut rng); - - // Valid sequences should error because handle() doesn't validate for BreakOnInvalid - // It only returns an error immediately - assert!(result.is_err()); - } - - // ==================== Set-to-Specific-Nucleotide Tests ==================== - - #[test] - fn test_set_to_a_policy() { - let policy = Policy::SetToA; - let sequence = b"ACGTNX"; - let mut output = Vec::new(); - let mut rng = StdRng::seed_from_u64(RNG_SEED); - - let should_process = policy.handle(sequence, &mut output, &mut rng).unwrap(); - - assert!(should_process); // Should return true to process this sequence - assert_eq!(output, b"ACGTAA"); // N and X should be replaced with A - } - - #[test] - fn test_set_to_c_policy() { - let policy = Policy::SetToC; - let sequence = b"ACGTNX"; - let mut output = Vec::new(); - let mut rng = StdRng::seed_from_u64(RNG_SEED); - - let should_process = policy.handle(sequence, &mut output, &mut rng).unwrap(); - - assert!(should_process); - assert_eq!(output, b"ACGTCC"); // N and X should be replaced with C - } - - #[test] - fn test_set_to_g_policy() { - let policy = Policy::SetToG; - let sequence = b"ACGTNX"; - let mut output = Vec::new(); - let mut rng = StdRng::seed_from_u64(RNG_SEED); - - let should_process = policy.handle(sequence, &mut output, &mut rng).unwrap(); - - assert!(should_process); - assert_eq!(output, b"ACGTGG"); // N and X should be replaced with G - } - - #[test] - fn test_set_to_t_policy() { - let policy = Policy::SetToT; - let sequence = b"ACGTNX"; - let mut output = Vec::new(); - let mut rng = StdRng::seed_from_u64(RNG_SEED); - - let should_process = policy.handle(sequence, &mut output, &mut rng).unwrap(); - - assert!(should_process); - assert_eq!(output, b"ACGTTT"); // N and X should be replaced with T - } - - #[test] - fn test_all_valid_nucleotides_unchanged() { - let policy = Policy::SetToA; - let sequence = b"ACGTACGT"; - let mut output = Vec::new(); - let mut rng = StdRng::seed_from_u64(RNG_SEED); - - let should_process = policy.handle(sequence, &mut output, &mut rng).unwrap(); - - assert!(should_process); - assert_eq!(output, b"ACGTACGT"); // All valid, should remain unchanged + fn test_set_to_policies() { + // (policy, input, expected output) + let cases: &[(Policy, &[u8], &[u8])] = &[ + (Policy::SetToA, b"ACGTNX", b"ACGTAA"), + (Policy::SetToC, b"ACGTNX", b"ACGTCC"), + (Policy::SetToG, b"ACGTNX", b"ACGTGG"), + (Policy::SetToT, b"ACGTNX", b"ACGTTT"), + // valid nucleotides remain unchanged + (Policy::SetToA, b"ACGTACGT", b"ACGTACGT"), + // empty sequence + (Policy::SetToA, b"", b""), + // all invalid + (Policy::SetToG, b"NNNXXX", b"GGGGGG"), + // lowercase and ambiguous codes are invalid + (Policy::SetToA, b"acgt", b"AAAA"), + (Policy::SetToC, b"AcGt", b"ACGC"), + (Policy::SetToT, b"RYWSMK", b"TTTTTT"), + ]; + for &(policy, seq, expected) in cases { + let (result, output) = run(policy, seq); + assert!(result.unwrap(), "{policy:?} on {seq:?}"); + assert_eq!(output, expected, "{policy:?} on {seq:?}"); + } } - // ==================== Random Draw Tests ==================== - #[test] fn test_random_draw_policy() { - let policy = Policy::RandomDraw; - let sequence = b"ACGTNX"; - let mut output = Vec::new(); - let mut rng = StdRng::seed_from_u64(RNG_SEED); - - let should_process = policy.handle(sequence, &mut output, &mut rng).unwrap(); - - assert!(should_process); - assert_eq!(output.len(), 6); // Same length as input - // First 4 nucleotides should be unchanged - assert_eq!(&output[0..4], b"ACGT"); - // Last 2 should be valid nucleotides (A, C, G, or T) - assert!(matches!(output[4], b'A' | b'C' | b'G' | b'T')); - assert!(matches!(output[5], b'A' | b'C' | b'G' | b'T')); + let (result, output) = run(Policy::RandomDraw, b"ACGTNX"); + assert!(result.unwrap()); + assert_eq!(&output[0..4], b"ACGT"); // valid prefix unchanged + assert!(output[4..].iter().all(|n| b"ACGT".contains(n))); } #[test] fn test_random_draw_deterministic_with_seed() { - let policy = Policy::RandomDraw; - let sequence = b"NNNN"; - let mut output1 = Vec::new(); - let mut output2 = Vec::new(); - let mut rng1 = StdRng::seed_from_u64(RNG_SEED); - let mut rng2 = StdRng::seed_from_u64(RNG_SEED); - - policy.handle(sequence, &mut output1, &mut rng1).unwrap(); - policy.handle(sequence, &mut output2, &mut rng2).unwrap(); - - // Same seed should produce same output - assert_eq!(output1, output2); + let (_, output1) = run(Policy::RandomDraw, b"NNNN"); + let (_, output2) = run(Policy::RandomDraw, b"NNNN"); + assert_eq!(output1, output2); // same seed, same output } - // ==================== Buffer Clearing Tests ==================== - #[test] fn test_buffer_cleared_before_processing() { let policy = Policy::SetToA; - let sequence = b"ACGT"; - let mut output = vec![b'X', b'Y', b'Z']; // Pre-fill buffer + let mut output = vec![b'X', b'Y', b'Z']; // pre-filled buffer let mut rng = StdRng::seed_from_u64(RNG_SEED); - - policy.handle(sequence, &mut output, &mut rng).unwrap(); - - // Buffer should be cleared and only contain new data - assert_eq!(output, b"ACGT"); - } - - #[test] - fn test_multiple_calls_clear_buffer() { - let policy = Policy::SetToA; - let mut output = Vec::new(); - let mut rng = StdRng::seed_from_u64(RNG_SEED); - policy.handle(b"ACGT", &mut output, &mut rng).unwrap(); assert_eq!(output, b"ACGT"); policy.handle(b"TT", &mut output, &mut rng).unwrap(); - assert_eq!(output, b"TT"); // Should only contain second sequence - } - - // ==================== Edge Case Tests ==================== - - #[test] - fn test_empty_sequence() { - let policy = Policy::SetToA; - let sequence = b""; - let mut output = Vec::new(); - let mut rng = StdRng::seed_from_u64(RNG_SEED); - - let should_process = policy.handle(sequence, &mut output, &mut rng).unwrap(); - - assert!(should_process); - assert!(output.is_empty()); - } - - #[test] - fn test_all_invalid_nucleotides() { - let policy = Policy::SetToG; - let sequence = b"NNNXXX"; - let mut output = Vec::new(); - let mut rng = StdRng::seed_from_u64(RNG_SEED); - - let should_process = policy.handle(sequence, &mut output, &mut rng).unwrap(); - - assert!(should_process); - assert_eq!(output, b"GGGGGG"); // All should be replaced with G - } - - #[test] - fn test_policy_clone() { - let policy1 = Policy::SetToA; - let policy2 = policy1; - - // Should be able to use both (tests Copy trait) - let mut output = Vec::new(); - let mut rng = StdRng::seed_from_u64(RNG_SEED); - - policy1.handle(b"NT", &mut output, &mut rng).unwrap(); - assert_eq!(output, b"AT"); - - policy2.handle(b"NT", &mut output, &mut rng).unwrap(); - assert_eq!(output, b"AT"); - } - - #[test] - fn test_policy_debug() { - let policy = Policy::SetToA; - let debug_str = format!("{policy:?}"); - assert!(debug_str.contains("SetToA")); - } - - // ==================== Various Invalid Character Tests ==================== - - #[test] - fn test_lowercase_nucleotides_treated_as_invalid() { - let policy = Policy::SetToA; - let sequence = b"acgt"; // lowercase - let mut output = Vec::new(); - let mut rng = StdRng::seed_from_u64(RNG_SEED); - - policy.handle(sequence, &mut output, &mut rng).unwrap(); - - // Lowercase nucleotides should be treated as invalid - assert_eq!(output, b"AAAA"); - } - - #[test] - fn test_mixed_case_nucleotides() { - let policy = Policy::SetToC; - let sequence = b"AcGt"; - let mut output = Vec::new(); - let mut rng = StdRng::seed_from_u64(RNG_SEED); - - policy.handle(sequence, &mut output, &mut rng).unwrap(); - - assert_eq!(output, b"ACGC"); // Only uppercase are valid - } - - #[test] - fn test_ambiguous_nucleotide_codes() { - let policy = Policy::SetToT; - let sequence = b"RYWSMK"; // R, Y, W, S, M, K are ambiguous codes - let mut output = Vec::new(); - let mut rng = StdRng::seed_from_u64(RNG_SEED); - - policy.handle(sequence, &mut output, &mut rng).unwrap(); - - assert_eq!(output, b"TTTTTT"); // All ambiguous codes replaced with T + assert_eq!(output, b"TT"); // cleared again on the next call } } From aa23e14292e42c099e5aec682c04817d6eebbaa0 Mon Sep 17 00:00:00 2001 From: noamteyssier <22600644+noamteyssier@users.noreply.github.com> Date: Tue, 25 Aug 2026 19:20:30 -0700 Subject: [PATCH 12/25] chore: restrict internal reader items to pub(crate) --- src/bq/reader.rs | 8 ++++---- src/vbq/reader.rs | 5 ++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/bq/reader.rs b/src/bq/reader.rs index 19b9692..398ad19 100644 --- a/src/bq/reader.rs +++ b/src/bq/reader.rs @@ -61,7 +61,7 @@ impl<'a> RefRecord<'a> { /// /// Panics if the buffer length doesn't match the expected size from the config #[must_use] - pub fn new(id: u64, buffer: &'a [u64], qbuf: &'a [u8], config: RecordConfig) -> Self { + pub(crate) fn new(id: u64, buffer: &'a [u64], qbuf: &'a [u8], config: RecordConfig) -> Self { assert_eq!(buffer.len(), config.record_size_u64()); Self { id, @@ -127,7 +127,7 @@ impl BinseqRecord for RefRecord<'_> { } /// A reference to a record in the map with a precomputed decoded buffer slice -pub struct BatchRecord<'a> { +pub(crate) struct BatchRecord<'a> { /// The underlying record view (encoded buffer, config, header) inner: RefRecord<'a>, /// Decoded buffer slice @@ -208,7 +208,7 @@ impl BinseqRecord for BatchRecord<'_> { /// It handles the translation between sequence lengths in base pairs /// and the number of u64 chunks needed to store the compressed data. #[derive(Clone, Copy)] -pub struct RecordConfig { +pub(crate) struct RecordConfig { /// The primary sequence length in base pairs slen: u64, /// The extended sequence length in base pairs @@ -785,7 +785,7 @@ impl StreamReader { /// /// This constant defines how many records each thread processes at a time /// during parallel processing operations. -pub const BATCH_SIZE: usize = 1024; +pub(crate) const BATCH_SIZE: usize = 1024; /// Parallel processing implementation for memory-mapped readers impl ParallelReader for MmapReader { diff --git a/src/vbq/reader.rs b/src/vbq/reader.rs index c298eb3..4e9a782 100644 --- a/src/vbq/reader.rs +++ b/src/vbq/reader.rs @@ -54,10 +54,9 @@ use std::ops::Range; use std::path::Path; use std::sync::Arc; +use crate::utils::read_u64_le; use bitnuc_deprec::BitSize; use memmap2::Mmap; - -use crate::utils::read_u64_le; use zstd::zstd_safe; use super::{ @@ -98,7 +97,7 @@ fn encoded_sequence_len(len: u64, bitsize: BitSize) -> usize { /// Represents a span (offset, length) into a buffer #[derive(Clone, Copy, Debug, Default)] -pub struct Span { +pub(crate) struct Span { offset: usize, len: usize, } From ccae95e545259f7c180cfb6352f4b6d89dc88ac3 Mon Sep 17 00:00:00 2001 From: noamteyssier <22600644+noamteyssier@users.noreply.github.com> Date: Tue, 25 Aug 2026 21:25:16 -0700 Subject: [PATCH 13/25] chore: remove cargo config from repo --- .cargo/config.toml | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 .cargo/config.toml diff --git a/.cargo/config.toml b/.cargo/config.toml deleted file mode 100644 index ddff440..0000000 --- a/.cargo/config.toml +++ /dev/null @@ -1,2 +0,0 @@ -[build] -rustflags = ["-C", "target-cpu=native"] From 0432230a621d615ade55486a232d4e1ee346bdf1 Mon Sep 17 00:00:00 2001 From: noamteyssier <22600644+noamteyssier@users.noreply.github.com> Date: Tue, 25 Aug 2026 21:25:38 -0700 Subject: [PATCH 14/25] chore: track lock --- .gitignore | 1 - Cargo.lock | 1029 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 1029 insertions(+), 1 deletion(-) create mode 100644 Cargo.lock diff --git a/.gitignore b/.gitignore index e147b57..703c3c8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ /target -Cargo.lock data/test* diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..5efeea4 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,1029 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bgzip" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b64fd8980fb64af5951bc05de7772b598150a6f7eac42ec17f73e8489915f99b" +dependencies = [ + "flate2", + "log", + "rayon", + "thiserror 1.0.69", +] + +[[package]] +name = "binseq" +version = "0.9.6" +dependencies = [ + "anyhow", + "bitnuc 0.4.1", + "bitnuc 0.5.4", + "bytemuck", + "clap", + "itoa", + "memchr", + "memmap2", + "paraseq", + "parking_lot", + "rand", + "sucds", + "tempfile", + "thiserror 2.0.20", + "zstd", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "bitnuc" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7871516cbb4e097e220623917e1491c995ee58c8018d929d4b1e76c378002bf" + +[[package]] +name = "bitnuc" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ff0fcddd0c1e70392497c43846f73f5345d0588bd1aee9a16338c13f77223ba" +dependencies = [ + "fearless_simd", + "thiserror 2.0.20", +] + +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0e56a716f1e132ff6bf4bdac1c944a3fcdc1cae65f70a4a2a1ac3b401d2d1f" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "bzip2" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c" +dependencies = [ + "libbz2-rs-sys", +] + +[[package]] +name = "cc" +version = "1.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "clap" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "crc32fast" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "defmt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror 2.0.20", +] + +[[package]] +name = "either" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" + +[[package]] +name = "env_filter" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "fearless_simd" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4beca3cb2444e3304ac30843cc091f44ed58932353cd492ce740067bfce6b12" + +[[package]] +name = "find-msvc-tools" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jiff" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" +dependencies = [ + "defmt", + "jiff-core", + "jiff-static", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", +] + +[[package]] +name = "jiff-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" +dependencies = [ + "defmt", +] + +[[package]] +name = "jiff-static" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + +[[package]] +name = "libbz2-rs-sys" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "liblzma" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fe0a34ca854fd4f20c07f696fc8675aec78f87d88d29f5e10257a7490a1b2e1" +dependencies = [ + "liblzma-sys", +] + +[[package]] +name = "liblzma-sys" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0dad045e4b1b7b170be4b60b54b780cafb4490165461bac7d1cf7b703f61d5f" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "memmap2" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +dependencies = [ + "libc", +] + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "niffler" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01baa3696744e580712b28344fe25ed4ac1e8c4b76cc5189a34e713924c25208" +dependencies = [ + "bgzip", + "bzip2", + "cfg-if", + "flate2", + "liblzma", + "thiserror 2.0.20", + "zstd", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "paraseq" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f0c7259499c750423489d3fe955dfc60183f998f203a87a6f2d4c56872a2f4" +dependencies = [ + "anyhow", + "crossbeam-channel", + "either", + "env_logger", + "itertools", + "log", + "memchr", + "niffler", + "num_cpus", + "parking_lot", + "smallvec", + "thiserror 2.0.20", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.13.1", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags 2.13.1", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "sucds" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd324eaa05be64f105ea5269bb8aabd70e5dd57fa5c673b167f451b07d6c0dcd" +dependencies = [ + "anyhow", + "num-traits", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl 2.0.20", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "zerocopy" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] From 0b7feec9ab04cf31ab03799d8dedaabbd08e8f99 Mon Sep 17 00:00:00 2001 From: noamteyssier <22600644+noamteyssier@users.noreply.github.com> Date: Tue, 25 Aug 2026 21:34:30 -0700 Subject: [PATCH 15/25] docs: update changelog and bump version --- CHANGELOG.md | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ Cargo.lock | 2 +- Cargo.toml | 2 +- 3 files changed, 51 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ab0efee..b9e070e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,55 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.10.0] - 2026-08-25 + +- Large cleanup of internal and public API. +- Runtime dependencies removed (`byteorder`, `num_cpus`, `auto_impl`, and `memchr`). +- Deprecated items removed from the public API. + +### Removed + +- **`bq::StreamWriter` / `bq::StreamWriterBuilder`** + - pure delegating wrappers; compose `bq::WriterBuilder` with `std::io::BufWriter` instead (see the `network_streaming` example). +- **Deprecated `write_record` / `write_paired_record`** on the `bq` and `vbq` writers + - use `push(SequencingRecord)`. +- **`bq::Encoder`** from the public API + - `bq` and `vbq` now share one internal encoder; + - sequence-length validation happens in `bq::Writer::push`. +- **`vbq::BlockIndex::from_vbq`** + - indexes are embedded in VBQ files - removing this holdover from way back when; + - use `MmapReader::load_index`. +- **Redundant header constructors** + - `bq::FileHeader::{new, new_extended, set_bitsize}` and `vbq::FileHeader::{new, with_capacity, set_bitsize}`; + - use the `FileHeaderBuilder`s. +- **`SequencingRecord` getters** + - the fields are now public; the unused `opt_s_header`, `opt_x_seq`, `opt_x_header`, and `opt_flag` builder setters are gone (`opt_s_qual` / `opt_x_qual` remain). +- **Never-constructed error variants** + - the five `WriteError` flag-mismatch variants and `CbqError::MissingSequenceOnSequencingRecord`; + - `BuilderError` folds into `HeaderError::MissingSequenceLength` and `FastxEncodingError` into `WriteError`. +- **`&R` / `&mut R` impls of `BinseqRecord`** + - the `auto_impl` dependency is dropped and nothing used the reference impls; + - pass records by value (they are already references anyways). +- Dead accessors throughout (`RefRecord::{config, set_id}`, `cbq` writer `usage` / `bytes_written`, `with_compression_level`, and similar zero-caller items), plus the redundant `streaming` and `write` examples. + +### Changed + +- FASTX encoding now drains only _completed_ blocks per batch and does a final drain per thread + - CBQ blocks stay full across batches for better compression and less time under the global writer lock. +- Dependencies: `byteorder` replaced with `std` little-endian conversions (byte-identical output), `num_cpus` replaced with `std::thread::available_parallelism`, `auto_impl` dropped, and `memchr` moved to dev-dependencies. +- VBQ writer flushes a full block _before_ encoding the incoming record, so a policy-skipped record may close a block earlier; + - **the on-disk format is unchanged**. +- Internal simplification: + - one shared `bq`/`vbq` encoder + - `BatchRecord` wraps `RefRecord` + - `cbq` `Span` replaced by `Range` + - unreachable `pub` items demoted to `pub(crate)`. + +### Fixed + +- The `error` test module was missing `#[cfg(test)]`, so a test-only enum compiled into release builds. +- `cbq::Index::average_block_size` reports `0.0` instead of `NaN` for files with fewer than two blocks. + ## [0.9.6] - 2026-08-12 ### Changed diff --git a/Cargo.lock b/Cargo.lock index 5efeea4..f7e9efd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -93,7 +93,7 @@ dependencies = [ [[package]] name = "binseq" -version = "0.9.6" +version = "0.10.0" dependencies = [ "anyhow", "bitnuc 0.4.1", diff --git a/Cargo.toml b/Cargo.toml index 9b39d01..1617f61 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "binseq" -version = "0.9.6" +version = "0.10.0" edition = "2024" description = "A high efficiency binary format for sequencing data" license = "MIT" From a2d315757814ea89ec69f2e091bec62838c386f0 Mon Sep 17 00:00:00 2001 From: noamteyssier <22600644+noamteyssier@users.noreply.github.com> Date: Tue, 25 Aug 2026 21:44:15 -0700 Subject: [PATCH 16/25] ci: fix ci to point to auto-write --- .github/workflows/ci.yml | 4 ++-- CHANGELOG.md | 3 +++ src/utils/fastx.rs | 10 ++++++++-- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d145442..92dd039 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,9 +57,9 @@ jobs: steps: - uses: actions/checkout@v7 - name: run example (single) ${{ matrix.ext }} - run: cargo run --release --example write -- ./data/subset_R1.fastq.gz -o ./output.${{ matrix.ext }} + run: cargo run --release --example auto-write -- ./data/subset_R1.fastq.gz -o ./output.${{ matrix.ext }} - name: run example (paired) ${{ matrix.ext }} - run: cargo run --release --example write -- ./data/subset_R1.fastq.gz ./data/subset_R2.fastq.gz -o ./output.${{ matrix.ext }} + run: cargo run --release --example auto-write -- ./data/subset_R1.fastq.gz ./data/subset_R2.fastq.gz -o ./output.${{ matrix.ext }} example_read: runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index b9e070e..de62094 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - The `error` test module was missing `#[cfg(test)]`, so a test-only enum compiled into release builds. - `cbq::Index::average_block_size` reports `0.0` instead of `NaN` for files with fewer than two blocks. +- `encode_fastx` with single non-paired input no longer records second read's length as `xlen` + - made fixed-length BQ output look paired and fail on interleaved encoding. + - The extended length now only probes when builder is configured paired. ## [0.9.6] - 2026-08-12 diff --git a/src/utils/fastx.rs b/src/utils/fastx.rs index 1143855..a0459c2 100644 --- a/src/utils/fastx.rs +++ b/src/utils/fastx.rs @@ -189,14 +189,20 @@ impl FastxEncoderBuilder { // build interleaved reader let mut reader = fastx::Reader::from_path(path).map_err(IntoBinseqError::into_binseq_error)?; - let (slen, xlen) = detect_seq_len(&mut reader, true)?; + // Only probe for an extended length when the writer is configured + // as paired (i.e. the single input is interleaved); otherwise a + // second read's length would mark fixed-length formats as paired. + let (slen, xlen) = detect_seq_len(&mut reader, self.builder.paired)?; self.builder = self.builder.slen(slen as u32).xlen(xlen as u32); (reader, None) } Some(FastxInput::Stdin) => { let mut reader = fastx::Reader::from_stdin().map_err(IntoBinseqError::into_binseq_error)?; - let (slen, xlen) = detect_seq_len(&mut reader, true)?; + // Only probe for an extended length when the writer is configured + // as paired (i.e. the single input is interleaved); otherwise a + // second read's length would mark fixed-length formats as paired. + let (slen, xlen) = detect_seq_len(&mut reader, self.builder.paired)?; self.builder = self.builder.slen(slen as u32).xlen(xlen as u32); (reader, None) } From 5e695d4f33356ccad3040cbf06d0adedc09a9d6d Mon Sep 17 00:00:00 2001 From: noamteyssier <22600644+noamteyssier@users.noreply.github.com> Date: Wed, 26 Aug 2026 09:28:08 -0700 Subject: [PATCH 17/25] fix(vbq): only flush a full block once the record will be written The push refactor flushed pre-encoding so a policy-skipped record that would have overflowed the block could close it early and fragment output. Encode first then flush via a split-borrow helper. Restores 0.9.x block packing exactly; adds a regression test asserting skipped records leave output identical. Also reattaches push's doc comment, which the earlier refactor had orphaned onto flush_block. --- CHANGELOG.md | 2 - src/vbq/writer.rs | 97 +++++++++++++++++++++++++++++++++++++---------- 2 files changed, 78 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index de62094..a74064a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,8 +41,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - FASTX encoding now drains only _completed_ blocks per batch and does a final drain per thread - CBQ blocks stay full across batches for better compression and less time under the global writer lock. - Dependencies: `byteorder` replaced with `std` little-endian conversions (byte-identical output), `num_cpus` replaced with `std::thread::available_parallelism`, `auto_impl` dropped, and `memchr` moved to dev-dependencies. -- VBQ writer flushes a full block _before_ encoding the incoming record, so a policy-skipped record may close a block earlier; - - **the on-disk format is unchanged**. - Internal simplification: - one shared `bq`/`vbq` encoder - `BatchRecord` wraps `RefRecord` diff --git a/src/vbq/writer.rs b/src/vbq/writer.rs index 8553c77..44f20bc 100644 --- a/src/vbq/writer.rs +++ b/src/vbq/writer.rs @@ -430,6 +430,40 @@ impl Writer { self.header.headers } + /// Flush the current block and record its range in the index. + /// + /// Free-standing over the individual fields so callers holding a borrow of + /// `self.encoder` (the encoded record buffers) can still flush. + fn flush_block_split( + inner: &mut W, + cblock: &mut BlockWriter, + ranges: &mut Vec, + bytes_written: &mut usize, + records_written: &mut usize, + ) -> Result<()> { + let block_header = cblock.flush(inner)?; + ranges.push(BlockRange::new( + *bytes_written as u64, + block_header.size, + block_header.records, + *records_written as u64, + )); + *bytes_written += block_header.size_with_header(); + *records_written += block_header.records as usize; + Ok(()) + } + + /// Flush the current block and record its range in the index + fn flush_block(&mut self) -> Result<()> { + Self::flush_block_split( + &mut self.inner, + &mut self.cblock, + &mut self.ranges, + &mut self.bytes_written, + &mut self.records_written, + ) + } + /// Writes a record using the unified [`SequencingRecord`] API /// /// This method provides a consistent interface with BQ and CBQ writers. @@ -474,20 +508,6 @@ impl Writer { /// writer.push(record).unwrap(); /// writer.finish().unwrap(); /// ``` - /// Flush the current block and record its range in the index - fn flush_block(&mut self) -> Result<()> { - let block_header = self.cblock.flush(&mut self.inner)?; - self.ranges.push(BlockRange::new( - self.bytes_written as u64, - block_header.size, - block_header.records, - self.records_written as u64, - )); - self.bytes_written += block_header.size_with_header(); - self.records_written += block_header.records as usize; - Ok(()) - } - pub fn push(&mut self, record: SequencingRecord) -> Result { // Check paired status - writer can require paired (record must have R2), // but if writer is single-end, we simply ignore any R2 data in the record. @@ -527,11 +547,6 @@ impl Writer { self.header.bits, ); - // flush the current block first if this record would overflow it - if self.cblock.exceeds_block_size(record_size)? { - self.flush_block()?; - } - // encode the sequence(s); a `None` means the record was skipped by policy let encoded = if self.header.is_paired() { self.encoder @@ -546,6 +561,19 @@ impl Writer { return Ok(false); }; + // flush the current block if this record would overflow it; done only + // once we know the record will actually be written, so policy-skipped + // records never fragment blocks (split-borrow: `encoded` holds the encoder) + if self.cblock.exceeds_block_size(record_size)? { + Self::flush_block_split( + &mut self.inner, + &mut self.cblock, + &mut self.ranges, + &mut self.bytes_written, + &mut self.records_written, + )?; + } + self.cblock.write_record(&record, sbuffer, xbuffer)?; Ok(true) } @@ -1048,6 +1076,37 @@ mod tests { use crate::utils::read_u64_le; use crate::vbq::{FileHeaderBuilder, header::SIZE_HEADER}; + #[test] + fn test_policy_skipped_records_do_not_fragment_blocks() -> super::Result<()> { + // Two writers fed the same valid records must produce identical output, + // even when one is also fed invalid (policy-skipped) records that would + // overflow the current block: a skipped record must never trigger a flush. + let header = FileHeaderBuilder::new().block(2048).build(); + let mut plain = WriterBuilder::default().header(header).build(Vec::new())?; + let mut skipped = WriterBuilder::default().header(header).build(Vec::new())?; + + let valid = b"ACGT".repeat(25); // 100 bp + let invalid = b"N".repeat(300); // always skipped by IgnoreSequence + for _ in 0..200 { + let invalid_record = SequencingRecordBuilder::default() + .s_seq(&invalid) + .build()?; + assert!(!skipped.push(invalid_record)?); + + let valid_record = SequencingRecordBuilder::default().s_seq(&valid).build()?; + assert!(plain.push(valid_record)?); + let valid_record = SequencingRecordBuilder::default().s_seq(&valid).build()?; + assert!(skipped.push(valid_record)?); + } + plain.finish()?; + skipped.finish()?; + + assert!(plain.ranges.len() > 1, "test should span multiple blocks"); + assert_eq!(plain.ranges.len(), skipped.ranges.len()); + assert_eq!(plain.inner, skipped.inner); + Ok(()) + } + #[test] fn test_headless_writer() -> super::Result<()> { let writer = WriterBuilder::default().headless(true).build(Vec::new())?; From 75ddfae6700cb09c4c8b8942059677c2c8d7e37c Mon Sep 17 00:00:00 2001 From: noamteyssier <22600644+noamteyssier@users.noreply.github.com> Date: Wed, 26 Aug 2026 09:28:33 -0700 Subject: [PATCH 18/25] style: formatting --- src/vbq/writer.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/vbq/writer.rs b/src/vbq/writer.rs index 44f20bc..e5641be 100644 --- a/src/vbq/writer.rs +++ b/src/vbq/writer.rs @@ -1088,9 +1088,7 @@ mod tests { let valid = b"ACGT".repeat(25); // 100 bp let invalid = b"N".repeat(300); // always skipped by IgnoreSequence for _ in 0..200 { - let invalid_record = SequencingRecordBuilder::default() - .s_seq(&invalid) - .build()?; + let invalid_record = SequencingRecordBuilder::default().s_seq(&invalid).build()?; assert!(!skipped.push(invalid_record)?); let valid_record = SequencingRecordBuilder::default().s_seq(&valid).build()?; From 5936d2f4666c8a9062f712c589aec94b22547160 Mon Sep 17 00:00:00 2001 From: noamteyssier <22600644+noamteyssier@users.noreply.github.com> Date: Wed, 26 Aug 2026 09:33:46 -0700 Subject: [PATCH 19/25] refactor: safe accessors for dbuf with slices --- src/bq/reader.rs | 4 ++-- src/cbq/core/block.rs | 8 ++++++-- src/vbq/reader.rs | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/bq/reader.rs b/src/bq/reader.rs index 398ad19..30a6b86 100644 --- a/src/bq/reader.rs +++ b/src/bq/reader.rs @@ -179,7 +179,7 @@ impl BinseqRecord for BatchRecord<'_> { lbound += scalar; rbound += scalar; } - &self.dbuf[lbound..rbound] + self.dbuf.get(lbound..rbound).unwrap_or_default() } /// Override this method since we can make use of block information fn xseq(&self) -> &[u8] { @@ -191,7 +191,7 @@ impl BinseqRecord for BatchRecord<'_> { lbound += scalar; rbound += scalar; } - &self.dbuf[lbound..rbound] + self.dbuf.get(lbound..rbound).unwrap_or_default() } fn squal(&self) -> &[u8] { self.inner.squal() diff --git a/src/cbq/core/block.rs b/src/cbq/core/block.rs index 185e75c..a022b60 100644 --- a/src/cbq/core/block.rs +++ b/src/cbq/core/block.rs @@ -929,13 +929,17 @@ impl BinseqRecord for RefRecord<'_> { } fn sseq(&self) -> &[u8] { - &self.block.seq[self.sseq_span.clone()] + self.block + .seq + .get(self.sseq_span.clone()) + .unwrap_or_default() } fn xseq(&self) -> &[u8] { self.xseq_span .as_ref() - .map_or(&[], |span| &self.block.seq[span.clone()]) + .and_then(|span| self.block.seq.get(span.clone())) + .unwrap_or_default() } fn has_quality(&self) -> bool { diff --git a/src/vbq/reader.rs b/src/vbq/reader.rs index 4e9a782..a4b2205 100644 --- a/src/vbq/reader.rs +++ b/src/vbq/reader.rs @@ -509,7 +509,7 @@ impl RecordBlock { } // Calculate offset in decoded buffer (accounting for padding) let offset = word_offset * bases_per_word(self.bitsize); - Some(&self.dbuf[offset..offset + len]) + self.dbuf.get(offset..offset + len) } /// Get decoded primary sequence for a record by index From 1758693f900425ea4e0f0314f8765e00366d48ec Mon Sep 17 00:00:00 2001 From: noamteyssier <22600644+noamteyssier@users.noreply.github.com> Date: Wed, 26 Aug 2026 12:07:19 -0700 Subject: [PATCH 20/25] feat!: make paraseq and anyhow opt-in anyhow currently is still in the dependency tree unless sucds can remove it (pr in progress). fixes an error where anyhow was not actually an optional feature since ? was auto-impl with thiserror. removing paraseq from the default build path and updating docs to show under optional features. Also update the example to write to CBQ --- .github/workflows/ci.yml | 13 ++++-- CHANGELOG.md | 4 ++ Cargo.toml | 12 ++++- examples/parallel_range.rs | 96 +++++++++++++++++++------------------- src/cbq/core/block.rs | 21 +++++++-- src/lib.rs | 2 + src/write.rs | 12 ++--- 7 files changed, 94 insertions(+), 66 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 92dd039..9310482 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,9 +16,14 @@ jobs: - name: Build run: cargo build --verbose - name: Run tests - run: cargo test --verbose + run: cargo test --verbose --all-features - name: Run tests (release) - run: cargo test --verbose --release + run: cargo test --verbose --release --all-features + - name: Check feature matrix + run: | + cargo check --no-default-features + cargo check --no-default-features --features paraseq + cargo check --no-default-features --features anyhow fmt_lint: runs-on: ubuntu-latest @@ -57,9 +62,9 @@ jobs: steps: - uses: actions/checkout@v7 - name: run example (single) ${{ matrix.ext }} - run: cargo run --release --example auto-write -- ./data/subset_R1.fastq.gz -o ./output.${{ matrix.ext }} + run: cargo run --release --example auto-write -F paraseq -- ./data/subset_R1.fastq.gz -o ./output.${{ matrix.ext }} - name: run example (paired) ${{ matrix.ext }} - run: cargo run --release --example auto-write -- ./data/subset_R1.fastq.gz ./data/subset_R2.fastq.gz -o ./output.${{ matrix.ext }} + run: cargo run --release --example auto-write -F paraseq -- ./data/subset_R1.fastq.gz ./data/subset_R2.fastq.gz -o ./output.${{ matrix.ext }} example_read: runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index a74064a..9e65b2a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - FASTX encoding now drains only _completed_ blocks per batch and does a final drain per thread - CBQ blocks stay full across batches for better compression and less time under the global writer lock. - Dependencies: `byteorder` replaced with `std` little-endian conversions (byte-identical output), `num_cpus` replaced with `std::thread::available_parallelism`, `auto_impl` dropped, and `memchr` moved to dev-dependencies. +- The `paraseq` feature is no longer enabled by default, reducing compile times when FASTX encoding is not needed. Enable it with `features = ["paraseq"]` to use `encode_fastx` (which remains visible on docs.rs with a feature banner). +- `anyhow` is now actually an optional dependency: + - the CBQ Elias-Fano codec is handled directly without the auto-derive from thiserror + - fixes the optional feature which actually never compiled when disabled. - Internal simplification: - one shared `bq`/`vbq` encoder - `BatchRecord` wraps `RefRecord` diff --git a/Cargo.toml b/Cargo.toml index 1617f61..81ef10b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ categories = ["science::bioinformatics", "encoding", "data-structures"] keywords = ["binary", "nucleotide", "sequencing", "genomics", "fastq"] [dependencies] -anyhow = {version = "1.0.103", optional = true} +anyhow = { version = "1.0.103", optional = true } bitnuc-deprec = { package ="bitnuc", version = "0.4.1" } bitnuc = { version = "0.5.1" } bytemuck = { version = "1.25.1", features = ["derive", "extern_crate_alloc"] } @@ -33,7 +33,7 @@ paraseq = "0.4.14" tempfile = "3.27.0" [features] -default = ["paraseq", "anyhow"] +default = [] anyhow = ["dep:anyhow"] paraseq = ["dep:paraseq", "dep:parking_lot"] @@ -44,3 +44,11 @@ cast_precision_loss = "allow" missing_errors_doc = "allow" struct_excessive_bools = "allow" fn_params_excessive_bools = "allow" + +[[example]] +name = "auto-write" +required-features = ["paraseq"] + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--cfg", "docsrs"] diff --git a/examples/parallel_range.rs b/examples/parallel_range.rs index 2d7eacf..0c2f2bf 100644 --- a/examples/parallel_range.rs +++ b/examples/parallel_range.rs @@ -1,7 +1,11 @@ -use binseq::{BinseqReader, BinseqRecord, ParallelProcessor, ParallelReader, Result}; +use std::path::PathBuf; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; +use clap::Parser; + +use binseq::{BinseqReader, BinseqRecord, ParallelProcessor, ParallelReader, Result}; + #[derive(Clone)] struct RangeProcessor { counter: Arc, @@ -63,78 +67,72 @@ impl ParallelProcessor for RangeProcessor { } } -fn main() -> Result<()> { - let args: Vec = std::env::args().collect(); - if args.len() < 2 { - eprintln!( - "Usage: {} [num_threads] [start] [end]", - args[0] - ); - eprintln!("Example: {} data/subset.bq 4 1000 5000", args[0]); - std::process::exit(1); - } +#[derive(Parser)] +struct Cli { + path: PathBuf, + + #[clap(short = 'T', long, default_value_t = 0)] + threads: usize, - let file_path = &args[1]; - let num_threads = args - .get(2) - .unwrap_or(&"4".to_string()) - .parse::() - .map_err(|e| binseq::Error::from(anyhow::Error::from(e)))?; + /// Optional start index for processing range + #[clap(long, default_value_t = 0)] + start: usize, + + /// Optional end index for processing range + #[clap(long, default_value_t = 10_000)] + end: usize, +} + +fn main() -> Result<()> { + let args = Cli::parse(); // Create reader to get total record count - let reader = BinseqReader::new(file_path)?; + let reader = BinseqReader::new(&args.path)?; let total_records = reader.num_records()?; - println!("File: {file_path}"); + println!("File: {}", args.path.display()); println!("Total records in file: {total_records}"); - // Parse range arguments or use defaults - let start = args - .get(3) - .map(|s| s.parse::()) - .transpose() - .map_err(|e| binseq::Error::from(anyhow::Error::from(e)))? - .unwrap_or(0); - let end = args - .get(4) - .map(|s| s.parse::()) - .transpose() - .map_err(|e| binseq::Error::from(anyhow::Error::from(e)))? - .unwrap_or(total_records.min(10_000)); // Default to first 10k records - // Validate range - if start >= total_records { - eprintln!("Error: Start index {start} is >= total records {total_records}"); + if args.start >= total_records { + eprintln!( + "Error: Start index {} is >= total records {total_records}", + args.start + ); std::process::exit(1); } - if end > total_records { + if args.end > total_records { eprintln!( - "Warning: End index {end} is > total records {total_records}, clamping to {total_records}" + "Warning: End index {} is > total records {total_records}, clamping to {total_records}", + args.end ); } - let end = end.min(total_records); + let end = args.end.min(total_records); - if start >= end { - eprintln!("Error: Start index {start} must be < end index {end}"); + if args.start >= end { + eprintln!( + "Error: Start index {} must be < end index {end}", + args.start + ); std::process::exit(1); } println!( "Processing range: {} to {} ({} records)", - start, + args.start, end, - end - start + end - args.start ); - println!("Using {num_threads} threads"); + println!("Using {} threads", args.threads); println!(); // Demonstrate processing the full file println!("=== Processing full file ==="); - let reader_full = BinseqReader::new(file_path)?; + let reader_full = BinseqReader::new(&args.path)?; let processor_full = RangeProcessor::new(0, total_records); let start_time = std::time::Instant::now(); - reader_full.process_parallel(processor_full.clone(), num_threads)?; + reader_full.process_parallel(processor_full.clone(), args.threads)?; let elapsed_full = start_time.elapsed(); println!("Full file processing completed!"); @@ -144,16 +142,16 @@ fn main() -> Result<()> { // Demonstrate processing a specific range println!("=== Processing specific range ==="); - let reader_range = BinseqReader::new(file_path)?; - let processor_range = RangeProcessor::new(start, end); + let reader_range = BinseqReader::new(args.path)?; + let processor_range = RangeProcessor::new(args.start, end); let start_time = std::time::Instant::now(); - reader_range.process_parallel_range(processor_range.clone(), num_threads, start..end)?; + reader_range.process_parallel_range(processor_range.clone(), args.threads, args.start..end)?; let elapsed_range = start_time.elapsed(); println!("Range processing completed!"); println!("Records processed: {}", processor_range.count()); - println!("Expected records: {}", end - start); + println!("Expected records: {}", end - args.start); println!("Time taken: {elapsed_range:.2?}"); // Compare performance diff --git a/src/cbq/core/block.rs b/src/cbq/core/block.rs index a022b60..d4768fd 100644 --- a/src/cbq/core/block.rs +++ b/src/cbq/core/block.rs @@ -372,8 +372,13 @@ impl ColumnarBlock { self.ef = None; Ok(()) } else { - let mut ef_builder = EliasFanoBuilder::new(self.seq.len(), self.npos.len())?; - ef_builder.extend(self.npos.iter().map(|idx| *idx as usize))?; + #[allow(clippy::redundant_closure_for_method_calls)] + let mut ef_builder = EliasFanoBuilder::new(self.seq.len(), self.npos.len()) + .map_err(|e| e.into_boxed_dyn_error())?; + #[allow(clippy::redundant_closure_for_method_calls)] + ef_builder + .extend(self.npos.iter().map(|idx| *idx as usize)) + .map_err(|e| e.into_boxed_dyn_error())?; let ef = ef_builder.build(); self.ef = Some(ef); @@ -404,7 +409,9 @@ impl ColumnarBlock { // compress N-positions (Elias-Fano encoded) if let Some(ef) = self.ef.as_ref() { - ef.serialize_into(&mut self.ef_bytes)?; + #[allow(clippy::redundant_closure_for_method_calls)] + ef.serialize_into(&mut self.ef_bytes) + .map_err(|e| e.into_boxed_dyn_error())?; self.len_nef = self.ef_bytes.len(); sized_compress(&mut self.z_npos, &self.ef_bytes, cctx)?; } @@ -457,7 +464,9 @@ impl ColumnarBlock { self.ef_bytes.reserve(self.len_nef); copy_decode(self.z_npos.as_slice(), &mut self.ef_bytes)?; - let ef = EliasFano::deserialize_from(self.ef_bytes.as_slice())?; + #[allow(clippy::redundant_closure_for_method_calls)] + let ef = EliasFano::deserialize_from(self.ef_bytes.as_slice()) + .map_err(|e| e.into_boxed_dyn_error())?; self.ef = Some(ef); } @@ -624,7 +633,9 @@ impl ColumnarBlock { .map_err(|e| io::Error::other(zstd_safe::get_error_name(e)))?; // reinitialize the EliasFano encoding - let ef = EliasFano::deserialize_from(self.ef_bytes.as_slice())?; + #[allow(clippy::redundant_closure_for_method_calls)] + let ef = EliasFano::deserialize_from(self.ef_bytes.as_slice()) + .map_err(|e| e.into_boxed_dyn_error())?; self.ef = Some(ef); } diff --git a/src/lib.rs b/src/lib.rs index b83e21c..5511d60 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,5 @@ +#![cfg_attr(docsrs, feature(doc_cfg))] +#![cfg_attr(docsrs, doc(auto_cfg))] #![doc = include_str!("../README.md")] //! //! # BINSEQ diff --git a/src/write.rs b/src/write.rs index 89dd1b0..f4b072f 100644 --- a/src/write.rs +++ b/src/write.rs @@ -305,7 +305,7 @@ impl BinseqWriterBuilder { /// Encode FASTX file(s) to BINSEQ format /// - /// This method returns a [`FastxEncoderBuilder`] that allows you to configure + /// This method returns a [`FastxEncoderBuilder`](crate::utils::FastxEncoderBuilder) that allows you to configure /// the input source and threading options before executing the encoding. /// /// This is an alternative to [`build`](Self::build) that directly processes @@ -321,19 +321,19 @@ impl BinseqWriterBuilder { /// use binseq::write::{BinseqWriterBuilder, Format}; /// use std::fs::File; /// - /// // Encode from stdin to VBQ - /// let writer = BinseqWriterBuilder::new(Format::Vbq) + /// // Encode from stdin to CBQ + /// let writer = BinseqWriterBuilder::new(Format::Cbq) /// .quality(true) /// .headers(true) - /// .encode_fastx(File::create("output.vbq")?) + /// .encode_fastx(File::create("output.cbq")?) /// .input_stdin() /// .threads(8) /// .run()?; /// /// // Encode paired-end reads - /// let writer = BinseqWriterBuilder::new(Format::Vbq) + /// let writer = BinseqWriterBuilder::new(Format::Cbq) /// .quality(true) - /// .encode_fastx(File::create("output.vbq")?) + /// .encode_fastx(File::create("output.cbq")?) /// .input_paired("R1.fastq", "R2.fastq") /// .run()?; /// # Ok::<(), binseq::Error>(()) From b117a14febfef4a364c30c9258ff1b9bb6f4228c Mon Sep 17 00:00:00 2001 From: noamteyssier <22600644+noamteyssier@users.noreply.github.com> Date: Wed, 26 Aug 2026 12:10:44 -0700 Subject: [PATCH 21/25] ci: don't need to run examples in release mode and fix args --- .github/workflows/ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9310482..615253d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,7 +42,7 @@ jobs: steps: - uses: actions/checkout@v7 - name: run example ${{ matrix.ext }} - run: cargo run --release --example grep -- ./data/subset.${{ matrix.ext }} "ACGTACGT" + run: cargo run --example grep -- ./data/subset.${{ matrix.ext }} "ACGTACGT" example_range: runs-on: ubuntu-latest @@ -52,7 +52,7 @@ jobs: steps: - uses: actions/checkout@v7 - name: run example ${{ matrix.ext }} - run: cargo run --release --example parallel_range -- ./data/subset.${{ matrix.ext }} 4 30 200 + run: cargo run --example parallel_range -- ./data/subset.${{ matrix.ext }} -T 4 --start 30 --end 200 example_write: runs-on: ubuntu-latest @@ -62,9 +62,9 @@ jobs: steps: - uses: actions/checkout@v7 - name: run example (single) ${{ matrix.ext }} - run: cargo run --release --example auto-write -F paraseq -- ./data/subset_R1.fastq.gz -o ./output.${{ matrix.ext }} + run: cargo run --example auto-write -F paraseq -- ./data/subset_R1.fastq.gz -o ./output.${{ matrix.ext }} - name: run example (paired) ${{ matrix.ext }} - run: cargo run --release --example auto-write -F paraseq -- ./data/subset_R1.fastq.gz ./data/subset_R2.fastq.gz -o ./output.${{ matrix.ext }} + run: cargo run --example auto-write -F paraseq -- ./data/subset_R1.fastq.gz ./data/subset_R2.fastq.gz -o ./output.${{ matrix.ext }} example_read: runs-on: ubuntu-latest @@ -74,4 +74,4 @@ jobs: steps: - uses: actions/checkout@v7 - name: run example ${{ matrix.ext }} - run: cargo run --release --example read -- ./data/subset.${{ matrix.ext }} + run: cargo run --example read -- ./data/subset.${{ matrix.ext }} From 9a5200dab13626c6aba4f10d149c7cf76829c583 Mon Sep 17 00:00:00 2001 From: noamteyssier <22600644+noamteyssier@users.noreply.github.com> Date: Wed, 26 Aug 2026 12:42:04 -0700 Subject: [PATCH 22/25] refactor: add back in opt flag --- CHANGELOG.md | 2 +- src/record/sequencing_record.rs | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e65b2a..767cd55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,7 +27,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `bq::FileHeader::{new, new_extended, set_bitsize}` and `vbq::FileHeader::{new, with_capacity, set_bitsize}`; - use the `FileHeaderBuilder`s. - **`SequencingRecord` getters** - - the fields are now public; the unused `opt_s_header`, `opt_x_seq`, `opt_x_header`, and `opt_flag` builder setters are gone (`opt_s_qual` / `opt_x_qual` remain). + - the fields are now public; the unused `opt_s_header`, `opt_x_seq`, and `opt_x_header` builder setters are gone (`opt_s_qual` / `opt_x_qual` / `opt_flag` remain). - **Never-constructed error variants** - the five `WriteError` flag-mismatch variants and `CbqError::MissingSequenceOnSequencingRecord`; - `BuilderError` folds into `HeaderError::MissingSequenceLength` and `FastxEncodingError` into `WriteError`. diff --git a/src/record/sequencing_record.rs b/src/record/sequencing_record.rs index ba3a876..2e83947 100644 --- a/src/record/sequencing_record.rs +++ b/src/record/sequencing_record.rs @@ -304,6 +304,13 @@ impl<'a> SequencingRecordBuilder<'a> { self } + /// Sets the flag value from an Option + #[must_use] + pub fn opt_flag(mut self, flag: Option) -> Self { + self.flag = flag; + self + } + /// Builds the `SequencingRecord` /// /// # Errors @@ -375,10 +382,12 @@ mod tests { .s_seq(b"ACGT") .opt_s_qual(Some(b"FFFF")) .opt_x_qual(None) + .opt_flag(Some(42)) .build() .unwrap(); assert_eq!(record.s_qual, Some(b"FFFF".as_slice())); assert_eq!(record.x_qual, None); + assert_eq!(record.flag, Some(42)); } #[test] From bea1c4cd93b8615b2de9505fdba853453f8f7bda Mon Sep 17 00:00:00 2001 From: noamteyssier <22600644+noamteyssier@users.noreply.github.com> Date: Wed, 26 Aug 2026 12:56:43 -0700 Subject: [PATCH 23/25] refactor: add back in all opt --- CHANGELOG.md | 3 +-- src/record/sequencing_record.rs | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 767cd55..715f772 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,8 +26,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Redundant header constructors** - `bq::FileHeader::{new, new_extended, set_bitsize}` and `vbq::FileHeader::{new, with_capacity, set_bitsize}`; - use the `FileHeaderBuilder`s. -- **`SequencingRecord` getters** - - the fields are now public; the unused `opt_s_header`, `opt_x_seq`, and `opt_x_header` builder setters are gone (`opt_s_qual` / `opt_x_qual` / `opt_flag` remain). - **Never-constructed error variants** - the five `WriteError` flag-mismatch variants and `CbqError::MissingSequenceOnSequencingRecord`; - `BuilderError` folds into `HeaderError::MissingSequenceLength` and `FastxEncodingError` into `WriteError`. @@ -38,6 +36,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- `SequencingRecord` fields are now public and the getters are removed - FASTX encoding now drains only _completed_ blocks per batch and does a final drain per thread - CBQ blocks stay full across batches for better compression and less time under the global writer lock. - Dependencies: `byteorder` replaced with `std` little-endian conversions (byte-identical output), `num_cpus` replaced with `std::thread::available_parallelism`, `auto_impl` dropped, and `memchr` moved to dev-dependencies. diff --git a/src/record/sequencing_record.rs b/src/record/sequencing_record.rs index 2e83947..cb912da 100644 --- a/src/record/sequencing_record.rs +++ b/src/record/sequencing_record.rs @@ -269,6 +269,12 @@ impl<'a> SequencingRecordBuilder<'a> { self } + /// Sets the primary header from an Option + pub fn opt_s_header(mut self, s_header: Option<&'a [u8]>) -> Self { + self.s_header = s_header; + self + } + /// Sets the extended/paired sequence #[must_use] pub fn x_seq(mut self, x_seq: &'a [u8]) -> Self { @@ -276,6 +282,13 @@ impl<'a> SequencingRecordBuilder<'a> { self } + /// Sets the extended/paired sequence from an Option + #[must_use] + pub fn opt_x_seq(mut self, x_seq: Option<&'a [u8]>) -> Self { + self.x_seq = x_seq; + self + } + /// Sets the extended quality scores #[must_use] pub fn x_qual(mut self, x_qual: &'a [u8]) -> Self { @@ -297,6 +310,13 @@ impl<'a> SequencingRecordBuilder<'a> { self } + /// Sets the extended header from an Option + #[must_use] + pub fn opt_x_header(mut self, x_header: Option<&'a [u8]>) -> Self { + self.x_header = x_header; + self + } + /// Sets the flag value #[must_use] pub fn flag(mut self, flag: u64) -> Self { @@ -381,12 +401,19 @@ mod tests { let record = SequencingRecordBuilder::default() .s_seq(b"ACGT") .opt_s_qual(Some(b"FFFF")) + .opt_s_header(Some(b"s_hdr")) + .opt_x_seq(Some(b"TGCA")) .opt_x_qual(None) + .opt_x_header(None) .opt_flag(Some(42)) .build() .unwrap(); + // field access and getters are equivalent assert_eq!(record.s_qual, Some(b"FFFF".as_slice())); + assert_eq!(record.s_header, Some(b"s_hdr".as_slice())); + assert_eq!(record.x_seq, Some(b"TGCA".as_slice())); assert_eq!(record.x_qual, None); + assert_eq!(record.x_header, None); assert_eq!(record.flag, Some(42)); } From 20a68bd5cacccb31357acc1b2ebd68dc29e3b1a9 Mon Sep 17 00:00:00 2001 From: noamteyssier <22600644+noamteyssier@users.noreply.github.com> Date: Wed, 26 Aug 2026 12:58:57 -0700 Subject: [PATCH 24/25] refactor: add back in with_compression_level --- CHANGELOG.md | 2 +- src/cbq/core/header.rs | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 715f772..4e72fdb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,7 +32,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **`&R` / `&mut R` impls of `BinseqRecord`** - the `auto_impl` dependency is dropped and nothing used the reference impls; - pass records by value (they are already references anyways). -- Dead accessors throughout (`RefRecord::{config, set_id}`, `cbq` writer `usage` / `bytes_written`, `with_compression_level`, and similar zero-caller items), plus the redundant `streaming` and `write` examples. +- Dead accessors throughout (`RefRecord::{config, set_id}`, `cbq` writer `usage` / `bytes_written`, and similar zero-caller items), plus the redundant `streaming` and `write` examples. ### Changed diff --git a/src/cbq/core/header.rs b/src/cbq/core/header.rs index 92c6145..5860a58 100644 --- a/src/cbq/core/header.rs +++ b/src/cbq/core/header.rs @@ -147,6 +147,10 @@ impl FileHeaderBuilder { self } + pub fn with_compression_level(&mut self, compression_level: usize) -> &mut Self { + self.with_optional_compression_level(Some(compression_level)) + } + pub fn with_block_size(&mut self, block_size: usize) -> &mut Self { self.block_size = Some(block_size); self From f91762a02bf2ca35ff24ab9670d55d891683cbc7 Mon Sep 17 00:00:00 2001 From: noamteyssier <22600644+noamteyssier@users.noreply.github.com> Date: Wed, 26 Aug 2026 12:59:37 -0700 Subject: [PATCH 25/25] style: clippy fix --- src/record/sequencing_record.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/record/sequencing_record.rs b/src/record/sequencing_record.rs index cb912da..7437d78 100644 --- a/src/record/sequencing_record.rs +++ b/src/record/sequencing_record.rs @@ -270,6 +270,7 @@ impl<'a> SequencingRecordBuilder<'a> { } /// Sets the primary header from an Option + #[must_use] pub fn opt_s_header(mut self, s_header: Option<&'a [u8]>) -> Self { self.s_header = s_header; self