diff --git a/AGENTS.md b/AGENTS.md index 96b8f23..79181fc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -98,8 +98,11 @@ each implementation. 2. **Non-owning indexes**: indexes use external immutable data where appropriate, commonly `std::span`. The caller keeps that data alive and stable for the index lifetime. -3. **SIMD conditional compilation**: use `PIXIE_AVX512_SUPPORT` and - `PIXIE_AVX2_SUPPORT` guards with a scalar fallback. +3. **SIMD primitive dispatch**: keep `PIXIE_AVX512_SUPPORT` and + `PIXIE_AVX2_SUPPORT` feature guards, SIMD implementations, and scalar + fallbacks inside the low-level primitive header that owns the operation. + Callers use one unconditional primitive API and must not branch on Pixie + SIMD feature macros. 4. **Target domain**: optimize for bit sequences and indexes up to `2^64` bits. 5. **Platform**: Linux/Unix is the current target platform. `MappedFile` @@ -115,6 +118,20 @@ each implementation. invalidated by owner resize or destruction. 8. **Optional adapters**: third-party implementations stay behind their build option and must not become a library runtime dependency. +9. **Serialization and residency**: `BinaryReader` is a parse-time cursor over + contiguous virtual address space; a byte span does not imply that every page + is resident in RAM. A reader over `MappedFile` relies on normal OS demand + paging. Zero-copy deserializers retain views into the backing storage, not + the reader, so the backing owner must remain alive and immutable while + queries access those views directly. +10. **Serialization output buffering**: `BinaryWriter` writes through an + explicit seekable sink and owns only a fixed-size staging buffer, or borrows + one supplied by the caller. `VectorOutputSink` is the explicit + whole-artifact-in-memory choice; `SpanOutputSink` and POSIX + `io::FileOutputSink` provide bounded-memory destinations. Call + `BinaryWriter::finish()` before consuming the sink. Current framing uses + backpatching, so non-seekable pipes, sockets, and compression streams would + require a future counting pass or format change. ### Why Header-Only? @@ -227,8 +244,9 @@ ctest --preset release -L rank_select_tests The registered test executables are `bit_algorithms_unittests`, `rank_select_unittests`, `rank_select_tests`, `benchmark_tests`, `test_rmm`, `tree_tests`, `wavelet_tree_tests`, `storage_tests`, -`excess_positions_tests`, `excess_record_lows_tests`, and `rmq_tests`. Run an -executable directly only when debugging a focused Google Test filter. +`serialization_tests`, `excess_positions_tests`, `excess_record_lows_tests`, +and `rmq_tests`. Run an executable directly only when debugging a focused +Google Test filter. ### Test Configuration via Environment Variables @@ -327,13 +345,17 @@ The script configures and builds the `coverage` preset, deletes stale ### Modifying SIMD Code -1. Keep an AVX-512 implementation, AVX2 implementation where useful, and a +1. Keep feature detection and SIMD/scalar dispatch local to the low-level + primitive that owns the operation. Callers must use its stable API + unconditionally; do not leak `PIXIE_*_SUPPORT` guards into data-structure + code. +2. Keep an AVX-512 implementation, AVX2 implementation where useful, and a scalar fallback behind the existing feature guards. -2. Include `` only in translation units or headers that use SIMD +3. Include `` only in translation units or headers that use SIMD intrinsics; do not make it a generic benchmark dependency. -3. Validate the fallback with the `asan` preset or an isolated +4. Validate the fallback with the `asan` preset or an isolated `DISABLE_AVX512=ON` build. -4. Build the relevant benchmark preset before claiming a performance result. +5. Build the relevant benchmark preset before claiming a performance result. Use `benchmarks-profile` for hardware counters when supported by the host. ### Adding Tests diff --git a/CMakeLists.txt b/CMakeLists.txt index 1b2e56e..59027b5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -138,6 +138,16 @@ target_compile_features(pixie INTERFACE cxx_std_20) target_include_directories(pixie INTERFACE $) +# Keep Pixie's own targets warning-clean without imposing -Werror on consumers. +function (pixie_enable_project_warnings target) + if (CMAKE_CXX_COMPILER_ID MATCHES "Clang|GNU") + target_compile_options(${target} PRIVATE + -Wall + -Wextra + -Werror) + endif () +endfunction () + if (PIXIE_DIAGNOSTICS) target_compile_definitions(pixie INTERFACE PIXIE_DIAGNOSTICS) target_link_libraries(pixie INTERFACE spdlog::spdlog_header_only) @@ -232,6 +242,15 @@ if (PIXIE_TESTS) gtest_main ${PIXIE_DIAGNOSTICS_LIBS}) + add_executable(serialization_tests + src/tests/serialization_tests.cpp) + target_include_directories(serialization_tests + PUBLIC include) + target_link_libraries(serialization_tests + gtest + gtest_main + ${PIXIE_DIAGNOSTICS_LIBS}) + add_executable(excess_positions_tests src/tests/excess_positions_tests.cpp) target_include_directories(excess_positions_tests @@ -281,11 +300,13 @@ if (PIXIE_TESTS) tree_tests wavelet_tree_tests storage_tests + serialization_tests excess_positions_tests select512_experimental_tests excess_record_lows_tests rmq_tests) foreach (test_target IN LISTS PIXIE_TEST_TARGETS) + pixie_enable_project_warnings(${test_target}) gtest_discover_tests(${test_target} DISCOVERY_MODE PRE_TEST TEST_PREFIX "${test_target}." @@ -373,6 +394,15 @@ if (PIXIE_BENCHMARKS) benchmark_main ${PIXIE_DIAGNOSTICS_LIBS}) + add_executable(serialization_benchmarks + src/benchmarks/serialization_benchmarks.cpp) + target_include_directories(serialization_benchmarks + PUBLIC include) + target_link_libraries(serialization_benchmarks + benchmark + benchmark_main + ${PIXIE_DIAGNOSTICS_LIBS}) + add_executable(bp_tree_benchmarks src/benchmarks/bp_tree_benchmarks.cpp) target_include_directories(bp_tree_benchmarks @@ -425,6 +455,27 @@ if (PIXIE_BENCHMARKS) benchmark benchmark_main ${PIXIE_DIAGNOSTICS_LIBS}) + + set(PIXIE_BENCHMARK_TARGETS + rank_select_benchmarks + rmm_benchmarks + rmm_btree_benchmarks + rmq_benchmarks + louds_tree_benchmarks + wavelet_tree_benchmarks + serialization_benchmarks + bp_tree_benchmarks + dfuds_tree_benchmarks + alignment_comparison_benchmarks + excess_positions_benchmarks + select512_benchmarks) + if (PIXIE_THIRD_PARTY_BACKENDS) + list(APPEND PIXIE_BENCHMARK_TARGETS + rmm_sdsl_benchmarks) + endif () + foreach (benchmark_target IN LISTS PIXIE_BENCHMARK_TARGETS) + pixie_enable_project_warnings(${benchmark_target}) + endforeach () endif () # --------------------------------------------------------------------------- diff --git a/agentic/local/cpp/skills/benchmarks/EXAMPLES.md b/agentic/local/cpp/skills/benchmarks/EXAMPLES.md index d034038..0855601 100644 --- a/agentic/local/cpp/skills/benchmarks/EXAMPLES.md +++ b/agentic/local/cpp/skills/benchmarks/EXAMPLES.md @@ -53,6 +53,37 @@ methodology, align tables visually in source, and exclude local JSON, failed probes, and before/after experiment history. Persist other results only for an explicitly experimental implementation that has a registered benchmark. +## Serialization Benchmarking + +The serialization benchmark binary is: + +```bash +./build/benchmarks/serialization_benchmarks +``` + +It separates primitive `BinaryReader` and `BinaryWriter` throughput, zero-copy +byte-span traversal, fixed-span and growable-vector sinks, warm memory-mapped +reads, page-cache file writes, framed records with backpatching, and end-to-end +rank/select, RmM, RMQ, and wavelet-tree serialization. Structure rows report +logical artifact throughput through `bytes_per_second`, plus `artifact_bytes` +and `items` counters. Setup, source generation, mapping, and initial structure +construction are outside the timed region. + +Use a pinned Release run and filter the subsystem being investigated. For +example: + +```bash +taskset -c 0 ./build/benchmarks/serialization_benchmarks \ + --benchmark_filter='^(BM_Binary(Reader|Writer)|BM_(RankSelect|RmM|Rmq|WaveletTree))' \ + --benchmark_report_aggregates_only=true \ + --benchmark_display_aggregates_only=true +``` + +The mapped-reader rows warm every mapped page before timing. The file-writer +rows use real time and close the file, but deliberately do not call `fsync()`; +interpret them as page-cache/file-system throughput rather than durable-storage +latency. + ## RMQ Benchmark Tables The primary RMQ benchmark binary is usually: diff --git a/include/pixie/bit_stream.h b/include/pixie/bit_stream.h deleted file mode 100644 index f5fa72e..0000000 --- a/include/pixie/bit_stream.h +++ /dev/null @@ -1,72 +0,0 @@ -#pragma once - -#include -#include -#include - -namespace pixie { - -class OutputBitStream { - private: - size_t size_; - std::vector data_; - - public: - OutputBitStream() : size_(0) {} - - /** - * @brief Writes one bit to the stream - */ - OutputBitStream& operator<<(bool bit) { - if (size_ % 64 == 0) { - data_.push_back(bit); - } else if (bit) { - data_.back() |= 1ull << (size_ % 64); - } - size_++; - return *this; - } - - /** - * @brief Writes bits of the integral number to the stream in little-endian - */ - template - OutputBitStream& operator<<(T bits) { - using UT = std::make_unsigned_t; - UT ubits = static_cast(bits); - constexpr size_t length = sizeof(T) * 8; - static_assert(length <= 64); - if (size_ % 64 == 0) { - data_.push_back(ubits); - } else { - const size_t prefix = std::min(length, 64 - (size_ % 64)); - data_.back() |= static_cast(ubits & ((1ull << prefix) - 1)) - << (size_ % 64); - if (prefix < length) { - data_.push_back(ubits >> prefix); - } - } - size_ += length; - return *this; - } - - /** - * @brief Returns the number of written bits - * - */ - size_t size() const { return size_; } - - /** - * @brief Reserves memory for "size" bits - * - */ - void reserve(size_t size) { data_.reserve((size + 63) / 64); } - - /** - * @brief Moves vector containing written bits. There must be no operator<< - * after extract() - */ - std::vector extract() { return std::move(data_); } -}; - -} // namespace pixie diff --git a/include/pixie/bits.h b/include/pixie/bits.h index 57ffc3a..6ecc980 100644 --- a/include/pixie/bits.h +++ b/include/pixie/bits.h @@ -306,11 +306,11 @@ static inline uint64_t rank_512(const uint64_t* x, uint64_t count) { #else - uint64_t last_uint = count < 512 ? count >> 6 : 8; + size_t last_uint = count < 512 ? count >> 6 : 8; uint64_t pop_val = 0; - for (int i = 0; i < last_uint; i++) { + for (size_t i = 0; i < last_uint; i++) { pop_val += std::popcount(x[i]); } @@ -412,7 +412,7 @@ template static inline uint64_t select_512_scalar_impl(const uint64_t* x, uint64_t rank) { size_t word = 0; - int count; + uint64_t count; if constexpr (Invert) { count = std::popcount(~x[0]); } else { diff --git a/include/pixie/detail/serialization.h b/include/pixie/detail/serialization.h new file mode 100644 index 0000000..ac7adb0 --- /dev/null +++ b/include/pixie/detail/serialization.h @@ -0,0 +1,145 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pixie::detail { + +inline constexpr std::uint8_t kLittleEndianMarker = 1; + +template +void write_magic(BinaryWriter& stream, + const std::array& magic) { + for (const std::uint8_t byte : magic) { + stream.write_u8(byte); + } +} + +template +void require_magic(BinaryReader& reader, + const std::array& expected) { + for (const std::uint8_t byte : expected) { + const std::size_t offset = reader.byte_offset(); + if (reader.read_u8() != byte) { + throw SerializationError("Serialized artifact has the wrong magic", + offset); + } + } +} + +template +void write_integral(BinaryWriter& stream, T value) { + if constexpr (std::same_as) { + stream.write_u8(static_cast(value)); + } else if constexpr (std::same_as) { + stream.write_size(value); + } else if constexpr (std::is_unsigned_v && sizeof(T) == 1) { + stream.write_u8(static_cast(value)); + } else if constexpr (std::is_unsigned_v && sizeof(T) == 2) { + stream.write_u16(static_cast(value)); + } else if constexpr (std::is_unsigned_v && sizeof(T) == 4) { + stream.write_u32(static_cast(value)); + } else if constexpr (std::is_unsigned_v && sizeof(T) == 8) { + stream.write_u64(static_cast(value)); + } else if constexpr (std::is_signed_v && sizeof(T) == 1) { + stream.write_i8(static_cast(value)); + } else if constexpr (std::is_signed_v && sizeof(T) == 2) { + stream.write_i16(static_cast(value)); + } else if constexpr (std::is_signed_v && sizeof(T) == 4) { + stream.write_i32(static_cast(value)); + } else if constexpr (std::is_signed_v && sizeof(T) == 8) { + stream.write_i64(static_cast(value)); + } else { + static_assert(sizeof(T) == 0, "Unsupported serialized integer type"); + } +} + +template +T read_integral(BinaryReader& reader) { + if constexpr (std::same_as) { + const std::uint8_t value = reader.read_u8(); + if (value > 1) { + throw SerializationError("Invalid serialized boolean value", + reader.byte_offset() - 1); + } + return value != 0; + } else if constexpr (std::same_as) { + return reader.read_size(); + } else if constexpr (std::is_unsigned_v && sizeof(T) == 1) { + return static_cast(reader.read_u8()); + } else if constexpr (std::is_unsigned_v && sizeof(T) == 2) { + return static_cast(reader.read_u16()); + } else if constexpr (std::is_unsigned_v && sizeof(T) == 4) { + return static_cast(reader.read_u32()); + } else if constexpr (std::is_unsigned_v && sizeof(T) == 8) { + return static_cast(reader.read_u64()); + } else if constexpr (std::is_signed_v && sizeof(T) == 1) { + return static_cast(reader.read_i8()); + } else if constexpr (std::is_signed_v && sizeof(T) == 2) { + return static_cast(reader.read_i16()); + } else if constexpr (std::is_signed_v && sizeof(T) == 4) { + return static_cast(reader.read_i32()); + } else if constexpr (std::is_signed_v && sizeof(T) == 8) { + return static_cast(reader.read_i64()); + } else { + static_assert(sizeof(T) == 0, "Unsupported serialized integer type"); + } +} + +template +inline constexpr std::size_t kSerializedIntegralBytes = + std::same_as ? sizeof(std::uint64_t) : sizeof(T); + +template +void write_vector(BinaryWriter& stream, std::span values) { + stream.write_size(values.size()); + for (const T value : values) { + write_integral(stream, value); + } +} + +template +std::vector read_vector(BinaryReader& reader) { + const std::size_t count = reader.read_size(); + const std::vector empty; + if (count > empty.max_size()) { + throw std::length_error("Serialized vector is too large"); + } + if (count > reader.remaining() / kSerializedIntegralBytes) { + throw SerializationError("Truncated serialized vector", + reader.byte_offset()); + } + std::vector result; + result.reserve(count); + for (std::size_t i = 0; i < count; ++i) { + result.push_back(read_integral(reader)); + } + return result; +} + +inline std::size_t checked_artifact_size(std::uint64_t encoded_size, + std::size_t header_size, + std::size_t available_size) { + if (encoded_size > std::numeric_limits::max()) { + throw std::length_error("Serialized artifact size does not fit in size_t"); + } + const std::size_t size = static_cast(encoded_size); + if (size < header_size || size > available_size) { + throw std::invalid_argument("Truncated serialized artifact"); + } + if (size % sizeof(std::uint64_t) != 0) { + throw std::invalid_argument("Serialized artifact is not word padded"); + } + return size; +} + +} // namespace pixie::detail diff --git a/include/pixie/io/file_output_sink.h b/include/pixie/io/file_output_sink.h new file mode 100644 index 0000000..7a08606 --- /dev/null +++ b/include/pixie/io/file_output_sink.h @@ -0,0 +1,178 @@ +#pragma once + +#if !defined(__unix__) && !defined(__APPLE__) +#error "pixie/io/file_output_sink.h requires POSIX file support" +#endif + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pixie::io { + +/** + * @brief Move-only seekable sink for a newly created or truncated POSIX file. + * + * @details Sequential writes use `write()`, while backpatches use `pwrite()` + * without changing the append position. `finish()` closes the descriptor and + * reports close errors, but does not provide durable `fsync()` semantics. + */ +class FileOutputSink { + public: + /** + * @brief Open @p path for binary output, creating or truncating it. + * @throws std::system_error if the file cannot be opened. + */ + explicit FileOutputSink(const std::filesystem::path& path) { + descriptor_ = ::open(path.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0666); + if (descriptor_ == -1) { + throw std::system_error(errno, std::generic_category(), + "Failed to open binary output file"); + } + } + + FileOutputSink(const FileOutputSink&) = delete; + FileOutputSink& operator=(const FileOutputSink&) = delete; + + /** @brief Transfer ownership of an open output file. */ + FileOutputSink(FileOutputSink&& other) noexcept + : descriptor_(std::exchange(other.descriptor_, -1)), + size_bytes_(std::exchange(other.size_bytes_, 0)) {} + + /** @brief Close the current file and transfer ownership from @p other. */ + FileOutputSink& operator=(FileOutputSink&& other) noexcept { + if (this != &other) { + close_noexcept(); + descriptor_ = std::exchange(other.descriptor_, -1); + size_bytes_ = std::exchange(other.size_bytes_, 0); + } + return *this; + } + + /** @brief Close the output file without reporting errors. */ + ~FileOutputSink() { close_noexcept(); } + + /** @brief Return the number of bytes appended to the output. */ + std::size_t size_bytes() const noexcept { return size_bytes_; } + + /** + * @brief Append all @p bytes to the file. + * @throws std::system_error for an operating-system write failure. + * @throws std::length_error if the resulting file offset is unsupported. + * @throws std::logic_error if the sink is already finished. + */ + void write(std::span bytes) { + require_open(); + require_file_range(size_bytes_, bytes.size()); + while (!bytes.empty()) { + const std::size_t chunk = std::min( + bytes.size(), + static_cast(std::numeric_limits::max())); + const ssize_t written = ::write(descriptor_, bytes.data(), chunk); + if (written == -1) { + if (errno == EINTR) { + continue; + } + throw std::system_error(errno, std::generic_category(), + "Failed to write binary output file"); + } + if (written == 0) { + throw std::system_error(EIO, std::generic_category(), + "Binary output file write made no progress"); + } + const std::size_t count = static_cast(written); + size_bytes_ += count; + bytes = bytes.subspan(count); + } + } + + /** + * @brief Replace already-written bytes beginning at @p position. + * @throws std::out_of_range if the range is outside the written file. + * @throws std::system_error for an operating-system write failure. + * @throws std::length_error if the file offset is unsupported. + * @throws std::logic_error if the sink is already finished. + */ + void write_at(std::size_t position, std::span bytes) { + require_open(); + if (position > size_bytes_ || bytes.size() > size_bytes_ - position) { + throw std::out_of_range("Binary output patch is outside the file"); + } + require_file_range(position, bytes.size()); + while (!bytes.empty()) { + const std::size_t chunk = std::min( + bytes.size(), + static_cast(std::numeric_limits::max())); + const off_t offset = static_cast(position); + const ssize_t written = + ::pwrite(descriptor_, bytes.data(), chunk, offset); + if (written == -1) { + if (errno == EINTR) { + continue; + } + throw std::system_error(errno, std::generic_category(), + "Failed to patch binary output file"); + } + if (written == 0) { + throw std::system_error(EIO, std::generic_category(), + "Binary output file patch made no progress"); + } + const std::size_t count = static_cast(written); + position += count; + bytes = bytes.subspan(count); + } + } + + /** + * @brief Close the file and report any close error. + * @details This operation is idempotent after success. + * @throws std::system_error if closing the file fails. + */ + void finish() { + if (descriptor_ == -1) { + return; + } + const int descriptor = std::exchange(descriptor_, -1); + if (::close(descriptor) == -1) { + throw std::system_error(errno, std::generic_category(), + "Failed to close binary output file"); + } + } + + private: + static void require_file_range(std::size_t position, std::size_t count) { + constexpr auto kMaximumOffset = std::numeric_limits::max(); + const auto maximum = static_cast(kMaximumOffset); + if (position > maximum || count > maximum - position) { + throw std::length_error("Binary output file offset is too large"); + } + } + + void require_open() const { + if (descriptor_ == -1) { + throw std::logic_error("Binary output file is already finished"); + } + } + + void close_noexcept() noexcept { + if (descriptor_ != -1) { + const int descriptor = std::exchange(descriptor_, -1); + (void)::close(descriptor); + } + } + + int descriptor_ = -1; + std::size_t size_bytes_ = 0; +}; + +} // namespace pixie::io diff --git a/include/pixie/packed_bit_builder.h b/include/pixie/packed_bit_builder.h new file mode 100644 index 0000000..11b894c --- /dev/null +++ b/include/pixie/packed_bit_builder.h @@ -0,0 +1,91 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace pixie { + +/** + * @brief Builder for a packed LSB-first bit sequence. + * + * @details This type is intended for constructing succinct bit vectors, not + * for persistent serialization. `take_words()` transfers the packed words and + * resets the builder to an empty, reusable state. + */ +class PackedBitBuilder { + private: + std::size_t size_ = 0; + std::vector data_; + + public: + /** + * @brief Append one bit. + */ + void write_bit(bool bit) { + if (size_ % 64 == 0) { + data_.push_back(static_cast(bit)); + } else if (bit) { + data_.back() |= 1ull << (size_ % 64); + } + ++size_; + } + + /** + * @brief Append the low @p width bits of @p bits, least-significant bit + * first. + * @throws std::invalid_argument if @p width is greater than 64. + * @throws std::length_error if the resulting bit count is not representable. + */ + void write_bits(std::uint64_t bits, std::size_t width) { + if (width > 64) { + throw std::invalid_argument("Packed bit width is greater than 64"); + } + if (width == 0) { + return; + } + if (size_ > std::numeric_limits::max() - width) { + throw std::length_error("Packed bit sequence is too large"); + } + + const std::size_t offset = size_ % 64; + if (offset == 0) { + data_.push_back(width == 64 ? bits : bits & ((1ull << width) - 1)); + } else { + const std::size_t prefix = std::min(width, 64 - offset); + const std::uint64_t prefix_mask = + prefix == 64 ? ~std::uint64_t{0} : (1ull << prefix) - 1; + data_.back() |= (bits & prefix_mask) << offset; + if (prefix < width) { + data_.push_back(bits >> prefix); + } + } + size_ += width; + } + + /** @brief Return the number of appended bits. */ + std::size_t size_bits() const noexcept { return size_; } + + /** + * @brief Reserve storage for at least @p size_bits bits. + */ + void reserve_bits(std::size_t size_bits) { + const std::size_t words = + size_bits / 64 + static_cast(size_bits % 64 != 0); + data_.reserve(words); + } + + /** + * @brief Transfer the packed words and reset this builder. + */ + std::vector take_words() { + size_ = 0; + return std::exchange(data_, {}); + } +}; + +} // namespace pixie diff --git a/include/pixie/rank_select/support.h b/include/pixie/rank_select/support.h index e5ea30b..0cc5d5d 100644 --- a/include/pixie/rank_select/support.h +++ b/include/pixie/rank_select/support.h @@ -1,7 +1,7 @@ #pragma once -#include #include +#include #include #include #include @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -102,6 +103,222 @@ class RankSelectSupport static_cast(SelectSupport::kSelect0)) != 0; } + static MetadataStorage deserialize_metadata_storage(BinaryReader& reader) + requires(std::same_as || + std::same_as) + { + const std::size_t size = reader.read_size(); + const std::span bytes = reader.read_bytes(size); + if constexpr (std::same_as) { + if (reinterpret_cast(bytes.data()) % + alignof(std::uint64_t) != + 0) { + throw std::invalid_argument( + "Serialized rank/select storage is not word aligned"); + } + return ReadOnlyStorageView(bytes); + } else { + if (size % kAlignedStorageLineBytes != 0) { + throw std::invalid_argument( + "Serialized rank/select storage is not cache-line aligned"); + } + if (size > std::numeric_limits::max() / 8) { + throw std::length_error("Serialized rank/select storage is too large"); + } + AlignedStorage result(size * 8); + std::ranges::copy(bytes, result.writable_bytes().begin()); + return result; + } + } + + void validate_deserialized_state(DeserializationValidation validation) const { + const std::size_t required_words = + num_bits_ == 0 ? 0 : 1 + (num_bits_ - 1) / kWordSize; + if (required_words > bits_.size()) { + throw std::invalid_argument( + "RankSelectSupport source bit span is too small"); + } + if (required_words > std::numeric_limits::max() / kWordSize) { + throw std::length_error("RankSelectSupport padded size is too large"); + } + if (padded_size_ != required_words * kWordSize || max_rank_ > num_bits_) { + throw std::invalid_argument( + "Invalid serialized rank/select size metadata"); + } + + const auto support_value = static_cast(select_support_); + if (support_value > static_cast(SelectSupport::kBoth) || + select0_samples_reversed_) { + throw std::invalid_argument( + "Invalid serialized rank/select configuration"); + } + + std::size_t num_superblocks = + 8 + (padded_size_ == 0 ? 0 : (padded_size_ - 1) / kSuperBlockSize); + if (num_superblocks > std::numeric_limits::max() - 7) { + throw std::length_error( + "Serialized rank/select super-block count is too large"); + } + num_superblocks = ((num_superblocks + 7) / 8) * 8; + if (num_superblocks > std::numeric_limits::max() / + (kBlocksPerSuperBlock * sizeof(std::uint16_t))) { + throw std::length_error( + "Serialized rank/select basic-block count is too large"); + } + const std::size_t expected_super_bytes = + num_superblocks * sizeof(std::uint64_t); + const std::size_t expected_basic_bytes = + num_superblocks * kBlocksPerSuperBlock * sizeof(std::uint16_t); + if (super_block_rank_.size_bytes() != expected_super_bytes || + basic_block_rank_.size_bytes() != expected_basic_bytes || + select_samples_.size_bytes() % sizeof(std::uint64_t) != 0) { + throw std::invalid_argument( + "Invalid serialized rank/select storage sizes"); + } + + const std::size_t sample_count = + select_samples_.size_bytes() / sizeof(std::uint64_t); + const auto samples_fit = [sample_count](std::size_t begin, + std::size_t count) { + return begin <= sample_count && count <= sample_count - begin; + }; + if (!samples_fit(select1_sample_begin_, select1_sample_count_) || + !samples_fit(select0_sample_begin_, select0_sample_count_) || + (builds_select1(select_support_) != (select1_sample_count_ != 0)) || + (builds_select0(select_support_) != (select0_sample_count_ != 0))) { + throw std::invalid_argument( + "Invalid serialized rank/select sample metadata"); + } + for (std::size_t i = 0; i < 8; ++i) { + if (delta_super[i] != i * kSuperBlockSize) { + throw std::invalid_argument( + "Invalid serialized rank/select SIMD metadata"); + } + } + for (std::size_t i = 0; i < 32; ++i) { + if (delta_basic[i] != i * kBasicBlockSize) { + throw std::invalid_argument( + "Invalid serialized rank/select SIMD metadata"); + } + } + + if (validation == DeserializationValidation::kFull) { + validate_full_source_metadata(); + return; + } + + const auto samples = select_samples_.as_words64(); + const auto sample_values_fit = [samples, num_superblocks]( + std::size_t begin, std::size_t count) { + return std::ranges::all_of(samples.subspan(begin, count), + [num_superblocks](std::uint64_t sample) { + return sample < num_superblocks; + }); + }; + if (!sample_values_fit(select1_sample_begin_, select1_sample_count_) || + !sample_values_fit(select0_sample_begin_, select0_sample_count_)) { + throw std::invalid_argument( + "Serialized rank/select sample references an invalid super block"); + } + } + + void validate_full_source_metadata() const { + const auto super_blocks = super_block_rank_.as_words64(); + const auto basic_blocks = basic_block_rank_.as_words16(); + const auto samples = select_samples_.as_words64(); + std::size_t next_select1 = select1_sample_begin_; + std::size_t next_select0 = select0_sample_begin_; + const std::size_t select1_end = + select1_sample_begin_ + select1_sample_count_; + const std::size_t select0_end = + select0_sample_begin_ + select0_sample_count_; + + const auto check_initial_sample = [&](bool enabled, std::size_t& next, + std::size_t end) { + if (enabled) { + if (next == end || samples[next] != 0) { + throw std::invalid_argument("Invalid serialized rank/select samples"); + } + ++next; + } + }; + check_initial_sample(builds_select1(select_support_), next_select1, + select1_end); + check_initial_sample(builds_select0(select_support_), next_select0, + select0_end); + + std::uint64_t rank1 = 0; + std::uint64_t rank0 = 0; + std::uint64_t super_rank = 0; + std::uint64_t basic_rank = 0; + std::uint64_t select1_milestone = kSelectSampleFrequency; + std::uint64_t select0_milestone = kSelectSampleFrequency; + const std::size_t metadata_word_count = + basic_blocks.size() * kWordsPerBlock; + for (std::size_t word_index = 0; word_index < metadata_word_count; + ++word_index) { + const std::size_t bit_position = word_index * kWordSize; + if (bit_position % kSuperBlockSize == 0) { + super_rank += basic_rank; + if (super_blocks[bit_position / kSuperBlockSize] != super_rank) { + throw std::invalid_argument( + "Serialized rank/select super-block ranks disagree with source"); + } + basic_rank = 0; + } + if (bit_position % kBasicBlockSize == 0 && + basic_blocks[bit_position / kBasicBlockSize] != basic_rank) { + throw std::invalid_argument( + "Serialized rank/select basic-block ranks disagree with source"); + } + + if (word_index >= logical_word_count()) { + continue; + } + const std::uint64_t word = logical_word(word_index); + const std::size_t word_bits = logical_word_bits(word_index); + const std::uint64_t ones = std::popcount(word); + const std::uint64_t zeros = word_bits - ones; + if (builds_select1(select_support_) && + rank1 + ones >= select1_milestone) { + const std::size_t position = + word_index * kWordSize + + select_64(word, select1_milestone - rank1 - 1); + const std::uint64_t expected = position / kSuperBlockSize; + if (next_select1 == select1_end || samples[next_select1] != expected) { + throw std::invalid_argument( + "Serialized rank/select one samples disagree with source"); + } + ++next_select1; + select1_milestone += kSelectSampleFrequency; + } + if (builds_select0(select_support_) && + rank0 + zeros >= select0_milestone) { + const std::uint64_t zero_word = + ~word & first_bits_mask(static_cast(word_bits)); + const std::size_t position = + word_index * kWordSize + + select_64(zero_word, select0_milestone - rank0 - 1); + const std::uint64_t expected = position / kSuperBlockSize; + if (next_select0 == select0_end || samples[next_select0] != expected) { + throw std::invalid_argument( + "Serialized rank/select zero samples disagree with source"); + } + ++next_select0; + select0_milestone += kSelectSampleFrequency; + } + basic_rank += ones; + rank1 += ones; + rank0 += zeros; + } + + if (super_rank + basic_rank != max_rank_ || rank1 != max_rank_ || + next_select1 != select1_end || next_select0 != select0_end) { + throw std::invalid_argument( + "Serialized rank/select totals disagree with source"); + } + } + size_t logical_word_count() const { return (num_bits_ + kWordSize - 1) / kWordSize; } @@ -861,55 +1078,110 @@ class RankSelectSupport return select_in_words(pos * kWordsPerBlock, rank0, false); } - void serialize(pixie::OutputBitStream& bs) const { - bs << num_bits_ << padded_size_ << max_rank_ << select1_sample_begin_ - << select1_sample_count_ << select0_sample_begin_ - << select0_sample_count_ << static_cast(select_support_) - << static_cast(select0_samples_reversed_); + /** + * @brief Serialize rank/select metadata in canonical little-endian form. + * + * @details A zero-copy `ReadOnlyStorageView` deserializer also requires each + * embedded metadata payload to begin at an address aligned for 64-bit words. + */ + void serialize(BinaryWriter& writer) const { + writer.write_size(num_bits_); + writer.write_size(padded_size_); + writer.write_size(max_rank_); + writer.write_size(select1_sample_begin_); + writer.write_size(select1_sample_count_); + writer.write_size(select0_sample_begin_); + writer.write_size(select0_sample_count_); + writer.write_u32(static_cast(select_support_)); + writer.write_u32(static_cast(select0_samples_reversed_)); for (const uint64_t delta : delta_super) { - bs << delta; + writer.write_u64(delta); } for (const uint16_t delta : delta_basic) { - bs << delta; + writer.write_u16(delta); } - super_block_rank_.serialize(bs); - basic_block_rank_.serialize(bs); - select_samples_.serialize(bs); + super_block_rank_.serialize(writer); + basic_block_rank_.serialize(writer); + select_samples_.serialize(writer); } - static RankSelectSupport deserialize(std::span source_bits, - std::span& data) - requires std::same_as + /** + * @brief Restore serialized metadata over caller-owned source bits. + * + * @details `AlignedStorage` restores owning metadata. A + * `ReadOnlyStorageView` specialization retains views into the reader's + * backing bytes, which must outlive the result. In both cases @p source_bits + * remains non-owning and must outlive the result. The reader is unchanged on + * failure. @p validation selects structural quick validation or exact + * source-derived metadata validation. + * + * @param source_bits Non-owning packed source words retained by the result. + * @param reader Input cursor, advanced only after successful validation. + * @param validation Quick structural or full source-derived validation. + * + * @throws std::invalid_argument for truncated or inconsistent metadata. + * @throws std::length_error when an encoded size is not representable. + */ + static RankSelectSupport deserialize( + std::span source_bits, + BinaryReader& reader, + DeserializationValidation validation = DeserializationValidation::kQuick) + requires(std::same_as || + std::same_as) { + BinaryReader candidate = reader; RankSelectSupport result; result.source_storage_ = ReadOnlyStorageView(std::as_bytes(source_bits)); result.bits_ = result.source_storage_.as_words64(); - auto read = [&data](auto& value) { - constexpr size_t length = sizeof(value); - std::memcpy(&value, data.data(), length); - data = data.subspan(length); - }; - read(result.num_bits_); - read(result.padded_size_); - read(result.max_rank_); - read(result.select1_sample_begin_); - read(result.select1_sample_count_); - read(result.select0_sample_begin_); - read(result.select0_sample_count_); - uint32_t buf; - read(buf); - result.select_support_ = static_cast(buf); - read(buf); - result.select0_samples_reversed_ = static_cast(buf); + result.num_bits_ = candidate.read_size(); + result.padded_size_ = candidate.read_size(); + result.max_rank_ = candidate.read_size(); + result.select1_sample_begin_ = candidate.read_size(); + result.select1_sample_count_ = candidate.read_size(); + result.select0_sample_begin_ = candidate.read_size(); + result.select0_sample_count_ = candidate.read_size(); + const std::uint32_t support = candidate.read_u32(); + if (support > static_cast(SelectSupport::kBoth)) { + throw std::invalid_argument( + "Invalid serialized rank/select configuration"); + } + result.select_support_ = static_cast(support); + const std::uint32_t reversed = candidate.read_u32(); + if (reversed > 1) { + throw std::invalid_argument( + "Invalid serialized rank/select boolean value"); + } + result.select0_samples_reversed_ = reversed != 0; for (uint64_t& delta : result.delta_super) { - read(delta); + delta = candidate.read_u64(); } for (uint16_t& delta : result.delta_basic) { - read(delta); + delta = candidate.read_u16(); } - result.super_block_rank_ = ReadOnlyStorageView::deserialize(data); - result.basic_block_rank_ = ReadOnlyStorageView::deserialize(data); - result.select_samples_ = ReadOnlyStorageView::deserialize(data); + result.super_block_rank_ = deserialize_metadata_storage(candidate); + result.basic_block_rank_ = deserialize_metadata_storage(candidate); + result.select_samples_ = deserialize_metadata_storage(candidate); + result.validate_deserialized_state(validation); + reader = candidate; + return result; + } + + /** + * @brief Restore metadata from @p data and advance the byte span on success. + * @param source_bits Non-owning packed source words retained by the result. + * @param data Input bytes, advanced only after successful validation. + * @param validation Quick structural or full source-derived validation. + */ + static RankSelectSupport deserialize( + std::span source_bits, + std::span& data, + DeserializationValidation validation = DeserializationValidation::kQuick) + requires(std::same_as || + std::same_as) + { + BinaryReader reader(data); + RankSelectSupport result = deserialize(source_bits, reader, validation); + data = data.subspan(reader.position()); return result; } }; diff --git a/include/pixie/rmm/sdsl.h b/include/pixie/rmm/sdsl.h index 60637c7..385e40d 100644 --- a/include/pixie/rmm/sdsl.h +++ b/include/pixie/rmm/sdsl.h @@ -35,7 +35,7 @@ class SdslRmMTree : public RmMBase { SdslRmMTree(std::span words, std::size_t bit_count, - std::size_t _ = 0) { + std::size_t = 0) { size_ = bit_count; const std::size_t valid_words = (bit_count + 63) / 64; for (std::size_t i = 0; i < valid_words; ++i) { diff --git a/include/pixie/rmm/tree.h b/include/pixie/rmm/tree.h index e24dfe5..6daf6bd 100644 --- a/include/pixie/rmm/tree.h +++ b/include/pixie/rmm/tree.h @@ -1,6 +1,7 @@ #pragma once #include #include +#include #include #include @@ -36,6 +37,11 @@ namespace pixie { * and immutable for the lifetime of the tree. */ class RmMTree : public RmMBase { + static constexpr std::array kSerializationMagic = { + 'P', 'I', 'X', 'I', 'E', 'R', 'M', 'M'}; + static constexpr std::uint32_t kSerializationVersion = 1; + static constexpr std::size_t kSerializationHeaderBytes = 32; + // ------------ bitvector ------------ std::span bits; // LSB-first, externally owned size_t num_bits = 0; // number of bits @@ -116,6 +122,130 @@ class RmMTree : public RmMBase { build_from_words(words, bit_count, leaf_block_bits, max_overhead); } + /** + * @brief Serialize the complete owning tree metadata. + * + * @details The external source bits are not serialized. The resulting + * artifact is versioned, canonical little-endian, and padded to an + * eight-byte boundary. + */ + void serialize(BinaryWriter& writer) const { + validate_serialized_state(DeserializationValidation::kQuick); + + const std::size_t artifact_begin = writer.size_bytes(); + detail::write_magic(writer, kSerializationMagic); + writer.write_u32(kSerializationVersion); + writer.write_u8(detail::kLittleEndianMarker); + writer.write_u8(sizeof(std::uint64_t)); + writer.write_u16(0); + const std::size_t artifact_size_position = writer.write_u64_placeholder(); + writer.write_size(num_bits); + + writer.write_size(num_bits); + writer.write_size(block_bits); + writer.write_size(leaf_count); + writer.write_size(first_leaf_index); + detail::write_vector(writer, + std::span(segment_size_bits)); + detail::write_vector(writer, + std::span(node_total_excess)); + detail::write_vector(writer, + std::span(node_min_prefix_excess)); + detail::write_vector(writer, + std::span(node_max_prefix_excess)); + detail::write_vector(writer, + std::span(node_min_count)); + detail::write_vector(writer, + std::span(node_pattern10_count)); + detail::write_vector(writer, std::span(node_first_bit)); + detail::write_vector(writer, std::span(node_last_bit)); + + const std::size_t unpadded_size = writer.size_bytes() - artifact_begin; + writer.write_zeros( + (sizeof(std::uint64_t) - unpadded_size % sizeof(std::uint64_t)) % + sizeof(std::uint64_t)); + const std::size_t artifact_size = writer.size_bytes() - artifact_begin; + writer.patch_u64(artifact_size_position, + static_cast(artifact_size)); + } + + /** + * @brief Restore owning tree metadata over caller-owned source words. + * + * @details The result retains a non-owning view of @p words. The caller must + * keep those words alive and immutable for the result's lifetime. On + * success, @p reader advances past exactly one artifact; on failure it is + * unchanged. @p validation selects quick structural checks or exact + * source-derived metadata validation. + * + * @param words Non-owning packed source words retained by the result. + * @param reader Input cursor, advanced only after successful validation. + * @param validation Quick structural or full source-derived validation. + * + * @throws std::invalid_argument for malformed, incompatible, truncated, or + * structurally inconsistent metadata. + * @throws std::length_error when an encoded size is not representable. + */ + static RmMTree deserialize(std::span words, + BinaryReader& reader, + DeserializationValidation validation = + DeserializationValidation::kQuick) { + BinaryReader candidate = reader; + const std::size_t available_size = candidate.remaining(); + detail::require_magic(candidate, kSerializationMagic); + if (candidate.read_u32() != kSerializationVersion || + candidate.read_u8() != detail::kLittleEndianMarker || + candidate.read_u8() != sizeof(std::uint64_t) || + candidate.read_u16() != 0) { + throw std::invalid_argument("Incompatible serialized RmM artifact"); + } + const std::size_t artifact_size = detail::checked_artifact_size( + candidate.read_u64(), kSerializationHeaderBytes, available_size); + const std::size_t source_bit_count = candidate.read_size(); + + BinaryReader payload = + candidate.read_subreader(artifact_size - kSerializationHeaderBytes); + RmMTree result; + result.bits = words; + result.num_bits = payload.read_size(); + result.block_bits = payload.read_size(); + result.leaf_count = payload.read_size(); + result.first_leaf_index = payload.read_size(); + result.segment_size_bits = detail::read_vector(payload); + result.node_total_excess = detail::read_vector(payload); + result.node_min_prefix_excess = detail::read_vector(payload); + result.node_max_prefix_excess = detail::read_vector(payload); + result.node_min_count = detail::read_vector(payload); + result.node_pattern10_count = detail::read_vector(payload); + result.node_first_bit = detail::read_vector(payload); + result.node_last_bit = detail::read_vector(payload); + payload.require_zero_padding(sizeof(std::uint64_t) - 1); + + if (source_bit_count != result.num_bits) { + throw std::invalid_argument( + "Serialized RmM source bit count is inconsistent"); + } + result.validate_serialized_state(validation); + reader = candidate; + return result; + } + + /** + * @brief Restore one artifact from @p data and advance it on success. + * @param words Non-owning packed source words retained by the result. + * @param data Input bytes, advanced only after successful validation. + * @param validation Quick structural or full source-derived validation. + */ + static RmMTree deserialize(std::span words, + std::span& data, + DeserializationValidation validation = + DeserializationValidation::kQuick) { + BinaryReader reader(data); + RmMTree result = deserialize(words, reader, validation); + data = data.subspan(reader.position()); + return result; + } + size_t size_impl() const { return num_bits; } // --------- queries: rank/select/excess ---------- @@ -978,6 +1108,220 @@ class RmMTree : public RmMBase { } private: + void validate_serialized_state(DeserializationValidation validation) const { + const std::size_t required_words = + num_bits == 0 ? 0 : 1 + (num_bits - 1) / 64; + if (required_words > bits.size()) { + throw std::invalid_argument("RmM source word span is too small"); + } + if (block_bits == 0 || !std::has_single_bit(block_bits) || + block_bits > std::numeric_limits::max()) { + throw std::invalid_argument("Invalid serialized RmM block size"); + } + + const std::size_t expected_leaf_count = + num_bits == 0 ? 0 : 1 + (num_bits - 1) / block_bits; + if (leaf_count != expected_leaf_count || + leaf_count > std::bit_floor(std::numeric_limits::max())) { + throw std::invalid_argument("Invalid serialized RmM leaf count"); + } + const std::size_t expected_first_leaf = + std::bit_ceil(std::max(1, leaf_count)); + if (first_leaf_index != expected_first_leaf || + leaf_count > + std::numeric_limits::max() - first_leaf_index) { + throw std::invalid_argument("Invalid serialized RmM tree shape"); + } + + const std::size_t expected_node_count = num_bits == 0 + ? segment_size_bits.size() + : first_leaf_index + leaf_count; + const bool supported_empty_shape = + num_bits != 0 || expected_node_count == 0 || expected_node_count == 1; + if (!supported_empty_shape || + (num_bits != 0 && segment_size_bits.size() != expected_node_count) || + node_total_excess.size() != segment_size_bits.size() || + node_min_prefix_excess.size() != segment_size_bits.size() || + node_max_prefix_excess.size() != segment_size_bits.size() || + node_min_count.size() != segment_size_bits.size() || + node_pattern10_count.size() != segment_size_bits.size() || + node_first_bit.size() != segment_size_bits.size() || + node_last_bit.size() != segment_size_bits.size()) { + throw std::invalid_argument( + "Invalid serialized RmM metadata vector sizes"); + } + + const std::size_t node_count = segment_size_bits.size(); + const auto node_is_zero = [&](std::size_t node) { + return segment_size_bits[node] == 0 && node_total_excess[node] == 0 && + node_min_prefix_excess[node] == 0 && + node_max_prefix_excess[node] == 0 && node_min_count[node] == 0 && + node_pattern10_count[node] == 0 && node_first_bit[node] == 0 && + node_last_bit[node] == 0; + }; + if (node_count != 0 && !node_is_zero(0)) { + throw std::invalid_argument("Invalid serialized RmM sentinel metadata"); + } + if (num_bits == 0) { + if (node_count == 1 && !node_is_zero(0)) { + throw std::invalid_argument("Invalid serialized empty RmM metadata"); + } + return; + } + + const auto signed_magnitude = [](std::int64_t value) { + return value < 0 + ? static_cast(-(value + 1)) + std::uint64_t{1} + : static_cast(value); + }; + + for (std::size_t leaf = 0; leaf < leaf_count; ++leaf) { + const std::size_t node = first_leaf_index + leaf; + const std::size_t begin = leaf * block_bits; + const std::size_t expected_size = std::min(block_bits, num_bits - begin); + if (validation == DeserializationValidation::kFull) { + std::int64_t total = 0; + std::int64_t minimum = std::numeric_limits::max(); + std::int64_t maximum = std::numeric_limits::min(); + std::uint64_t minimum_count = 0; + std::uint64_t pattern10_count = 0; + std::uint8_t previous_bit = 0; + for (std::size_t position = begin; position < begin + expected_size; + ++position) { + const std::uint8_t current_bit = + static_cast(bit(position)); + if (position != begin && previous_bit == 1 && current_bit == 0) { + ++pattern10_count; + } + total += current_bit != 0 ? 1 : -1; + if (total < minimum) { + minimum = total; + minimum_count = 1; + } else if (total == minimum) { + ++minimum_count; + } + maximum = std::max(maximum, total); + previous_bit = current_bit; + } + if (segment_size_bits[node] != expected_size || + node_total_excess[node] != total || + node_min_prefix_excess[node] != minimum || + node_max_prefix_excess[node] != maximum || + node_min_count[node] != minimum_count || + node_pattern10_count[node] != pattern10_count || + node_first_bit[node] != bit(begin) || + node_last_bit[node] != previous_bit) { + throw std::invalid_argument( + "Serialized RmM leaf metadata disagrees with source"); + } + continue; + } + const std::int64_t total = node_total_excess[node]; + const std::int64_t minimum = node_min_prefix_excess[node]; + const std::int64_t maximum = node_max_prefix_excess[node]; + const std::int64_t expected_signed_size = + static_cast(expected_size); + if (segment_size_bits[node] != expected_size || + signed_magnitude(total) > expected_size || + ((expected_signed_size + total) & 1) != 0 || minimum > total || + maximum < total || minimum < -expected_signed_size || + maximum > expected_signed_size || node_min_count[node] == 0 || + node_min_count[node] > expected_size || + node_pattern10_count[node] >= expected_size || + node_first_bit[node] > 1 || node_last_bit[node] > 1) { + throw std::invalid_argument("Invalid serialized RmM leaf metadata"); + } + } + + for (std::size_t node = first_leaf_index; node-- > 1;) { + if (validation == DeserializationValidation::kQuick) { + const std::size_t size = segment_size_bits[node]; + const std::int64_t minimum = node_min_prefix_excess[node]; + const std::int64_t maximum = node_max_prefix_excess[node]; + if (size > num_bits || + signed_magnitude(node_total_excess[node]) > size || + (minimum < 0 && signed_magnitude(minimum) > size) || + (maximum > 0 && static_cast(maximum) > size) || + node_min_prefix_excess[node] > node_max_prefix_excess[node] || + node_min_count[node] > segment_size_bits[node] || + node_pattern10_count[node] > segment_size_bits[node] || + node_first_bit[node] > 1 || node_last_bit[node] > 1) { + throw std::invalid_argument( + "Invalid serialized RmM internal metadata"); + } + continue; + } + const std::size_t left = node << 1; + const std::size_t right = left | 1; + const bool has_left = left < node_count && segment_size_bits[left] != 0; + const bool has_right = + right < node_count && segment_size_bits[right] != 0; + if (!has_left && !has_right) { + if (!node_is_zero(node)) { + throw std::invalid_argument( + "Invalid serialized empty RmM internal node"); + } + continue; + } + + const std::size_t first = has_left ? left : right; + if (has_left != has_right) { + if (segment_size_bits[node] != segment_size_bits[first] || + node_total_excess[node] != node_total_excess[first] || + node_min_prefix_excess[node] != node_min_prefix_excess[first] || + node_max_prefix_excess[node] != node_max_prefix_excess[first] || + node_min_count[node] != node_min_count[first] || + node_pattern10_count[node] != node_pattern10_count[first] || + node_first_bit[node] != node_first_bit[first] || + node_last_bit[node] != node_last_bit[first]) { + throw std::invalid_argument( + "Invalid serialized unary RmM internal node"); + } + continue; + } + + const std::uint64_t expected_size = + static_cast(segment_size_bits[left]) + + segment_size_bits[right]; + const std::int64_t expected_total = + static_cast(node_total_excess[left]) + + node_total_excess[right]; + const std::int64_t right_min = + static_cast(node_total_excess[left]) + + node_min_prefix_excess[right]; + const std::int64_t right_max = + static_cast(node_total_excess[left]) + + node_max_prefix_excess[right]; + const std::int64_t expected_min = + std::min(node_min_prefix_excess[left], right_min); + const std::int64_t expected_max = + std::max(node_max_prefix_excess[left], right_max); + const std::uint64_t expected_min_count = + (node_min_prefix_excess[left] == expected_min ? node_min_count[left] + : 0) + + (right_min == expected_min ? node_min_count[right] : 0); + const std::uint64_t expected_pattern_count = + static_cast(node_pattern10_count[left]) + + node_pattern10_count[right] + + (node_last_bit[left] == 1 && node_first_bit[right] == 0 ? 1 : 0); + if (segment_size_bits[node] != expected_size || + node_total_excess[node] != expected_total || + node_min_prefix_excess[node] != expected_min || + node_max_prefix_excess[node] != expected_max || + node_min_count[node] != expected_min_count || + node_pattern10_count[node] != expected_pattern_count || + node_first_bit[node] != node_first_bit[left] || + node_last_bit[node] != node_last_bit[right]) { + throw std::invalid_argument( + "Invalid serialized binary RmM internal node"); + } + } + if (segment_size_bits[1] != num_bits) { + throw std::invalid_argument( + "Serialized RmM root does not cover the source"); + } + } + /** * @brief Count "10" occurrences inside a 64-bit slice of given logical * length @p length. diff --git a/include/pixie/rmq/cartesian_hybrid_btree.h b/include/pixie/rmq/cartesian_hybrid_btree.h index c63ddbe..b4c7897 100644 --- a/include/pixie/rmq/cartesian_hybrid_btree.h +++ b/include/pixie/rmq/cartesian_hybrid_btree.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -23,6 +24,13 @@ namespace pixie::rmq { +template +class CartesianHybridBTree; + namespace detail { /** @@ -58,6 +66,9 @@ template class HybridBTreePlusMinusOne { + template + friend class ::pixie::rmq::CartesianHybridBTree; + public: static_assert(std::is_unsigned_v, "HybridBTreePlusMinusOne index type must be unsigned"); @@ -354,6 +365,8 @@ class HybridBTreePlusMinusOne { } private: + friend class HybridBTreePlusMinusOne; + /** * @brief Prepend one BP bit while building the sequence right-to-left. */ @@ -439,6 +452,394 @@ class HybridBTreePlusMinusOne { static_assert(sizeof(Bp512Selector) == 64); + void serialize_metadata(BinaryWriter& writer) const { + if (depth_count_ != 0 && + (owned_rank_index_.has_value() || external_rank_index_ == nullptr)) { + throw std::invalid_argument( + "Depth RMQ serialization requires external rank support"); + } + if (depth_count_ != 0) { + validate_serialized_state(*external_rank_index_, + DeserializationValidation::kQuick); + } else { + validate_serialized_state(RankSelectSupport<>(), + DeserializationValidation::kQuick); + } + + writer.write_size(depth_count_); + writer.write_size(internal_selectors_.size()); + for (const Bp512Selector& selector : internal_selectors_) { + for (const std::uint64_t word : selector.bp_bits_) { + writer.write_u64(word); + } + } + pixie::detail::write_vector( + writer, std::span(internal_min_positions_)); + pixie::detail::write_vector( + writer, std::span(internal_min_depths_)); + + writer.write_size(high_child_metadata_.size()); + for (const HighChildMetadata& metadata : high_child_metadata_) { + writer.write_size(metadata.position_begin); + writer.write_size(metadata.position_end); + pixie::detail::write_integral(writer, metadata.min_position); + writer.write_i64(metadata.min_depth); + } + pixie::detail::write_vector( + writer, std::span(high_sparse_min_slots_)); + pixie::detail::write_vector( + writer, std::span(internal_level_offsets_)); + pixie::detail::write_vector( + writer, std::span(min_summary_level_offsets_)); + pixie::detail::write_vector( + writer, std::span(high_level_offsets_)); + pixie::detail::write_vector(writer, + std::span(level_sizes_)); + pixie::detail::write_vector( + writer, std::span(level_position_spans_)); + pixie::detail::write_vector(writer, + std::span(level_fanouts_)); + writer.write_size(high_level_begin_); + } + + static HybridBTreePlusMinusOne deserialize_metadata( + std::span bits, + const RankSelectSupport<>& rank_index, + BinaryReader& reader) { + HybridBTreePlusMinusOne result; + result.input_bits_ = bits; + result.depth_count_ = reader.read_size(); + result.external_rank_index_ = &rank_index; + + const std::size_t selector_count = reader.read_size(); + const std::vector empty_selectors; + if (selector_count > empty_selectors.max_size()) { + throw std::length_error( + "Serialized depth RMQ selector count is too large"); + } + if (selector_count > + reader.remaining() / (kSelectorWords * sizeof(std::uint64_t))) { + throw std::invalid_argument("Truncated serialized depth RMQ selectors"); + } + result.internal_selectors_.resize(selector_count); + for (Bp512Selector& selector : result.internal_selectors_) { + for (std::uint64_t& word : selector.bp_bits_) { + word = reader.read_u64(); + } + } + result.internal_min_positions_ = pixie::detail::read_vector(reader); + result.internal_min_depths_ = + pixie::detail::read_vector(reader); + + const std::size_t metadata_count = reader.read_size(); + const std::vector empty_metadata; + if (metadata_count > empty_metadata.max_size()) { + throw std::length_error( + "Serialized depth RMQ child metadata is too large"); + } + constexpr std::size_t kSerializedMetadataBytes = + 2 * sizeof(std::uint64_t) + sizeof(Index) + sizeof(std::int64_t); + if (metadata_count > reader.remaining() / kSerializedMetadataBytes) { + throw std::invalid_argument( + "Truncated serialized depth RMQ child metadata"); + } + result.high_child_metadata_.resize(metadata_count); + for (HighChildMetadata& metadata : result.high_child_metadata_) { + metadata.position_begin = reader.read_size(); + metadata.position_end = reader.read_size(); + metadata.min_position = pixie::detail::read_integral(reader); + metadata.min_depth = reader.read_i64(); + } + result.high_sparse_min_slots_ = + pixie::detail::read_vector(reader); + result.internal_level_offsets_ = + pixie::detail::read_vector(reader); + result.min_summary_level_offsets_ = + pixie::detail::read_vector(reader); + result.high_level_offsets_ = + pixie::detail::read_vector(reader); + result.level_sizes_ = pixie::detail::read_vector(reader); + result.level_position_spans_ = + pixie::detail::read_vector(reader); + result.level_fanouts_ = pixie::detail::read_vector(reader); + result.high_level_begin_ = reader.read_size(); + return result; + } + + void restore_external_sources( + std::span bits, + const RankSelectSupport<>& rank_index) noexcept { + input_bits_ = bits; + owned_rank_index_.reset(); + external_rank_index_ = &rank_index; + } + + void restore_empty_sources() noexcept { + input_bits_ = {}; + owned_rank_index_.reset(); + external_rank_index_ = nullptr; + } + + void validate_serialized_state(const RankSelectSupport<>& rank_index, + DeserializationValidation validation) const { + if (depth_count_ == 0) { + if (!input_bits_.empty() || owned_rank_index_.has_value() || + !internal_selectors_.empty() || !internal_min_positions_.empty() || + !internal_min_depths_.empty() || !high_child_metadata_.empty() || + !high_sparse_min_slots_.empty() || !internal_level_offsets_.empty() || + !min_summary_level_offsets_.empty() || !high_level_offsets_.empty() || + !level_sizes_.empty() || !level_position_spans_.empty() || + !level_fanouts_.empty() || + high_level_begin_ != std::numeric_limits::max()) { + throw std::invalid_argument( + "Invalid serialized empty depth RMQ metadata"); + } + return; + } + + const std::size_t delta_count = depth_count_ - 1; + const std::size_t required_words = + delta_count == 0 ? 0 : 1 + (delta_count - 1) / 64; + if (required_words > input_bits_.size() || + rank_index.size() < delta_count || owned_rank_index_.has_value()) { + throw std::invalid_argument( + "Invalid serialized depth RMQ source metadata"); + } + + const std::size_t level_count = level_sizes_.size(); + if (level_count == 0 || internal_level_offsets_.size() != level_count || + min_summary_level_offsets_.size() != level_count || + high_level_offsets_.size() != level_count || + level_position_spans_.size() != level_count || + level_fanouts_.size() != level_count) { + throw std::invalid_argument( + "Invalid serialized depth RMQ topology sizes"); + } + + const std::size_t expected_leaf_count = 1 + (depth_count_ - 1) / LeafSize; + if (level_sizes_[0] != expected_leaf_count || + level_position_spans_[0] != LeafSize || level_fanouts_[0] != 0 || + internal_level_offsets_[0] != 0 || + min_summary_level_offsets_[0] != npos || high_level_offsets_[0] != 0) { + throw std::invalid_argument("Invalid serialized depth RMQ leaf topology"); + } + + std::size_t current_count = expected_leaf_count; + std::size_t current_span = LeafSize; + std::size_t expected_internal_count = 0; + for (std::size_t level = 1; level < level_count; ++level) { + const std::size_t expected_fanout = + current_count > kHighLevelFanout * kHighLevelFanout + ? kMiddleFanout + : kHighLevelFanout; + current_count = ceil_div(current_count, expected_fanout); + current_span = saturating_product(current_span, expected_fanout); + if (level_fanouts_[level] != expected_fanout || + level_sizes_[level] != current_count || + level_position_spans_[level] != current_span || + internal_level_offsets_[level] != expected_internal_count) { + throw std::invalid_argument( + "Invalid serialized depth RMQ level topology"); + } + if (current_count > + std::numeric_limits::max() - expected_internal_count) { + throw std::length_error( + "Serialized depth RMQ internal count is too large"); + } + expected_internal_count += current_count; + } + if (current_count != 1 || + internal_selectors_.size() != expected_internal_count) { + throw std::invalid_argument( + "Invalid serialized depth RMQ selector topology"); + } + + const std::size_t root_level = level_count - 1; + const std::size_t expected_high_begin = + level_count == 1 + ? std::numeric_limits::max() + : (level_fanouts_[root_level] == kHighLevelFanout ? root_level + : level_count); + if (high_level_begin_ != expected_high_begin) { + throw std::invalid_argument( + "Invalid serialized depth RMQ high-level topology"); + } + + std::size_t high_node_count = 0; + std::size_t side_summary_count = 0; + for (std::size_t level = 1; level < level_count; ++level) { + const bool high_level = level >= expected_high_begin; + if (high_level_offsets_[level] != high_node_count) { + throw std::invalid_argument( + "Invalid serialized depth RMQ high-level offsets"); + } + if (high_level) { + high_node_count += level_sizes_[level]; + } + const bool embeds = + !high_level && level_fanouts_[level] <= kEmbeddedSummaryMaxEntries; + const std::size_t expected_summary_offset = + embeds ? npos : side_summary_count; + if (min_summary_level_offsets_[level] != expected_summary_offset) { + throw std::invalid_argument( + "Invalid serialized depth RMQ summary offsets"); + } + if (!embeds) { + side_summary_count += level_sizes_[level]; + } + } + if (internal_min_positions_.size() != side_summary_count || + internal_min_depths_.size() != side_summary_count || + high_node_count > + std::numeric_limits::max() / kHighLevelFanout || + high_child_metadata_.size() != high_node_count * kHighLevelFanout || + high_node_count > + std::numeric_limits::max() / kHighSparseSlotsPerNode || + high_sparse_min_slots_.size() != + high_node_count * kHighSparseSlotsPerNode) { + throw std::invalid_argument( + "Invalid serialized depth RMQ metadata counts"); + } + + for (const Index position : internal_min_positions_) { + if (position == invalid_index || + static_cast(position) >= depth_count_) { + throw std::invalid_argument( + "Invalid serialized depth RMQ minimum position"); + } + } + for (std::size_t level = expected_high_begin; level < level_count; + ++level) { + for (std::size_t node = 0; node < level_sizes_[level]; ++node) { + const std::size_t count = entry_count(level, node); + const std::size_t first_child = node * level_fanouts_[level]; + const std::size_t flat = high_level_offsets_[level] + node; + for (std::size_t slot = 0; slot < count; ++slot) { + const HighChildMetadata& metadata = + high_child_metadata_[flat * kHighLevelFanout + slot]; + const std::size_t child = first_child + slot; + const std::size_t expected_begin = + child * level_position_spans_[level - 1]; + const std::size_t expected_end = std::min( + depth_count_, expected_begin + level_position_spans_[level - 1]); + if (metadata.position_begin != expected_begin || + metadata.position_end != expected_end || + metadata.min_position == invalid_index || + metadata.min_position < expected_begin || + metadata.min_position >= expected_end) { + throw std::invalid_argument( + "Invalid serialized depth RMQ child metadata"); + } + } + } + } + if (validation == DeserializationValidation::kFull) { + validate_exact_metadata(); + } + } + + void validate_exact_metadata() const { + for (std::size_t level = 1; level < level_count(); ++level) { + for (std::size_t node = 0; node < level_sizes_[level]; ++node) { + const std::size_t count = entry_count(level, node); + const std::size_t first_child = node * level_fanouts_[level]; + std::array child_minima{}; + for (std::size_t slot = 0; slot < count; ++slot) { + child_minima[slot] = + subtree_min_candidate(level - 1, first_child + slot); + } + + Bp512Selector expected_selector; + expected_selector.build(count, + [&](std::size_t left, std::size_t right) { + return strictly_better_candidate( + child_minima[left], child_minima[right]); + }); + std::size_t best_slot = 0; + for (std::size_t slot = 1; slot < count; ++slot) { + if (strictly_better_candidate(child_minima[slot], + child_minima[best_slot])) { + best_slot = slot; + } + } + const DepthCandidate expected_minimum = child_minima[best_slot]; + if (level_embeds_min_summary(level)) { + expected_selector.set_embedded_min_summary(expected_minimum.position, + expected_minimum.depth); + } else { + const std::size_t flat = min_summary_flat_index(level, node); + if (internal_min_positions_[flat] != expected_minimum.position || + internal_min_depths_[flat] != expected_minimum.depth) { + throw std::invalid_argument( + "Serialized depth RMQ minimum disagrees with source"); + } + } + if (selector_at(level, node).bp_bits_ != expected_selector.bp_bits_) { + throw std::invalid_argument( + "Serialized depth RMQ selector disagrees with source"); + } + + if (!is_high_level(level)) { + continue; + } + const std::size_t high_flat = high_flat_index(level, node); + for (std::size_t slot = 0; slot < kHighLevelFanout; ++slot) { + HighChildMetadata expected; + if (slot < count) { + const std::size_t child = first_child + slot; + expected.position_begin = node_position_begin(level - 1, child); + expected.position_end = node_position_end(level - 1, child); + expected.min_position = + static_cast(child_minima[slot].position); + expected.min_depth = child_minima[slot].depth; + } + const HighChildMetadata& actual = + high_child_metadata_at(high_flat, slot); + if (actual.position_begin != expected.position_begin || + actual.position_end != expected.position_end || + actual.min_position != expected.min_position || + actual.min_depth != expected.min_depth) { + throw std::invalid_argument( + "Serialized depth RMQ child metadata disagrees with source"); + } + } + + std::array expected_sparse{}; + for (std::size_t slot = 0; slot < count; ++slot) { + expected_sparse[slot] = static_cast(slot); + } + for (std::size_t sparse_level = 1; + sparse_level < kHighSparseTableLevels; ++sparse_level) { + const std::size_t span = std::size_t{1} << sparse_level; + if (span > count) { + break; + } + const std::size_t half_span = span >> 1; + const std::size_t previous = (sparse_level - 1) * kHighLevelFanout; + const std::size_t current = sparse_level * kHighLevelFanout; + for (std::size_t slot = 0; slot + span <= count; ++slot) { + const std::size_t left = expected_sparse[previous + slot]; + const std::size_t right = + expected_sparse[previous + slot + half_span]; + expected_sparse[current + slot] = static_cast( + strictly_better_candidate(child_minima[right], + child_minima[left]) + ? right + : left); + } + } + const std::uint8_t* actual_sparse = + high_sparse_min_slots_begin(high_flat); + if (!std::ranges::equal( + expected_sparse, + std::span(actual_sparse, kHighSparseSlotsPerNode))) { + throw std::invalid_argument( + "Serialized depth RMQ sparse metadata disagrees with source"); + } + } + } + } + /** * @brief Return whether a stored position is one of the missing sentinels. */ @@ -1357,6 +1758,32 @@ class CartesianHybridBTree LeafSize, UseTopSparseOverlay>, T> { + private: + using BpDepthRmq = detail::HybridBTreePlusMinusOne; + + struct TopCandidate { + Index position = std::numeric_limits::max(); + }; + static_assert(sizeof(TopCandidate) == sizeof(Index)); + + struct LoadedState { + pixie::AlignedStorage bp_bits_; + std::size_t bp_bit_count_ = 0; + std::vector top_sparse_candidates_; + std::size_t top_block_size_ = 4096; + std::size_t top_block_count_ = 0; + std::size_t top_sparse_levels_ = 0; + std::optional> bp_index_; + BpDepthRmq bp_depth_rmq_; + }; + + struct LoadTag {}; + + static constexpr std::array kSerializationMagic = { + 'P', 'I', 'X', 'I', 'E', 'R', 'M', 'Q'}; + static constexpr std::uint32_t kSerializationVersion = 1; + static constexpr std::size_t kSerializationHeaderBytes = 48; + public: static_assert(std::is_unsigned_v, "CartesianHybridBTree index type must be unsigned"); @@ -1372,6 +1799,11 @@ class CartesianHybridBTree static constexpr std::size_t kMinTopSparseBlockSize = 4096; static constexpr std::size_t kMaxTopSparseBlocks = std::size_t{1} << 14; static constexpr bool kUseTopSparseOverlay = UseTopSparseOverlay; + static constexpr bool kSerializationSupported = + std::same_as && + std::same_as> && + std::same_as && LeafSize == 512 && + UseTopSparseOverlay; /** * @brief Construct an empty Cartesian-tree RMQ index. @@ -1390,6 +1822,172 @@ class CartesianHybridBTree build(); } + /** + * @brief Serialize the complete owning RMQ metadata. + * + * @details Available only for the exact default + * `CartesianHybridBTree` specialization. The external values + * are not serialized. The artifact is versioned, canonical little-endian, + * and padded to an eight-byte boundary. + */ + void serialize(BinaryWriter& writer) const + requires(kSerializationSupported) + { + validate_serialized_state(DeserializationValidation::kQuick); + + const std::size_t artifact_begin = writer.size_bytes(); + pixie::detail::write_magic(writer, kSerializationMagic); + writer.write_u32(kSerializationVersion); + writer.write_u8(pixie::detail::kLittleEndianMarker); + writer.write_u8(sizeof(std::uint64_t)); + writer.write_u8(sizeof(T)); + writer.write_u8(sizeof(Index)); + const std::size_t artifact_size_position = writer.write_u64_placeholder(); + writer.write_size(values_.size()); + writer.write_size(LeafSize); + writer.write_u32(static_cast(UseTopSparseOverlay)); + writer.write_u32(0); + + writer.write_size(bp_bit_count_); + writer.write_size(top_block_size_); + writer.write_size(top_block_count_); + writer.write_size(top_sparse_levels_); + bp_bits_.serialize(writer); + writer.write_size(top_sparse_candidates_.size()); + for (const TopCandidate candidate : top_sparse_candidates_) { + pixie::detail::write_integral(writer, candidate.position); + } + writer.write_u8(static_cast(bp_index_.has_value())); + if (bp_index_) { + bp_index_->serialize(writer); + } + bp_depth_rmq_.serialize_metadata(writer); + + const std::size_t unpadded_size = writer.size_bytes() - artifact_begin; + writer.write_zeros( + (sizeof(std::uint64_t) - unpadded_size % sizeof(std::uint64_t)) % + sizeof(std::uint64_t)); + const std::size_t artifact_size = writer.size_bytes() - artifact_begin; + writer.patch_u64(artifact_size_position, + static_cast(artifact_size)); + } + + /** + * @brief Restore owning RMQ metadata over caller-owned values. + * + * @details Available only for the exact default + * `CartesianHybridBTree` specialization. The result retains a + * non-owning view of @p values, which must remain alive and immutable. On + * success, @p reader advances past exactly one artifact; on failure it is + * unchanged. @p validation selects quick structural checks or exact + * source-derived metadata validation. + * + * @param values Non-owning values retained by the result. + * @param reader Input cursor, advanced only after successful validation. + * @param validation Quick structural or full source-derived validation. + * + * @throws std::invalid_argument for malformed, incompatible, truncated, or + * structurally inconsistent metadata. + * @throws std::length_error when an encoded size is not representable. + */ + static Self deserialize( + std::span values, + BinaryReader& reader, + DeserializationValidation validation = DeserializationValidation::kQuick) + requires(kSerializationSupported) + { + BinaryReader candidate = reader; + const std::size_t available_size = candidate.remaining(); + pixie::detail::require_magic(candidate, kSerializationMagic); + if (candidate.read_u32() != kSerializationVersion || + candidate.read_u8() != pixie::detail::kLittleEndianMarker || + candidate.read_u8() != sizeof(std::uint64_t) || + candidate.read_u8() != sizeof(T) || + candidate.read_u8() != sizeof(Index)) { + throw std::invalid_argument("Incompatible serialized RMQ artifact"); + } + const std::size_t artifact_size = pixie::detail::checked_artifact_size( + candidate.read_u64(), kSerializationHeaderBytes, available_size); + const std::size_t source_value_count = candidate.read_size(); + if (candidate.read_size() != LeafSize || + candidate.read_u32() != + static_cast(UseTopSparseOverlay) || + candidate.read_u32() != 0) { + throw std::invalid_argument("Incompatible serialized RMQ configuration"); + } + + BinaryReader payload = + candidate.read_subreader(artifact_size - kSerializationHeaderBytes); + LoadedState state; + state.bp_bit_count_ = payload.read_size(); + state.top_block_size_ = payload.read_size(); + state.top_block_count_ = payload.read_size(); + state.top_sparse_levels_ = payload.read_size(); + state.bp_bits_ = deserialize_aligned_storage(payload); + + const std::size_t candidate_count = payload.read_size(); + const std::vector empty_candidates; + if (candidate_count > empty_candidates.max_size()) { + throw std::length_error( + "Serialized RMQ top candidate count is too large"); + } + if (candidate_count > payload.remaining() / sizeof(Index)) { + throw std::invalid_argument("Truncated serialized RMQ top candidates"); + } + state.top_sparse_candidates_.resize(candidate_count); + for (TopCandidate& candidate : state.top_sparse_candidates_) { + candidate.position = pixie::detail::read_integral(payload); + } + + const std::uint8_t has_rank_index = payload.read_u8(); + if (has_rank_index > 1) { + throw std::invalid_argument("Invalid serialized RMQ rank-index marker"); + } + const std::size_t bp_word_count = + state.bp_bit_count_ == 0 ? 0 : 1 + (state.bp_bit_count_ - 1) / 64; + if (bp_word_count > state.bp_bits_.as_words64().size()) { + throw std::invalid_argument("Serialized RMQ BP storage is too small"); + } + if (has_rank_index != 0) { + state.bp_index_ = RankSelectSupport<>::deserialize( + state.bp_bits_.as_words64().first(bp_word_count), payload, + validation); + } + + const RankSelectSupport<> empty_rank_index; + const RankSelectSupport<>& rank_index = + state.bp_index_ ? *state.bp_index_ : empty_rank_index; + state.bp_depth_rmq_ = BpDepthRmq::deserialize_metadata( + state.bp_bits_.as_words64(), rank_index, payload); + payload.require_zero_padding(sizeof(std::uint64_t) - 1); + + if (source_value_count != values.size()) { + throw std::invalid_argument( + "Serialized RMQ source value count is inconsistent"); + } + validate_loaded_state(values, state, validation); + reader = candidate; + return Self(LoadTag{}, values, std::move(state)); + } + + /** + * @brief Restore one artifact from @p data and advance it on success. + * @param values Non-owning values retained by the result. + * @param data Input bytes, advanced only after successful validation. + * @param validation Quick structural or full source-derived validation. + */ + static Self deserialize( + std::span values, + std::span& data, + DeserializationValidation validation = DeserializationValidation::kQuick) + requires(kSerializationSupported) + { + BinaryReader reader(data); + Self result = deserialize(values, reader, validation); + data = data.subspan(reader.position()); + return result; + } + /** * @brief Copy an RMQ index and rebuild internal non-owning views. */ @@ -1556,12 +2154,231 @@ class CartesianHybridBTree } private: - using BpDepthRmq = detail::HybridBTreePlusMinusOne; + static pixie::AlignedStorage deserialize_aligned_storage( + BinaryReader& reader) { + const std::size_t size = reader.read_size(); + const std::span bytes = reader.read_bytes(size); + if (size % kAlignedStorageLineBytes != 0) { + throw std::invalid_argument( + "Serialized RMQ storage is not cache-line aligned"); + } + if (size > std::numeric_limits::max() / 8) { + throw std::length_error("Serialized RMQ storage is too large"); + } + pixie::AlignedStorage result(size * 8); + std::ranges::copy(bytes, result.writable_bytes().begin()); + return result; + } - struct TopCandidate { - Index position = invalid_index; - }; - static_assert(sizeof(TopCandidate) == sizeof(Index)); + template + static void validate_loaded_state(std::span values, + const State& state, + DeserializationValidation validation) { + if (values.size() > (static_cast(invalid_index) - 1) / 2) { + throw std::length_error("Serialized RMQ value count is too large"); + } + const std::size_t expected_bp_bit_count = 2 * values.size(); + if (state.bp_bit_count_ != expected_bp_bit_count) { + throw std::invalid_argument("Invalid serialized RMQ BP bit count"); + } + + std::size_t expected_padded_bits = 0; + if (expected_bp_bit_count != 0) { + const std::size_t depth_count = expected_bp_bit_count + 1; + if (depth_count > std::numeric_limits::max()) { + throw std::length_error("Serialized RMQ BP depth count is too large"); + } + const std::size_t leaf_count = 1 + (depth_count - 1) / LeafSize; + if (leaf_count > std::numeric_limits::max() / LeafSize) { + throw std::length_error("Serialized RMQ padded BP size is too large"); + } + expected_padded_bits = leaf_count * LeafSize; + } + if (state.bp_bits_.size_bits() != expected_padded_bits) { + throw std::invalid_argument("Invalid serialized RMQ BP storage size"); + } + + if (values.empty()) { + if (state.top_block_size_ != kMinTopSparseBlockSize || + state.top_block_count_ != 0 || state.top_sparse_levels_ != 0 || + !state.top_sparse_candidates_.empty() || + state.bp_index_.has_value() || !state.bp_depth_rmq_.empty()) { + throw std::invalid_argument("Invalid serialized empty RMQ metadata"); + } + const RankSelectSupport<> empty_rank; + state.bp_depth_rmq_.validate_serialized_state(empty_rank, validation); + return; + } + + const std::size_t expected_block_size = + top_sparse_block_size_for(values.size()); + const std::size_t expected_block_count = + top_sparse_block_count_for(values.size()); + const std::size_t expected_levels = std::bit_width(expected_block_count); + if (state.top_block_size_ != expected_block_size || + state.top_block_count_ != expected_block_count || + state.top_sparse_levels_ != expected_levels || + expected_block_count > + std::numeric_limits::max() / expected_levels || + state.top_sparse_candidates_.size() != + expected_block_count * expected_levels || + !state.bp_index_.has_value() || + state.bp_index_->size() != expected_bp_bit_count || + state.bp_index_->supports_select1() || + !state.bp_index_->supports_select0() || + state.bp_depth_rmq_.size() != expected_bp_bit_count + 1) { + throw std::invalid_argument("Invalid serialized RMQ index metadata"); + } + + for (std::size_t level = 0; + validation == DeserializationValidation::kQuick && + level < expected_levels; + ++level) { + const std::size_t span = std::size_t{1} << level; + for (std::size_t block = 0; block < expected_block_count; ++block) { + const std::size_t position = static_cast( + state.top_sparse_candidates_[level * expected_block_count + block] + .position); + const bool populated = block + span <= expected_block_count; + if (!populated) { + if (position != static_cast(invalid_index)) { + throw std::invalid_argument( + "Invalid serialized RMQ sparse-table padding"); + } + continue; + } + const std::size_t begin = block * expected_block_size; + const std::size_t end = + std::min(values.size(), (block + span) * expected_block_size); + if (position < begin || position >= end) { + throw std::invalid_argument( + "Invalid serialized RMQ sparse-table candidate"); + } + } + } + state.bp_depth_rmq_.validate_serialized_state(*state.bp_index_, validation); + if (validation == DeserializationValidation::kFull) { + validate_exact_source(values, state, expected_padded_bits); + } + } + + template + static void validate_exact_source(std::span values, + const State& state, + std::size_t expected_padded_bits) { + const std::span bp_words = state.bp_bits_.as_words64(); + const auto bp_bit = [bp_words](std::size_t position) { + return ((bp_words[position >> 6] >> (position & 63)) & 1u) != 0; + }; + const auto require_bp_bit = [&](std::size_t position, bool expected) { + if (bp_bit(position) != expected) { + throw std::invalid_argument( + "Serialized RMQ Cartesian BP disagrees with source values"); + } + }; + const auto better_position = [&](std::size_t left, std::size_t right) { + if (right == static_cast(invalid_index)) { + return left; + } + if (left == static_cast(invalid_index)) { + return right; + } + if (std::less{}(values[right], values[left])) { + return right; + } + if (std::less{}(values[left], values[right])) { + return left; + } + return std::min(left, right); + }; + + utils::SuccinctIncreasingStack stack(values.size()); + std::size_t write_position = 2 * values.size(); + std::size_t block_minimum = static_cast(invalid_index); + for (std::size_t position = values.size(); position-- > 0;) { + while (!stack.empty()) { + const std::size_t top_position = values.size() - 1 - stack.top(); + if (std::less{}(values[top_position], values[position])) { + break; + } + stack.pop(); + require_bp_bit(--write_position, true); + } + stack.push(values.size() - 1 - position); + require_bp_bit(--write_position, false); + + block_minimum = better_position(block_minimum, position); + if (position % state.top_block_size_ == 0) { + const std::size_t block = position / state.top_block_size_; + if (static_cast( + state.top_sparse_candidates_[block].position) != + block_minimum) { + throw std::invalid_argument( + "Serialized RMQ block minimum disagrees with source values"); + } + block_minimum = static_cast(invalid_index); + } + } + while (write_position != 0) { + require_bp_bit(--write_position, true); + } + + for (std::size_t position = 2 * values.size(); + position < expected_padded_bits; ++position) { + if (bp_bit(position)) { + throw std::invalid_argument("Serialized RMQ BP padding is non-zero"); + } + } + + for (std::size_t level = 1; level < state.top_sparse_levels_; ++level) { + const std::size_t span = std::size_t{1} << level; + const std::size_t half_span = span >> 1; + const std::size_t current_offset = level * state.top_block_count_; + const std::size_t previous_offset = (level - 1) * state.top_block_count_; + for (std::size_t block = 0; block < state.top_block_count_; ++block) { + const std::size_t actual = static_cast( + state.top_sparse_candidates_[current_offset + block].position); + if (block + span > state.top_block_count_) { + if (actual != static_cast(invalid_index)) { + throw std::invalid_argument( + "Invalid serialized RMQ sparse-table padding"); + } + continue; + } + const std::size_t left = static_cast( + state.top_sparse_candidates_[previous_offset + block].position); + const std::size_t right = static_cast( + state.top_sparse_candidates_[previous_offset + block + half_span] + .position); + if (actual != better_position(left, right)) { + throw std::invalid_argument( + "Serialized RMQ sparse-table candidate disagrees with source"); + } + } + } + } + + void validate_serialized_state(DeserializationValidation validation) const { + validate_loaded_state(values_, *this, validation); + } + + CartesianHybridBTree(LoadTag, std::span values, LoadedState&& state) + : values_(values), + compare_(), + bp_bits_(std::move(state.bp_bits_)), + bp_bit_count_(state.bp_bit_count_), + top_sparse_candidates_(std::move(state.top_sparse_candidates_)), + top_block_size_(state.top_block_size_), + top_block_count_(state.top_block_count_), + top_sparse_levels_(state.top_sparse_levels_), + bp_index_(std::move(state.bp_index_)), + bp_depth_rmq_(std::move(state.bp_depth_rmq_)) { + if (bp_index_) { + bp_depth_rmq_.restore_external_sources(bp_bits_.as_words64(), *bp_index_); + } else { + bp_depth_rmq_.restore_empty_sources(); + } + } /** * @brief Return the first minimum position through the Cartesian BP diff --git a/include/pixie/serialization.h b/include/pixie/serialization.h new file mode 100644 index 0000000..b16532d --- /dev/null +++ b/include/pixie/serialization.h @@ -0,0 +1,680 @@ +#pragma once + +/** + * @file serialization.h + * @brief Checked byte-oriented binary serialization primitives. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pixie { + +/** + * @brief Validation strength used while restoring serialized indexes. + */ +enum class DeserializationValidation : std::uint8_t { + /** + * @brief Check framing, dimensions, references, and other conditions needed + * for memory-safe terminating queries without scanning large sources. + */ + kQuick, + + /** + * @brief Additionally authenticate all source-derived metadata against the + * supplied source contents. + */ + kFull, +}; + +/** + * @brief Error raised while decoding malformed serialized data. + */ +class SerializationError : public std::invalid_argument { + public: + /** + * @brief Construct an error for @p byte_offset in the input artifact. + */ + SerializationError(std::string message, std::size_t byte_offset) + : std::invalid_argument(std::move(message) + " at byte " + + std::to_string(byte_offset)), + byte_offset_(byte_offset) {} + + /** @brief Return the byte offset associated with the error. */ + std::size_t byte_offset() const noexcept { return byte_offset_; } + + private: + std::size_t byte_offset_; +}; + +/** + * @brief Contract for an append-only output that supports checked overwrites. + * + * @details A sink starts empty. `write()` appends all bytes or throws; + * `write_at()` replaces bytes within the already-written prefix without + * changing the append position; and `finish()` completes the output. A sink + * may become unusable after `finish()`. + */ +template +concept SeekableBinaryOutputSink = requires(Sink& sink, + std::size_t position, + std::span bytes) { + { sink.write(bytes) } -> std::same_as; + { sink.write_at(position, bytes) } -> std::same_as; + { sink.finish() } -> std::same_as; +}; + +/** + * @brief Explicit owning sink for binary data collected in memory. + * + * @details This sink intentionally grows with the complete output. Prefer a + * fixed span or file sink when bounded additional memory is required. + */ +class VectorOutputSink { + public: + /** @brief Return whether no output bytes have been written. */ + bool empty() const noexcept { return data_.empty(); } + + /** @brief Return the number of output bytes written. */ + std::size_t size_bytes() const noexcept { return data_.size(); } + + /** @brief Return a view of all output bytes. */ + std::span bytes() const noexcept { return data_; } + + /** @brief Reserve memory for at least @p size_bytes output bytes. */ + void reserve_bytes(std::size_t size_bytes) { data_.reserve(size_bytes); } + + /** @brief Append @p bytes to the output. */ + void write(std::span bytes) { + if (bytes.size() > data_.max_size() - data_.size()) { + throw std::length_error("Binary output is too large"); + } + data_.insert(data_.end(), bytes.begin(), bytes.end()); + } + + /** + * @brief Replace output bytes beginning at @p position. + * @throws std::out_of_range if the range is outside the existing output. + */ + void write_at(std::size_t position, std::span bytes) { + if (position > data_.size() || bytes.size() > data_.size() - position) { + throw std::out_of_range("Binary output patch is outside the output"); + } + std::ranges::copy(bytes, data_.begin() + position); + } + + /** @brief Complete this in-memory output. */ + void finish() noexcept {} + + /** @brief Transfer the bytes and reset this sink to an empty state. */ + std::vector take() { return std::exchange(data_, {}); } + + private: + std::vector data_; +}; + +static_assert(SeekableBinaryOutputSink); + +/** + * @brief Non-owning fixed-capacity output sink. + * + * @details The caller keeps the writable span alive and stable until the + * associated writer is finished. + */ +class SpanOutputSink { + public: + /** @brief Construct an empty output over caller-owned storage. */ + explicit SpanOutputSink(std::span storage) : storage_(storage) {} + + /** @brief Return whether no output bytes have been written. */ + bool empty() const noexcept { return size_bytes_ == 0; } + + /** @brief Return the number of output bytes written. */ + std::size_t size_bytes() const noexcept { return size_bytes_; } + + /** @brief Return the fixed output capacity in bytes. */ + std::size_t capacity_bytes() const noexcept { return storage_.size(); } + + /** @brief Return the prefix written so far. */ + std::span bytes() const noexcept { + return storage_.first(size_bytes_); + } + + /** + * @brief Append @p bytes to the output. + * @throws std::length_error if the fixed output capacity is insufficient. + */ + void write(std::span bytes) { + if (bytes.size() > storage_.size() - size_bytes_) { + throw std::length_error("Fixed binary output capacity exceeded"); + } + std::ranges::copy(bytes, storage_.begin() + size_bytes_); + size_bytes_ += bytes.size(); + } + + /** + * @brief Replace output bytes beginning at @p position. + * @throws std::out_of_range if the range is outside the written prefix. + */ + void write_at(std::size_t position, std::span bytes) { + if (position > size_bytes_ || bytes.size() > size_bytes_ - position) { + throw std::out_of_range("Binary output patch is outside the output"); + } + std::ranges::copy(bytes, storage_.begin() + position); + } + + /** @brief Complete this fixed-span output. */ + void finish() noexcept {} + + private: + std::span storage_; + std::size_t size_bytes_ = 0; +}; + +static_assert(SeekableBinaryOutputSink); + +/** + * @brief Bounded-buffer writer for canonical little-endian binary data. + * + * @details The writer does not own its sink. Small fields are accumulated in + * a fixed-size staging buffer; large caller-owned byte spans may be sent + * directly to the sink. Integer methods encode a fixed number of bytes, + * independent of the host ABI. Call `finish()` to report output errors and + * deliver the final buffered bytes before inspecting or consuming the sink. + * Destruction does not perform I/O. + */ +class BinaryWriter { + public: + /** @brief Default bounded staging-buffer size. */ + static constexpr std::size_t kDefaultBufferBytes = 64 * 1024; + + /** + * @brief Construct a writer with an owned, fixed-size staging buffer. + * + * @param sink Empty sink that remains alive and unmoved until `finish()`. + * @param buffer_bytes Maximum staging-buffer memory owned by the writer. + * @throws std::invalid_argument if @p buffer_bytes is zero. + */ + template + explicit BinaryWriter(Sink& sink, + std::size_t buffer_bytes = kDefaultBufferBytes) + : owned_buffer_(allocate_buffer(buffer_bytes)), + buffer_(owned_buffer_.get(), buffer_bytes) { + bind(sink); + } + + /** + * @brief Construct a writer over a caller-owned staging buffer. + * + * @param sink Empty sink that remains alive and unmoved until `finish()`. + * @param buffer Writable staging memory that remains alive and unmoved until + * `finish()`. + * @throws std::invalid_argument if @p buffer is empty. + */ + template + BinaryWriter(Sink& sink, std::span buffer) : buffer_(buffer) { + if (buffer.empty()) { + throw std::invalid_argument( + "BinaryWriter staging buffer must be non-empty"); + } + bind(sink); + } + + BinaryWriter(const BinaryWriter&) = delete; + BinaryWriter& operator=(const BinaryWriter&) = delete; + BinaryWriter(BinaryWriter&&) = delete; + BinaryWriter& operator=(BinaryWriter&&) = delete; + + /** @brief Return whether no bytes have been written. */ + bool empty() const noexcept { return size_bytes() == 0; } + + /** @brief Return the logical number of bytes written. */ + std::size_t size_bytes() const noexcept { + return flushed_bytes_ + buffered_bytes_; + } + + /** @brief Return the maximum staging-buffer size. */ + std::size_t buffer_size_bytes() const noexcept { return buffer_.size(); } + + /** @brief Write an unsigned eight-bit integer. */ + void write_u8(std::uint8_t value) { write_unsigned(value); } + + /** @brief Write an unsigned 16-bit integer in little-endian order. */ + void write_u16(std::uint16_t value) { write_unsigned(value); } + + /** @brief Write an unsigned 32-bit integer in little-endian order. */ + void write_u32(std::uint32_t value) { write_unsigned(value); } + + /** @brief Write an unsigned 64-bit integer in little-endian order. */ + void write_u64(std::uint64_t value) { write_unsigned(value); } + + /** @brief Write a signed eight-bit integer. */ + void write_i8(std::int8_t value) { + write_unsigned(std::bit_cast(value)); + } + + /** @brief Write a signed 16-bit integer in little-endian order. */ + void write_i16(std::int16_t value) { + write_unsigned(std::bit_cast(value)); + } + + /** @brief Write a signed 32-bit integer in little-endian order. */ + void write_i32(std::int32_t value) { + write_unsigned(std::bit_cast(value)); + } + + /** @brief Write a signed 64-bit integer in little-endian order. */ + void write_i64(std::int64_t value) { + write_unsigned(std::bit_cast(value)); + } + + /** + * @brief Write a platform size as an unsigned 64-bit integer. + * @throws std::length_error if `size_t` is wider than the wire type. + */ + void write_size(std::size_t value) { + if constexpr (sizeof(std::size_t) > sizeof(std::uint64_t)) { + if (value > std::numeric_limits::max()) { + throw std::length_error("Size does not fit in the wire format"); + } + } + write_u64(static_cast(value)); + } + + /** @brief Append @p bytes without interpretation. */ + void write_bytes(std::span bytes) { + require_open(); + require_output_size(bytes.size()); + while (!bytes.empty()) { + if (buffered_bytes_ != 0) { + const std::size_t count = + std::min(bytes.size(), buffer_.size() - buffered_bytes_); + std::ranges::copy(bytes.first(count), + buffer_.begin() + buffered_bytes_); + buffered_bytes_ += count; + bytes = bytes.subspan(count); + if (buffered_bytes_ == buffer_.size()) { + flush_buffer(); + } + continue; + } + + if (bytes.size() >= buffer_.size()) { + const std::size_t count = bytes.size(); + write_to_sink(bytes); + flushed_bytes_ += count; + return; + } + + std::ranges::copy(bytes, buffer_.begin()); + buffered_bytes_ = bytes.size(); + return; + } + } + + /** @brief Append @p count zero bytes. */ + void write_zeros(std::size_t count) { + require_open(); + require_output_size(count); + while (count != 0) { + const std::size_t writable = buffer_.size() - buffered_bytes_; + const std::size_t chunk = std::min(count, writable); + std::fill_n(buffer_.begin() + buffered_bytes_, chunk, std::byte{0}); + buffered_bytes_ += chunk; + count -= chunk; + if (buffered_bytes_ == buffer_.size()) { + flush_buffer(); + } + } + } + + /** + * @brief Pad with zero bytes to the next multiple of @p alignment. + * @throws std::invalid_argument if @p alignment is zero. + */ + void align_to(std::size_t alignment) { + require_open(); + if (alignment == 0) { + throw std::invalid_argument("Serialization alignment must be non-zero"); + } + const std::size_t remainder = size_bytes() % alignment; + if (remainder != 0) { + write_zeros(alignment - remainder); + } + } + + /** + * @brief Write a zero 64-bit field and return its byte position. + * + * @details Use `patch_u64()` to fill framed lengths after writing a payload. + */ + std::size_t write_u64_placeholder() { + const std::size_t position = size_bytes(); + write_u64(0); + return position; + } + + /** + * @brief Replace an existing 64-bit field with @p value. + * @throws std::out_of_range if the field is outside the current output. + */ + void patch_u64(std::size_t position, std::uint64_t value) { + require_open(); + if (position > size_bytes() || size_bytes() - position < sizeof(value)) { + throw std::out_of_range("Serialization patch is outside the output"); + } + + std::array encoded{}; + for (std::size_t byte = 0; byte < sizeof(value); ++byte) { + encoded[byte] = static_cast((value >> (byte * 8)) & 0xffu); + } + + if (position >= flushed_bytes_) { + const std::size_t buffered_position = position - flushed_bytes_; + std::ranges::copy(encoded, buffer_.begin() + buffered_position); + return; + } + + if (position + encoded.size() > flushed_bytes_) { + flush_buffer(); + } + write_at_sink(position, encoded); + } + + /** + * @brief Deliver currently buffered bytes while keeping the writer open. + * @throws Any exception reported by the sink. A failed writer is unusable. + */ + void flush() { + require_open(); + flush_buffer(); + } + + /** + * @brief Deliver all bytes and complete the sink. + * + * @details This operation is idempotent after success. A sink failure is + * propagated and leaves the writer unusable; already-written output is not + * rolled back. + */ + void finish() { + if (state_ == State::kFinished) { + return; + } + require_open(); + flush_buffer(); + try { + finish_sink_(sink_); + state_ = State::kFinished; + } catch (...) { + state_ = State::kFailed; + throw; + } + } + + private: + enum class State { kOpen, kFinished, kFailed }; + + using WriteSink = void (*)(void*, std::span); + using WriteAtSink = void (*)(void*, std::size_t, std::span); + using FinishSink = void (*)(void*); + + static std::unique_ptr allocate_buffer( + std::size_t buffer_bytes) { + if (buffer_bytes == 0) { + throw std::invalid_argument( + "BinaryWriter staging buffer must be non-empty"); + } + return std::make_unique(buffer_bytes); + } + + template + void bind(Sink& sink) noexcept { + sink_ = std::addressof(sink); + write_sink_ = [](void* output, std::span bytes) { + static_cast(output)->write(bytes); + }; + write_at_sink_ = [](void* output, std::size_t position, + std::span bytes) { + static_cast(output)->write_at(position, bytes); + }; + finish_sink_ = [](void* output) { static_cast(output)->finish(); }; + } + + void require_open() const { + if (state_ == State::kFinished) { + throw std::logic_error("BinaryWriter is already finished"); + } + if (state_ == State::kFailed) { + throw std::logic_error("BinaryWriter is unusable after sink failure"); + } + } + + void require_output_size(std::size_t count) const { + if (count > std::numeric_limits::max() - size_bytes()) { + throw std::length_error("Serialized output is too large"); + } + } + + void flush_buffer() { + if (buffered_bytes_ == 0) { + return; + } + const std::size_t count = buffered_bytes_; + write_to_sink(buffer_.first(count)); + flushed_bytes_ += count; + buffered_bytes_ = 0; + } + + void write_to_sink(std::span bytes) { + try { + write_sink_(sink_, bytes); + } catch (...) { + state_ = State::kFailed; + throw; + } + } + + void write_at_sink(std::size_t position, std::span bytes) { + try { + write_at_sink_(sink_, position, bytes); + } catch (...) { + state_ = State::kFailed; + throw; + } + } + + template + void write_unsigned(T value) { + std::array encoded{}; + for (std::size_t byte = 0; byte < sizeof(T); ++byte) { + encoded[byte] = static_cast((value >> (byte * 8)) & T{0xff}); + } + write_bytes(encoded); + } + + void* sink_ = nullptr; + WriteSink write_sink_ = nullptr; + WriteAtSink write_at_sink_ = nullptr; + FinishSink finish_sink_ = nullptr; + std::unique_ptr owned_buffer_; + std::span buffer_; + std::size_t flushed_bytes_ = 0; + std::size_t buffered_bytes_ = 0; + State state_ = State::kOpen; +}; + +/** + * @brief Bounds-checked reader for canonical little-endian binary data. + * + * @details The reader does not own its input. Copies have independent cursors, + * which allows a compound deserializer to parse into a temporary reader and + * assign it back only after validation succeeds. + */ +class BinaryReader { + public: + /** @brief Construct a reader over caller-owned @p data. */ + explicit BinaryReader(std::span data) : data_(data) {} + + /** @brief Return whether all input bytes have been consumed. */ + bool empty() const noexcept { return remaining() == 0; } + + /** @brief Return the total number of bytes in this reader. */ + std::size_t size_bytes() const noexcept { return data_.size(); } + + /** @brief Return the number of bytes consumed by this reader. */ + std::size_t position() const noexcept { return offset_; } + + /** @brief Return the current byte offset in the outermost input. */ + std::size_t byte_offset() const noexcept { return absolute_position(); } + + /** @brief Return the number of unconsumed bytes. */ + std::size_t remaining() const noexcept { return data_.size() - offset_; } + + /** @brief Return all currently unconsumed bytes. */ + std::span remaining_bytes() const noexcept { + return data_.subspan(offset_); + } + + /** @brief Read an unsigned eight-bit integer. */ + std::uint8_t read_u8() { return read_unsigned(); } + + /** @brief Read an unsigned little-endian 16-bit integer. */ + std::uint16_t read_u16() { return read_unsigned(); } + + /** @brief Read an unsigned little-endian 32-bit integer. */ + std::uint32_t read_u32() { return read_unsigned(); } + + /** @brief Read an unsigned little-endian 64-bit integer. */ + std::uint64_t read_u64() { return read_unsigned(); } + + /** @brief Read a signed eight-bit integer. */ + std::int8_t read_i8() { + return std::bit_cast(read_unsigned()); + } + + /** @brief Read a signed little-endian 16-bit integer. */ + std::int16_t read_i16() { + return std::bit_cast(read_unsigned()); + } + + /** @brief Read a signed little-endian 32-bit integer. */ + std::int32_t read_i32() { + return std::bit_cast(read_unsigned()); + } + + /** @brief Read a signed little-endian 64-bit integer. */ + std::int64_t read_i64() { + return std::bit_cast(read_unsigned()); + } + + /** + * @brief Read an unsigned 64-bit size and convert it to `size_t`. + * @throws std::length_error if the value is not representable. + */ + std::size_t read_size() { + BinaryReader candidate = *this; + const std::size_t field_offset = candidate.absolute_position(); + const std::uint64_t encoded = candidate.read_u64(); + if (encoded > std::numeric_limits::max()) { + throw std::length_error("Serialized size at byte " + + std::to_string(field_offset) + + " does not fit in size_t"); + } + *this = candidate; + return static_cast(encoded); + } + + /** + * @brief Read exactly @p count uninterpreted bytes. + * @throws SerializationError if the input is truncated. + */ + std::span read_bytes(std::size_t count) { + require(count); + const std::span result = data_.subspan(offset_, count); + offset_ += count; + return result; + } + + /** + * @brief Read a bounded region as an independent child reader. + * @throws SerializationError if the input is truncated. + */ + BinaryReader read_subreader(std::size_t count) { + const std::size_t origin = absolute_position(); + return BinaryReader(read_bytes(count), origin); + } + + /** + * @brief Advance by exactly @p count bytes. + * @throws SerializationError if the input is truncated. + */ + void skip(std::size_t count) { + require(count); + offset_ += count; + } + + /** + * @brief Consume at most @p maximum trailing zero-padding bytes. + * @throws SerializationError for excessive or non-zero trailing bytes. + */ + void require_zero_padding(std::size_t maximum) { + if (remaining() > maximum) { + throw SerializationError("Unexpected serialized payload bytes", + absolute_position()); + } + BinaryReader candidate = *this; + while (!candidate.empty()) { + const std::size_t byte_offset = candidate.absolute_position(); + if (candidate.read_u8() != 0) { + throw SerializationError("Non-zero serialized payload padding", + byte_offset); + } + } + *this = candidate; + } + + private: + BinaryReader(std::span data, std::size_t origin) + : data_(data), origin_(origin) {} + + std::size_t absolute_position() const noexcept { return origin_ + offset_; } + + void require(std::size_t count) const { + if (count > remaining()) { + throw SerializationError("Truncated serialized input", + absolute_position()); + } + } + + template + T read_unsigned() { + require(sizeof(T)); + T value = 0; + for (std::size_t byte = 0; byte < sizeof(T); ++byte) { + value |= + static_cast(std::to_integer(data_[offset_ + byte])) + << (byte * 8); + } + offset_ += sizeof(T); + return value; + } + + std::span data_; + std::size_t offset_ = 0; + std::size_t origin_ = 0; +}; + +} // namespace pixie diff --git a/include/pixie/storage.h b/include/pixie/storage.h index 0882ee6..ec17220 100644 --- a/include/pixie/storage.h +++ b/include/pixie/storage.h @@ -8,7 +8,7 @@ * types. */ -#include +#include #include #include @@ -69,12 +69,12 @@ class StorageBase { return impl().view_impl(offset_bytes, count_bytes); } - /** @brief Serialize the exposed byte sequence with a size prefix. */ - void serialize(OutputBitStream& stream) const { - stream << size_bytes(); - for (const std::byte byte : as_bytes()) { - stream << static_cast(byte); - } + /** + * @brief Serialize the exposed bytes with a 64-bit little-endian size prefix. + */ + void serialize(BinaryWriter& writer) const { + writer.write_size(size_bytes()); + writer.write_bytes(as_bytes()); } /** @brief Resize mutable storage to hold at least @p size_bits bits. */ diff --git a/include/pixie/storage/read_only_view.h b/include/pixie/storage/read_only_view.h index 8c2af70..7f9b169 100644 --- a/include/pixie/storage/read_only_view.h +++ b/include/pixie/storage/read_only_view.h @@ -2,9 +2,7 @@ #include -#include #include -#include namespace pixie { @@ -38,20 +36,33 @@ class ReadOnlyStorageView : public StorageBase { } /** - * @brief Deserialize a size-prefixed view and advance @p data. + * @brief Deserialize a size-prefixed view and advance @p reader. + * + * @details The returned view references the reader's backing byte sequence, + * which must remain alive and stable for the view's lifetime. The reader is + * unchanged on failure. + * * @throws std::invalid_argument if the size prefix or payload is truncated. + * @throws std::length_error if the encoded size is not representable. + */ + static ReadOnlyStorageView deserialize(BinaryReader& reader) { + BinaryReader candidate = reader; + const std::size_t size = candidate.read_size(); + ReadOnlyStorageView result(candidate.read_bytes(size)); + reader = candidate; + return result; + } + + /** + * @brief Deserialize from @p data and advance it past the storage payload. + * + * @details This compatibility overload has the same lifetime and failure + * behavior as the `BinaryReader` overload. */ static ReadOnlyStorageView deserialize(std::span& data) { - if (data.size() < sizeof(std::size_t)) { - throw std::invalid_argument("Truncated storage size prefix"); - } - std::size_t size = 0; - std::memcpy(&size, data.data(), sizeof(size)); - if (size > data.size() - sizeof(size)) { - throw std::invalid_argument("Truncated storage payload"); - } - ReadOnlyStorageView result(data.subspan(sizeof(size), size)); - data = data.subspan(sizeof(size) + size); + BinaryReader reader(data); + ReadOnlyStorageView result = deserialize(reader); + data = data.subspan(reader.position()); return result; } diff --git a/include/pixie/wavelet_tree/index.h b/include/pixie/wavelet_tree/index.h index 4c6a8fc..3f32f30 100644 --- a/include/pixie/wavelet_tree/index.h +++ b/include/pixie/wavelet_tree/index.h @@ -1,10 +1,14 @@ #pragma once +#include +#include #include #include -#include -#include +#include +#include +#include +#include #include #include #include @@ -19,13 +23,17 @@ class WaveletTreeIndex : public WaveletTreeBase> { private: using node_index_t = size_t; static constexpr node_index_t npos = std::numeric_limits::max(); + static constexpr std::array kSerializationMagic = { + 'P', 'X', 'W', 'A', 'V', 'E', 'T', '\0'}; + static constexpr std::uint32_t kSerializationVersion = 1; + static constexpr std::size_t kSerializationHeaderBytes = 24; struct PreWaveletNode { node_index_t parent = npos; node_index_t left_child = npos; node_index_t right_child = npos; uint64_t middle; - OutputBitStream stream; + PackedBitBuilder stream; explicit PreWaveletNode(uint64_t middle) : middle(middle) {} }; @@ -48,7 +56,7 @@ class WaveletTreeIndex : public WaveletTreeBase> { AlignedStorage result(data.size() * 64); auto view = result.writable_words64(); std::copy(data.begin(), data.end(), view.begin()); - return std::move(result); + return result; } WaveletNode() = default; @@ -58,34 +66,36 @@ class WaveletTreeIndex : public WaveletTreeBase> { : parent(node.parent), left_child(node.left_child), right_child(node.right_child), - middle(node.middle), - bit_vector_data(std::move(align(node.stream.extract()))), - data(bit_vector_data.as_words64(), node.stream.size()) {} + middle(node.middle) { + const std::size_t bit_count = node.stream.size_bits(); + bit_vector_data = align(node.stream.take_words()); + data = + RankSelectSupport(bit_vector_data.as_words64(), bit_count); + } - /** @brief Writes a node serialization to the bit stream */ - void serialize(pixie::OutputBitStream& bs) const { - bs << parent << left_child << right_child << middle; - bit_vector_data.serialize(bs); - data.serialize(bs); + /** @brief Write one node in canonical little-endian form. */ + void serialize(BinaryWriter& writer) const { + writer.write_size(parent); + writer.write_size(left_child); + writer.write_size(right_child); + writer.write_u64(middle); + bit_vector_data.serialize(writer); + data.serialize(writer); } - /** @brief Constructs wavelet node out of the raw bytes */ - static WaveletNode deserialize(std::span& data) + /** @brief Construct one checked, non-owning node from @p reader. */ + static WaveletNode deserialize(BinaryReader& reader, + DeserializationValidation validation) requires(std::same_as) { WaveletNode result; - auto read = [&data](auto& value) { - constexpr size_t length = sizeof(value); - std::memcpy(&value, data.data(), length); - data = data.subspan(length); - }; - read(result.parent); - read(result.left_child); - read(result.right_child); - read(result.middle); - result.bit_vector_data = ReadOnlyStorageView::deserialize(data); + result.parent = reader.read_size(); + result.left_child = reader.read_size(); + result.right_child = reader.read_size(); + result.middle = reader.read_u64(); + result.bit_vector_data = ReadOnlyStorageView::deserialize(reader); result.data = RankSelectSupport::deserialize( - result.bit_vector_data.as_words64(), data); + result.bit_vector_data.as_words64(), reader, validation); return result; } }; @@ -96,6 +106,109 @@ class WaveletTreeIndex : public WaveletTreeBase> { std::vector leaves_; std::vector permutation_, inverse_permutation_; + void validate_deserialized_topology( + DeserializationValidation validation) const { + if (root_ == npos) { + if (validation == DeserializationValidation::kFull && + std::ranges::any_of(leaves_, + [](node_index_t leaf) { return leaf != npos; })) { + throw std::invalid_argument( + "Invalid serialized empty wavelet-tree leaves"); + } + return; + } + if (nodes_[root_].parent != npos) { + throw std::invalid_argument("Serialized wavelet-tree root has a parent"); + } + + std::vector incoming_edges(nodes_.size()); + for (node_index_t parent = 0; parent < nodes_.size(); ++parent) { + const WaveletNode& node = nodes_[parent]; + for (const node_index_t child : {node.left_child, node.right_child}) { + if (child == npos) { + continue; + } + if (nodes_[child].parent != parent) { + throw std::invalid_argument( + "Serialized wavelet-tree parent/child links disagree"); + } + if (incoming_edges[child] != 0) { + throw std::invalid_argument( + "Serialized wavelet-tree node has multiple parents"); + } + ++incoming_edges[child]; + } + } + + for (node_index_t node = 0; node < nodes_.size(); ++node) { + const std::size_t expected_edges = node == root_ ? 0 : 1; + if (incoming_edges[node] != expected_edges) { + throw std::invalid_argument( + "Serialized wavelet-tree node is detached from its parent"); + } + } + + struct PendingNode { + node_index_t node; + std::size_t symbol_begin; + std::size_t symbol_end; + }; + std::vector reached(nodes_.size()); + std::vector pending = {{root_, 0, alphabet_size_}}; + while (!pending.empty()) { + const PendingNode current = pending.back(); + pending.pop_back(); + const node_index_t node = current.node; + reached[node] = true; + const WaveletNode& metadata = nodes_[node]; + if (metadata.middle <= current.symbol_begin || + metadata.middle >= current.symbol_end) { + throw std::invalid_argument( + "Serialized wavelet-tree split is outside its symbol range"); + } + + const std::size_t one_count = + validation == DeserializationValidation::kFull + ? metadata.data.rank(metadata.data.size()) + : 0; + const std::size_t zero_count = + validation == DeserializationValidation::kFull + ? metadata.data.size() - one_count + : 0; + const auto validate_branch = [&](node_index_t child, + std::size_t symbol_begin, + std::size_t symbol_end, + std::size_t expected_size) { + if (child != npos) { + if (validation == DeserializationValidation::kFull && + nodes_[child].data.size() != expected_size) { + throw std::invalid_argument( + "Serialized wavelet-tree child has the wrong length"); + } + pending.push_back({child, symbol_begin, symbol_end}); + return; + } + if (validation == DeserializationValidation::kFull) { + for (std::size_t symbol = symbol_begin; symbol < symbol_end; + ++symbol) { + if (leaves_[symbol] != node) { + throw std::invalid_argument( + "Serialized wavelet-tree leaf map disagrees with topology"); + } + } + } + }; + validate_branch(metadata.left_child, current.symbol_begin, + metadata.middle, zero_count); + validate_branch(metadata.right_child, metadata.middle, current.symbol_end, + one_count); + } + if (std::ranges::find(reached, false) != reached.end()) { + throw std::invalid_argument( + "Serialized wavelet-tree contains unreachable nodes"); + } + } + /** * @brief Recursive building of the nodes * @@ -136,7 +249,7 @@ class WaveletTreeIndex : public WaveletTreeBase> { middle = begin + (middle == npos ? (end - begin) / 2 : middle); nodes.emplace_back(middle); - nodes[result].stream.reserve(prefix_sum[end] - prefix_sum[begin]); + nodes[result].stream.reserve_bits(prefix_sum[end] - prefix_sum[begin]); nodes[result].parent = parent; nodes[result].left_child = build_node(begin, middle, result, get_middle, prefix_sum, nodes); @@ -305,7 +418,7 @@ class WaveletTreeIndex : public WaveletTreeBase> { for (node_index_t current = root_; current != npos;) { auto& node = nodes[current]; bool go_right = index >= node.middle; - node.stream << go_right; + node.stream.write_bit(go_right); if (go_right) { current = node.right_child; } else { @@ -377,6 +490,12 @@ class WaveletTreeIndex : public WaveletTreeBase> { * @param end End of the segment * @return Queried segment of data * + * @details Queries packed bit vectors and rank/select metadata directly + * through the node storage. A deserialized view does not consult or retain + * its `BinaryReader`. The current implementation materializes the requested + * output and an equally sized temporary buffer, for peak auxiliary and + * result storage of two `uint64_t` values per returned symbol. + * */ std::vector get_segment_impl(size_t begin, size_t end) const { if (alphabet_size_ == 0 || data_size_ == 0 || begin >= end) [[unlikely]] { @@ -398,51 +517,173 @@ class WaveletTreeIndex : public WaveletTreeBase> { size_t size_impl() const { return data_size_; } /** - * @brief Writes a wavelet tree serialization to the bit stream + * @brief Write a versioned canonical little-endian wavelet-tree artifact. * + * @throws std::invalid_argument if the artifact would not begin at an + * eight-byte-aligned writer offset required by zero-copy deserialization. */ - void serialize(pixie::OutputBitStream& bs) const { - bs << alphabet_size_ << data_size_ << root_ << nodes_.size(); + void serialize(BinaryWriter& writer) const { + if (writer.size_bytes() % alignof(std::uint64_t) != 0) { + throw std::invalid_argument( + "Wavelet-tree serialization requires an aligned writer offset"); + } + const std::size_t artifact_begin = writer.size_bytes(); + detail::write_magic(writer, kSerializationMagic); + writer.write_u32(kSerializationVersion); + writer.write_u32(0); + const std::size_t artifact_size_position = writer.write_u64_placeholder(); + + writer.write_size(alphabet_size_); + writer.write_size(data_size_); + writer.write_size(root_); + writer.write_size(nodes_.size()); for (const WaveletNode& node : nodes_) { - node.serialize(bs); + node.serialize(writer); } for (const node_index_t leaf : leaves_) { - bs << leaf; + writer.write_size(leaf); } for (const size_t idx : permutation_) { - bs << idx; + writer.write_size(idx); } + + const std::size_t unpadded_size = writer.size_bytes() - artifact_begin; + writer.write_zeros( + (sizeof(std::uint64_t) - unpadded_size % sizeof(std::uint64_t)) % + sizeof(std::uint64_t)); + writer.patch_u64( + artifact_size_position, + static_cast(writer.size_bytes() - artifact_begin)); } + /** + * @brief Restore one checked, non-owning wavelet-tree artifact. + * + * @details The result retains views into the reader's backing bytes. Those + * bytes must remain alive, immutable, and aligned for 64-bit access for the + * result's lifetime. On success @p reader advances past exactly one framed + * artifact; on failure it is unchanged. @p validation selects quick + * structural checks or exact bitvector-derived metadata validation. + * + * @param reader Input cursor, advanced only after successful validation. + * @param validation Quick structural or full bitvector-derived validation. + * + * @throws std::invalid_argument for malformed, truncated, incompatible, or + * structurally inconsistent metadata. + * @throws std::length_error when an encoded count is not representable. + */ static WaveletTreeIndex deserialize( - std::span& data) { + BinaryReader& reader, + DeserializationValidation validation = + DeserializationValidation::kQuick) { + BinaryReader candidate = reader; + if (reinterpret_cast(candidate.remaining_bytes().data()) % + alignof(std::uint64_t) != + 0) { + throw std::invalid_argument( + "Serialized wavelet-tree artifact is not word aligned"); + } + const std::size_t available_size = candidate.remaining(); + detail::require_magic(candidate, kSerializationMagic); + if (candidate.read_u32() != kSerializationVersion || + candidate.read_u32() != 0) { + throw std::invalid_argument( + "Incompatible serialized wavelet-tree artifact"); + } + const std::size_t artifact_size = detail::checked_artifact_size( + candidate.read_u64(), kSerializationHeaderBytes, available_size); + BinaryReader payload = + candidate.read_subreader(artifact_size - kSerializationHeaderBytes); + WaveletTreeIndex result; - auto read = [&data](auto& value) { - constexpr size_t length = sizeof(value); - std::memcpy(&value, data.data(), length); - data = data.subspan(length); - }; - read(result.alphabet_size_); - read(result.data_size_); - read(result.root_); - size_t size; - read(size); - result.nodes_.resize(size); + result.alphabet_size_ = payload.read_size(); + result.data_size_ = payload.read_size(); + result.root_ = payload.read_size(); + const std::size_t node_count = payload.read_size(); + const std::vector empty_nodes; + if (node_count > empty_nodes.max_size()) { + throw std::length_error( + "Serialized wavelet-tree node count is too large"); + } + constexpr std::size_t kMinimumNodeBytes = + 4 * sizeof(std::uint64_t) + sizeof(std::uint64_t); + if (node_count > payload.remaining() / kMinimumNodeBytes) { + throw SerializationError("Truncated serialized wavelet-tree nodes", + payload.byte_offset()); + } + result.nodes_.resize(node_count); for (auto& node : result.nodes_) { - node = WaveletNode::deserialize(data); + node = WaveletNode::deserialize(payload, validation); + } + const std::vector empty_indices; + if (result.alphabet_size_ > empty_indices.max_size() || + result.alphabet_size_ > + payload.remaining() / (2 * sizeof(std::uint64_t))) { + throw std::length_error("Serialized wavelet-tree alphabet is too large"); } result.leaves_.resize(result.alphabet_size_); for (node_index_t& leaf : result.leaves_) { - read(leaf); + leaf = payload.read_size(); } result.permutation_.resize(result.alphabet_size_); for (size_t& index : result.permutation_) { - read(index); + index = payload.read_size(); } result.inverse_permutation_.resize(result.alphabet_size_); + std::vector seen(result.alphabet_size_); for (size_t i = 0; i < result.alphabet_size_; i++) { + if (result.permutation_[i] >= result.alphabet_size_ || + seen[result.permutation_[i]]) { + throw std::invalid_argument( + "Invalid serialized wavelet-tree permutation"); + } + seen[result.permutation_[i]] = true; result.inverse_permutation_[result.permutation_[i]] = i; } + const auto valid_node_index = [&result](node_index_t index) { + return index == npos || index < result.nodes_.size(); + }; + if (!valid_node_index(result.root_) || + (result.nodes_.empty() != (result.root_ == npos))) { + throw std::invalid_argument("Invalid serialized wavelet-tree root"); + } + for (const node_index_t leaf : result.leaves_) { + if (!valid_node_index(leaf)) { + throw std::invalid_argument("Invalid serialized wavelet-tree leaf"); + } + } + for (const WaveletNode& node : result.nodes_) { + if (!valid_node_index(node.parent) || + !valid_node_index(node.left_child) || + !valid_node_index(node.right_child) || + node.data.size() > node.bit_vector_data.size_bits() || + node.middle == 0 || node.middle >= result.alphabet_size_) { + throw std::invalid_argument("Invalid serialized wavelet-tree node"); + } + } + result.validate_deserialized_topology(validation); + if (result.root_ != npos && + result.nodes_[result.root_].data.size() != result.data_size_) { + throw std::invalid_argument( + "Serialized wavelet-tree root has the wrong length"); + } + payload.require_zero_padding(sizeof(std::uint64_t) - 1); + reader = candidate; + return result; + } + + /** + * @brief Restore one artifact from @p data and advance it on success. + * @param data Input bytes, advanced only after successful validation. + * @param validation Quick structural or full bitvector-derived validation. + */ + static WaveletTreeIndex deserialize( + std::span& data, + DeserializationValidation validation = + DeserializationValidation::kQuick) { + BinaryReader reader(data); + auto result = deserialize(reader, validation); + data = data.subspan(reader.position()); return result; } }; diff --git a/src/benchmarks/serialization_benchmarks.cpp b/src/benchmarks/serialization_benchmarks.cpp new file mode 100644 index 0000000..9b6c877 --- /dev/null +++ b/src/benchmarks/serialization_benchmarks.cpp @@ -0,0 +1,619 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +constexpr std::size_t kKiB = 1024; +constexpr std::size_t kMiB = 1024 * kKiB; +constexpr std::size_t kRecordBytes = 8 * sizeof(std::uint64_t); +constexpr std::size_t kDefaultStagingBytes = 64 * kKiB; +constexpr std::size_t kWaveletAlphabetSize = 256; +constexpr double kBenchmarkWarmupSeconds = 0.1; +constexpr double kBenchmarkMinSeconds = 0.5; + +std::span as_writable_span(std::vector& bytes) { + return {bytes.data(), bytes.size()}; +} + +std::span as_const_span(const std::vector& bytes) { + return {bytes.data(), bytes.size()}; +} + +std::uint64_t mix(std::uint64_t value) { + value += 0x9e3779b97f4a7c15ULL; + value = (value ^ (value >> 30)) * 0xbf58476d1ce4e5b9ULL; + value = (value ^ (value >> 27)) * 0x94d049bb133111ebULL; + return value ^ (value >> 31); +} + +std::vector make_payload(std::size_t size) { + std::vector result(size); + for (std::size_t i = 0; i < size; ++i) { + result[i] = static_cast(mix(i) & 0xff); + } + return result; +} + +std::vector make_words(std::size_t bit_count) { + const std::size_t word_count = bit_count == 0 ? 0 : 1 + (bit_count - 1) / 64; + std::vector result(word_count); + for (std::size_t i = 0; i < word_count; ++i) { + result[i] = mix(i); + } + return result; +} + +std::vector make_values(std::size_t count) { + std::vector result(count); + for (std::size_t i = 0; i < count; ++i) { + result[i] = static_cast(mix(i) & 0x7fffffffffffffffULL); + } + return result; +} + +std::vector make_symbols(std::size_t count) { + std::vector result(count); + for (std::size_t i = 0; i < count; ++i) { + result[i] = mix(i) % kWaveletAlphabetSize; + } + return result; +} + +template +std::vector serialize_to_vector(const Serializable& value) { + pixie::VectorOutputSink sink; + pixie::BinaryWriter writer(sink); + value.serialize(writer); + writer.finish(); + return sink.take(); +} + +class AlignedArtifact { + public: + explicit AlignedArtifact(std::span bytes) + : words_((bytes.size() + sizeof(std::uint64_t) - 1) / + sizeof(std::uint64_t)), + size_bytes_(bytes.size()) { + std::ranges::copy(bytes, writable_bytes().begin()); + } + + std::span bytes() const { + return std::as_bytes(std::span(words_)) + .first(size_bytes_); + } + + private: + std::span writable_bytes() { + return std::as_writable_bytes(std::span(words_)); + } + + std::vector words_; + std::size_t size_bytes_; +}; + +class ScopedTemporaryPath { + public: + explicit ScopedTemporaryPath(std::string filename) + : path_(std::filesystem::temp_directory_path() / std::move(filename)) { + remove(); + } + + ScopedTemporaryPath(const ScopedTemporaryPath&) = delete; + ScopedTemporaryPath& operator=(const ScopedTemporaryPath&) = delete; + + ~ScopedTemporaryPath() { remove(); } + + const std::filesystem::path& path() const { return path_; } + + private: + void remove() noexcept { + std::error_code error; + std::filesystem::remove(path_, error); + } + + std::filesystem::path path_; +}; + +void set_throughput(benchmark::State& state, std::size_t bytes_per_iteration) { + const auto iterations = static_cast(state.iterations()); + const auto bytes = static_cast(bytes_per_iteration); + state.SetBytesProcessed(static_cast(iterations * bytes)); +} + +void set_artifact_counters(benchmark::State& state, + std::size_t item_count, + std::size_t artifact_bytes) { + set_throughput(state, artifact_bytes); + state.counters["artifact_bytes"] = + benchmark::Counter(static_cast(artifact_bytes)); + state.counters["items"] = benchmark::Counter(static_cast(item_count)); +} + +template +void serialize_iterations(benchmark::State& state, + const Serializable& value, + std::span destination, + std::span staging) { + for (auto _ : state) { + (void)_; + pixie::SpanOutputSink sink(destination); + pixie::BinaryWriter writer(sink, staging); + value.serialize(writer); + writer.finish(); + benchmark::DoNotOptimize(sink.size_bytes()); + benchmark::ClobberMemory(); + } +} + +template +void deserialize_iterations(benchmark::State& state, + std::span artifact, + Deserialize deserialize) { + for (auto _ : state) { + (void)_; + pixie::BinaryReader reader(artifact); + auto value = deserialize(reader); + benchmark::DoNotOptimize(value); + benchmark::DoNotOptimize(reader.position()); + } +} + +std::vector make_framed_artifact(std::size_t payload_bytes) { + const std::size_t record_count = payload_bytes / kRecordBytes; + pixie::VectorOutputSink sink; + pixie::BinaryWriter writer(sink); + for (std::size_t record = 0; record < record_count; ++record) { + const std::size_t record_begin = writer.size_bytes(); + const std::size_t size_position = writer.write_u64_placeholder(); + for (std::size_t field = 0; field < 7; ++field) { + writer.write_u64(mix(record * 7 + field)); + } + writer.patch_u64(size_position, writer.size_bytes() - record_begin); + } + writer.finish(); + return sink.take(); +} + +void BM_BinaryWriterU64Span(benchmark::State& state) { + const std::size_t requested_bytes = static_cast(state.range(0)); + const std::size_t payload_bytes = + requested_bytes - requested_bytes % sizeof(std::uint64_t); + const std::size_t field_count = payload_bytes / sizeof(std::uint64_t); + const std::size_t buffer_bytes = static_cast(state.range(1)); + std::vector fields(field_count); + for (std::size_t field = 0; field < field_count; ++field) { + fields[field] = mix(field); + } + std::vector destination(payload_bytes); + std::vector staging(buffer_bytes); + for (auto _ : state) { + (void)_; + pixie::SpanOutputSink sink(as_writable_span(destination)); + pixie::BinaryWriter writer(sink, as_writable_span(staging)); + for (std::size_t field = 0; field < field_count; ++field) { + writer.write_u64(fields[field]); + } + writer.finish(); + benchmark::DoNotOptimize(sink.size_bytes()); + benchmark::ClobberMemory(); + } + set_throughput(state, payload_bytes); +} + +void BM_BinaryReaderU64(benchmark::State& state) { + const std::size_t requested_bytes = static_cast(state.range(0)); + const std::size_t payload_bytes = + requested_bytes - requested_bytes % sizeof(std::uint64_t); + const std::size_t field_count = payload_bytes / sizeof(std::uint64_t); + const std::vector source = make_framed_artifact(payload_bytes); + for (auto _ : state) { + (void)_; + pixie::BinaryReader reader(as_const_span(source)); + std::uint64_t checksum = 0; + for (std::size_t field = 0; field < field_count; ++field) { + checksum ^= reader.read_u64(); + } + benchmark::DoNotOptimize(checksum); + } + set_throughput(state, payload_bytes); +} + +void BM_BinaryWriterBytesSpan(benchmark::State& state) { + const std::size_t payload_bytes = static_cast(state.range(0)); + const std::vector source = make_payload(payload_bytes); + std::vector destination(payload_bytes); + std::vector staging(kDefaultStagingBytes); + for (auto _ : state) { + (void)_; + pixie::SpanOutputSink sink(as_writable_span(destination)); + pixie::BinaryWriter writer(sink, as_writable_span(staging)); + writer.write_bytes(as_const_span(source)); + writer.finish(); + benchmark::DoNotOptimize(sink.size_bytes()); + benchmark::ClobberMemory(); + } + set_throughput(state, payload_bytes); +} + +void BM_BinaryWriterBytesVector(benchmark::State& state) { + const std::size_t payload_bytes = static_cast(state.range(0)); + const std::vector source = make_payload(payload_bytes); + std::vector staging(kDefaultStagingBytes); + for (auto _ : state) { + (void)_; + pixie::VectorOutputSink sink; + pixie::BinaryWriter writer(sink, as_writable_span(staging)); + writer.write_bytes(as_const_span(source)); + writer.finish(); + benchmark::DoNotOptimize(sink.bytes().data()); + benchmark::DoNotOptimize(sink.size_bytes()); + } + set_throughput(state, payload_bytes); +} + +void BM_BinaryReaderByteSpans(benchmark::State& state) { + const std::size_t payload_bytes = static_cast(state.range(0)); + const std::size_t chunk_bytes = static_cast(state.range(1)); + const std::vector source = make_payload(payload_bytes); + for (auto _ : state) { + (void)_; + pixie::BinaryReader reader(as_const_span(source)); + std::uint64_t checksum = 0; + while (!reader.empty()) { + const std::size_t count = std::min(chunk_bytes, reader.remaining()); + const std::span chunk = reader.read_bytes(count); + checksum += std::to_integer(chunk.front()); + checksum += std::to_integer(chunk.back()); + } + benchmark::DoNotOptimize(checksum); + } + set_throughput(state, payload_bytes); + const std::size_t spans_per_iteration = + (payload_bytes + chunk_bytes - 1) / chunk_bytes; + state.SetItemsProcessed(static_cast(state.iterations()) * + static_cast(spans_per_iteration)); +} + +void BM_BinaryWriterFramedSpan(benchmark::State& state) { + const std::size_t requested_bytes = static_cast(state.range(0)); + const std::size_t payload_bytes = + requested_bytes - requested_bytes % kRecordBytes; + const std::size_t record_count = payload_bytes / kRecordBytes; + const std::size_t buffer_bytes = static_cast(state.range(1)); + std::vector fields(record_count * 7); + for (std::size_t field = 0; field < fields.size(); ++field) { + fields[field] = mix(field); + } + std::vector destination(payload_bytes); + std::vector staging(buffer_bytes); + for (auto _ : state) { + (void)_; + pixie::SpanOutputSink sink(as_writable_span(destination)); + pixie::BinaryWriter writer(sink, as_writable_span(staging)); + for (std::size_t record = 0; record < record_count; ++record) { + const std::size_t record_begin = writer.size_bytes(); + const std::size_t size_position = writer.write_u64_placeholder(); + for (std::size_t field = 0; field < 7; ++field) { + writer.write_u64(fields[record * 7 + field]); + } + writer.patch_u64(size_position, writer.size_bytes() - record_begin); + } + writer.finish(); + benchmark::DoNotOptimize(sink.size_bytes()); + benchmark::ClobberMemory(); + } + set_throughput(state, payload_bytes); +} + +void BM_BinaryReaderFramed(benchmark::State& state) { + const std::size_t requested_bytes = static_cast(state.range(0)); + const std::size_t payload_bytes = + requested_bytes - requested_bytes % kRecordBytes; + const std::vector source = make_framed_artifact(payload_bytes); + for (auto _ : state) { + (void)_; + pixie::BinaryReader reader(as_const_span(source)); + std::uint64_t checksum = 0; + while (!reader.empty()) { + const std::size_t record_bytes = reader.read_size(); + pixie::BinaryReader record = + reader.read_subreader(record_bytes - sizeof(std::uint64_t)); + while (!record.empty()) { + checksum ^= record.read_u64(); + } + } + benchmark::DoNotOptimize(checksum); + } + set_throughput(state, payload_bytes); +} + +void BM_BinaryWriterBytesFile(benchmark::State& state) { + const std::size_t payload_bytes = static_cast(state.range(0)); + const std::vector source = make_payload(payload_bytes); + std::vector staging(kDefaultStagingBytes); + ScopedTemporaryPath temporary("pixie_serialization_writer_benchmark.bin"); + for (auto _ : state) { + (void)_; + pixie::io::FileOutputSink sink(temporary.path()); + pixie::BinaryWriter writer(sink, as_writable_span(staging)); + writer.write_bytes(as_const_span(source)); + writer.finish(); + benchmark::DoNotOptimize(sink.size_bytes()); + } + set_throughput(state, payload_bytes); +} + +void BM_BinaryReaderU64MappedWarm(benchmark::State& state) { + const std::size_t requested_bytes = static_cast(state.range(0)); + const std::size_t payload_bytes = + requested_bytes - requested_bytes % sizeof(std::uint64_t); + const std::size_t field_count = payload_bytes / sizeof(std::uint64_t); + const std::vector source = make_framed_artifact(payload_bytes); + ScopedTemporaryPath temporary("pixie_serialization_reader_benchmark.bin"); + { + pixie::io::FileOutputSink sink(temporary.path()); + sink.write(as_const_span(source)); + sink.finish(); + } + pixie::io::MappedFile mapped(temporary.path()); + std::uint64_t warmup = 0; + constexpr std::size_t kPageBytes = 4096; + for (std::size_t offset = 0; offset < mapped.size_bytes(); + offset += kPageBytes) { + warmup += std::to_integer(mapped.as_bytes()[offset]); + } + benchmark::DoNotOptimize(warmup); + for (auto _ : state) { + (void)_; + pixie::BinaryReader reader(mapped.as_bytes()); + std::uint64_t checksum = 0; + for (std::size_t field = 0; field < field_count; ++field) { + checksum ^= reader.read_u64(); + } + benchmark::DoNotOptimize(checksum); + } + set_throughput(state, payload_bytes); +} + +void BM_RankSelectSerialize(benchmark::State& state) { + const std::size_t bit_count = static_cast(state.range(0)); + const std::vector words = make_words(bit_count); + const pixie::RankSelectSupport<> index(words, bit_count); + const std::vector artifact = serialize_to_vector(index); + std::vector destination(artifact.size()); + std::vector staging(kDefaultStagingBytes); + serialize_iterations(state, index, as_writable_span(destination), + as_writable_span(staging)); + set_artifact_counters(state, bit_count, artifact.size()); +} + +template +void BM_RankSelectDeserializeOwningImpl(benchmark::State& state) { + const std::size_t bit_count = static_cast(state.range(0)); + const std::vector words = make_words(bit_count); + const pixie::RankSelectSupport<> index(words, bit_count); + const std::vector artifact = serialize_to_vector(index); + deserialize_iterations(state, as_const_span(artifact), + [&](pixie::BinaryReader& reader) { + return pixie::RankSelectSupport<>::deserialize( + words, reader, Validation); + }); + set_artifact_counters(state, bit_count, artifact.size()); +} + +template +void BM_RankSelectDeserializeViewImpl(benchmark::State& state) { + const std::size_t bit_count = static_cast(state.range(0)); + const std::vector words = make_words(bit_count); + const pixie::RankSelectSupport<> index(words, bit_count); + const std::vector serialized = serialize_to_vector(index); + const AlignedArtifact artifact(as_const_span(serialized)); + deserialize_iterations( + state, artifact.bytes(), [&](pixie::BinaryReader& reader) { + return pixie::RankSelectSupport< + pixie::ReadOnlyStorageView>::deserialize(words, reader, Validation); + }); + set_artifact_counters(state, bit_count, artifact.bytes().size()); +} + +void BM_RmMSerialize(benchmark::State& state) { + const std::size_t bit_count = static_cast(state.range(0)); + const std::vector words = make_words(bit_count); + const pixie::RmMTree index(words, bit_count); + const std::vector artifact = serialize_to_vector(index); + std::vector destination(artifact.size()); + std::vector staging(kDefaultStagingBytes); + serialize_iterations(state, index, as_writable_span(destination), + as_writable_span(staging)); + set_artifact_counters(state, bit_count, artifact.size()); +} + +template +void BM_RmMDeserializeImpl(benchmark::State& state) { + const std::size_t bit_count = static_cast(state.range(0)); + const std::vector words = make_words(bit_count); + const pixie::RmMTree index(words, bit_count); + const std::vector artifact = serialize_to_vector(index); + deserialize_iterations( + state, as_const_span(artifact), [&](pixie::BinaryReader& reader) { + return pixie::RmMTree::deserialize(words, reader, Validation); + }); + set_artifact_counters(state, bit_count, artifact.size()); +} + +using RmqIndex = pixie::rmq::CartesianHybridBTree; + +void BM_RmqSerialize(benchmark::State& state) { + const std::size_t value_count = static_cast(state.range(0)); + const std::vector values = make_values(value_count); + const RmqIndex index(values); + const std::vector artifact = serialize_to_vector(index); + std::vector destination(artifact.size()); + std::vector staging(kDefaultStagingBytes); + serialize_iterations(state, index, as_writable_span(destination), + as_writable_span(staging)); + set_artifact_counters(state, value_count, artifact.size()); +} + +template +void BM_RmqDeserializeImpl(benchmark::State& state) { + const std::size_t value_count = static_cast(state.range(0)); + const std::vector values = make_values(value_count); + const RmqIndex index(values); + const std::vector artifact = serialize_to_vector(index); + deserialize_iterations( + state, as_const_span(artifact), [&](pixie::BinaryReader& reader) { + return RmqIndex::deserialize(values, reader, Validation); + }); + set_artifact_counters(state, value_count, artifact.size()); +} + +void BM_WaveletTreeSerialize(benchmark::State& state) { + const std::size_t symbol_count = static_cast(state.range(0)); + const std::vector symbols = make_symbols(symbol_count); + const pixie::WaveletTree index(kWaveletAlphabetSize, symbols); + const std::vector artifact = serialize_to_vector(index); + std::vector destination(artifact.size()); + std::vector staging(kDefaultStagingBytes); + serialize_iterations(state, index, as_writable_span(destination), + as_writable_span(staging)); + set_artifact_counters(state, symbol_count, artifact.size()); +} + +template +void BM_WaveletTreeDeserializeViewImpl(benchmark::State& state) { + const std::size_t symbol_count = static_cast(state.range(0)); + const std::vector symbols = make_symbols(symbol_count); + const pixie::WaveletTree index(kWaveletAlphabetSize, symbols); + const std::vector serialized = serialize_to_vector(index); + const AlignedArtifact artifact(as_const_span(serialized)); + deserialize_iterations( + state, artifact.bytes(), [](pixie::BinaryReader& reader) { + return pixie::WaveletTreeView::deserialize(reader, Validation); + }); + set_artifact_counters(state, symbol_count, artifact.bytes().size()); +} + +#define PIXIE_DESERIALIZATION_WRAPPERS(name) \ + void name##Quick(benchmark::State& state) { \ + name##Impl(state); \ + } \ + void name##Full(benchmark::State& state) { \ + name##Impl(state); \ + } + +PIXIE_DESERIALIZATION_WRAPPERS(BM_RankSelectDeserializeOwning) +PIXIE_DESERIALIZATION_WRAPPERS(BM_RankSelectDeserializeView) +PIXIE_DESERIALIZATION_WRAPPERS(BM_RmMDeserialize) +PIXIE_DESERIALIZATION_WRAPPERS(BM_RmqDeserialize) +PIXIE_DESERIALIZATION_WRAPPERS(BM_WaveletTreeDeserializeView) + +#undef PIXIE_DESERIALIZATION_WRAPPERS + +#define PIXIE_SERIALIZATION_TIMING() \ + MinWarmUpTime(kBenchmarkWarmupSeconds)->MinTime(kBenchmarkMinSeconds) + +BENCHMARK(BM_BinaryWriterU64Span) + ->ArgsProduct({{4 * kKiB, 1 * kMiB, 64 * kMiB}, + {4 * kKiB, 64 * kKiB, 1 * kMiB}}) + ->ArgNames({"payload_bytes", "buffer_bytes"}) + ->PIXIE_SERIALIZATION_TIMING(); + +BENCHMARK(BM_BinaryReaderU64) + ->Arg(4 * kKiB) + ->Arg(1 * kMiB) + ->Arg(64 * kMiB) + ->ArgName("payload_bytes") + ->PIXIE_SERIALIZATION_TIMING(); + +BENCHMARK(BM_BinaryWriterBytesSpan) + ->Arg(4 * kKiB) + ->Arg(1 * kMiB) + ->Arg(64 * kMiB) + ->ArgName("payload_bytes") + ->PIXIE_SERIALIZATION_TIMING(); + +BENCHMARK(BM_BinaryWriterBytesVector) + ->Arg(4 * kKiB) + ->Arg(1 * kMiB) + ->Arg(64 * kMiB) + ->ArgName("payload_bytes") + ->PIXIE_SERIALIZATION_TIMING(); + +BENCHMARK(BM_BinaryReaderByteSpans) + ->Args({1 * kMiB, 4 * kKiB}) + ->Args({1 * kMiB, 64 * kKiB}) + ->Args({64 * kMiB, 4 * kKiB}) + ->Args({64 * kMiB, 64 * kKiB}) + ->ArgNames({"payload_bytes", "chunk_bytes"}) + ->PIXIE_SERIALIZATION_TIMING(); + +BENCHMARK(BM_BinaryWriterFramedSpan) + ->ArgsProduct({{4 * kKiB, 1 * kMiB, 64 * kMiB}, + {4 * kKiB, 64 * kKiB, 1 * kMiB}}) + ->ArgNames({"payload_bytes", "buffer_bytes"}) + ->PIXIE_SERIALIZATION_TIMING(); + +BENCHMARK(BM_BinaryReaderFramed) + ->Arg(4 * kKiB) + ->Arg(1 * kMiB) + ->Arg(64 * kMiB) + ->ArgName("payload_bytes") + ->PIXIE_SERIALIZATION_TIMING(); + +BENCHMARK(BM_BinaryWriterBytesFile) + ->Arg(1 * kMiB) + ->Arg(64 * kMiB) + ->ArgName("payload_bytes") + ->UseRealTime() + ->PIXIE_SERIALIZATION_TIMING(); + +BENCHMARK(BM_BinaryReaderU64MappedWarm) + ->Arg(1 * kMiB) + ->Arg(64 * kMiB) + ->ArgName("payload_bytes") + ->PIXIE_SERIALIZATION_TIMING(); + +#define PIXIE_STRUCTURE_BENCHMARK(function) \ + BENCHMARK(function) \ + ->Arg(1 << 10) \ + ->Arg(1 << 16) \ + ->Arg(1 << 20) \ + ->ArgName("N") \ + ->PIXIE_SERIALIZATION_TIMING() + +PIXIE_STRUCTURE_BENCHMARK(BM_RankSelectSerialize); +PIXIE_STRUCTURE_BENCHMARK(BM_RankSelectDeserializeOwningQuick); +PIXIE_STRUCTURE_BENCHMARK(BM_RankSelectDeserializeOwningFull); +PIXIE_STRUCTURE_BENCHMARK(BM_RankSelectDeserializeViewQuick); +PIXIE_STRUCTURE_BENCHMARK(BM_RankSelectDeserializeViewFull); +PIXIE_STRUCTURE_BENCHMARK(BM_RmMSerialize); +PIXIE_STRUCTURE_BENCHMARK(BM_RmMDeserializeQuick); +PIXIE_STRUCTURE_BENCHMARK(BM_RmMDeserializeFull); +PIXIE_STRUCTURE_BENCHMARK(BM_RmqSerialize); +PIXIE_STRUCTURE_BENCHMARK(BM_RmqDeserializeQuick); +PIXIE_STRUCTURE_BENCHMARK(BM_RmqDeserializeFull); +PIXIE_STRUCTURE_BENCHMARK(BM_WaveletTreeSerialize); +PIXIE_STRUCTURE_BENCHMARK(BM_WaveletTreeDeserializeViewQuick); +PIXIE_STRUCTURE_BENCHMARK(BM_WaveletTreeDeserializeViewFull); + +#undef PIXIE_STRUCTURE_BENCHMARK +#undef PIXIE_SERIALIZATION_TIMING + +} // namespace diff --git a/src/benchmarks/wavelet_tree_benchmarks.cpp b/src/benchmarks/wavelet_tree_benchmarks.cpp index 1ac1249..35ae3d4 100644 --- a/src/benchmarks/wavelet_tree_benchmarks.cpp +++ b/src/benchmarks/wavelet_tree_benchmarks.cpp @@ -85,16 +85,16 @@ static void BM_WaveletTreeViewSelect(benchmark::State& state) { } WaveletTree orig_tree(alphabet_size, data); - pixie::OutputBitStream bs; - orig_tree.serialize(bs); - std::vector serialized_data = bs.extract(); - std::span byte_span( - reinterpret_cast(serialized_data.data()), - serialized_data.size() * sizeof(uint64_t)); + pixie::VectorOutputSink output; + pixie::BinaryWriter writer(output); + orig_tree.serialize(writer); + writer.finish(); + std::vector serialized_data = output.take(); + pixie::BinaryReader reader(serialized_data); state.ResumeTiming(); - auto view_tree = pixie::WaveletTreeView::deserialize(byte_span); + auto view_tree = pixie::WaveletTreeView::deserialize(reader); benchmark::DoNotOptimize(view_tree); for (size_t i = 0; i < query; i++) { @@ -119,16 +119,16 @@ static void BM_WaveletTreeViewRank(benchmark::State& state) { generate_random_data(query, data_size + 1, rng); WaveletTree orig_tree(alphabet_size, data); - pixie::OutputBitStream bs; - orig_tree.serialize(bs); - std::vector serialized_data = bs.extract(); - std::span byte_span( - reinterpret_cast(serialized_data.data()), - serialized_data.size() * sizeof(uint64_t)); + pixie::VectorOutputSink output; + pixie::BinaryWriter writer(output); + orig_tree.serialize(writer); + writer.finish(); + std::vector serialized_data = output.take(); + pixie::BinaryReader reader(serialized_data); state.ResumeTiming(); - auto view_tree = pixie::WaveletTreeView::deserialize(byte_span); + auto view_tree = pixie::WaveletTreeView::deserialize(reader); benchmark::DoNotOptimize(view_tree); for (size_t i = 0; i < query; i++) { @@ -152,16 +152,16 @@ static void BM_WaveletTreeViewSegment(benchmark::State& state) { generate_random_data(query, data_size + 1 - length, rng); WaveletTree orig_tree(alphabet_size, data); - pixie::OutputBitStream bs; - orig_tree.serialize(bs); - std::vector serialized_data = bs.extract(); - std::span byte_span( - reinterpret_cast(serialized_data.data()), - serialized_data.size() * sizeof(uint64_t)); + pixie::VectorOutputSink output; + pixie::BinaryWriter writer(output); + orig_tree.serialize(writer); + writer.finish(); + std::vector serialized_data = output.take(); + pixie::BinaryReader reader(serialized_data); state.ResumeTiming(); - auto view_tree = pixie::WaveletTreeView::deserialize(byte_span); + auto view_tree = pixie::WaveletTreeView::deserialize(reader); benchmark::DoNotOptimize(view_tree); for (size_t i = 0; i < query; i++) { diff --git a/src/docs/benchmark_results.md b/src/docs/benchmark_results.md index 05b543f..18519ed 100644 --- a/src/docs/benchmark_results.md +++ b/src/docs/benchmark_results.md @@ -1,6 +1,8 @@ # Benchmark Results -These results were generated on 2026-05-18 from `build/release` binaries on the local benchmark host. JSON inputs are kept under `src/docs/benchmarks`. +Unless a section says otherwise, these results were generated on 2026-05-18 +from optimized binaries on the local benchmark host. JSON inputs are kept +under `src/docs/benchmarks`. ## Excess Positions @@ -28,28 +30,6 @@ python3 scripts/excess_benchmark_table.py \ | LUTAVX512 | 12.33 ns | 18.34 ns | 18.06 ns | 18.21 ns | 12.75 ns | | Scalar | 304.42 ns | 389.58 ns | 446.94 ns | 399.80 ns | 316.23 ns | -## BitVector Size Sweep - -The BitVector plot uses the 50/50 fill variants for rank/select over the registered benchmark size grid. The benchmark definitions use fixed repeats, so the command-line repetition value is not used for this binary. - -Command: - -```sh -./build/release/benchmarks \ - --benchmark_filter='BM_(RankInterleaved|RankNonInterleaved|RankZeroNonInterleaved|SelectNonInterleaved|SelectZeroNonInterleaved)/' \ - --benchmark_report_aggregates_only=true \ - --benchmark_display_aggregates_only=true \ - --benchmark_format=json \ - --benchmark_out=src/docs/benchmarks/bitvector_size.json -python3 scripts/plot_size_benchmarks.py \ - src/docs/benchmarks/bitvector_size.json \ - -o src/docs/images/benchmarks/bitvector_size.png \ - --size-key n \ - --title 'BitVector benchmark time vs size' -``` - -![BitVector benchmark time vs size](images/benchmarks/bitvector_size.png) - ## RmM Tree Size Sweep The RmM comparison uses operations available in both Pixie and sdsl-lite over the same power-of-two tree sizes. Pixie's benchmark harness only constructs query pools needed by the selected operations. @@ -57,7 +37,7 @@ The RmM comparison uses operations available in both Pixie and sdsl-lite over th Pixie command: ```sh -./build/release/bench_rmm \ +./build/benchmarks/rmm_benchmarks \ --ops=rank1,rank0,select1,excess,range_min_query_pos,range_min_query_val,close,open,enclose \ --explicit_sizes=16384,32768,65536,131072,262144,524288,1048576,2097152,4194304 \ --Q=32768 \ @@ -71,7 +51,7 @@ Pixie command: sdsl-lite command: ```sh -./build/release/bench_rmm_sdsl \ +./build/benchmark-all-backends/rmm_sdsl_benchmarks \ --ops=rank1,rank0,select1,excess,range_min_query_pos,range_min_query_val,close,open,enclose \ --explicit_sizes=16384,32768,65536,131072,262144,524288,1048576,2097152,4194304 \ --Q=32768 \ diff --git a/src/tests/benchmark_tests.cpp b/src/tests/benchmark_tests.cpp index 5586754..dbc3bc2 100644 --- a/src/tests/benchmark_tests.cpp +++ b/src/tests/benchmark_tests.cpp @@ -11,7 +11,7 @@ TEST(RankSelectBenchmarkTest, Select10PercentFill) { std::vector bits(((8 + n / 64) / 8) * 8); size_t num_ones = n * 0.1; - for (int i = 0; i < num_ones; i++) { + for (size_t i = 0; i < num_ones; i++) { uint64_t pos = rng() % n; bits[pos / 64] |= (1ULL << pos % 64); } @@ -32,7 +32,7 @@ TEST(RankSelectBenchmarkTest, SelectZero10PercentFill) { std::vector bits(((8 + n / 64) / 8) * 8); size_t num_ones = n * 0.1; - for (int i = 0; i < num_ones; i++) { + for (size_t i = 0; i < num_ones; i++) { uint64_t pos = rng() % n; bits[pos / 64] |= (1ULL << pos % 64); } @@ -53,7 +53,7 @@ TEST(RankSelectBenchmarkTest, Select90PercentFill) { std::vector bits(((8 + n / 64) / 8) * 8); size_t num_ones = n * 0.9; - for (int i = 0; i < num_ones; i++) { + for (size_t i = 0; i < num_ones; i++) { uint64_t pos = rng() % n; bits[pos / 64] |= (1ULL << pos % 64); } @@ -74,7 +74,7 @@ TEST(RankSelectBenchmarkTest, SelectZero90PercentFill) { std::vector bits(((8 + n / 64) / 8) * 8); size_t num_ones = n * 0.9; - for (int i = 0; i < num_ones; i++) { + for (size_t i = 0; i < num_ones; i++) { uint64_t pos = rng() % n; bits[pos / 64] |= (1ULL << pos % 64); } diff --git a/src/tests/excess_positions_tests.cpp b/src/tests/excess_positions_tests.cpp index 6321478..faad6f4 100644 --- a/src/tests/excess_positions_tests.cpp +++ b/src/tests/excess_positions_tests.cpp @@ -342,7 +342,7 @@ TEST(ExcessPositions128, MinMatchesNaiveFixedCases) { }}; for (const auto& s : cases) { - for (const auto [left, right] : ranges) { + for (const auto& [left, right] : ranges) { const ExcessResult result = excess_min_128(s.data(), left, right); const ExcessResult expected = naive_excess_min_128(s.data(), left, right); EXPECT_EQ(result.min_excess, expected.min_excess) @@ -377,7 +377,7 @@ TEST(ExcessPositions64, MinMatchesNaiveFixedCases) { }}; for (const auto& s : cases) { - for (const auto [left, right] : ranges) { + for (const auto& [left, right] : ranges) { const ExcessResult result = excess_min_64(s.data(), left, right); const ExcessResult expected = naive_excess_min_64(s.data(), left, right); EXPECT_EQ(result.min_excess, expected.min_excess) @@ -527,7 +527,7 @@ TEST(ExcessPositions64, DisjointBoundaryPairMatchesIndependentFixedCases) { for (const auto& suffix : cases) { for (const auto& prefix : cases) { - for (const auto [suffix_left, prefix_right] : ranges) { + for (const auto& [suffix_left, prefix_right] : ranges) { const ExcessBoundaryPairResult result = excess_min_64_disjoint_suffix_prefix(suffix.data(), suffix_left, prefix.data(), prefix_right); @@ -575,7 +575,7 @@ TEST(ExcessPositions128, DisjointBoundaryPairMatchesIndependentFixedCases) { for (const auto& suffix : cases) { for (const auto& prefix : cases) { - for (const auto [suffix_left, prefix_right] : ranges) { + for (const auto& [suffix_left, prefix_right] : ranges) { check_boundary_pair_matches_independent(suffix, suffix_left, prefix, prefix_right); } @@ -630,7 +630,7 @@ TEST(ExcessPositions128Experimental, MinVariantsMatchNaive) { int case_id = 0; for (const auto& s : cases) { - for (const auto [left, right] : ranges) { + for (const auto& [left, right] : ranges) { check_min_matches_naive(excess_min_128_scalar_bits, "scalar_bits", s.data(), left, right, case_id); check_min_matches_naive(excess_min_128_nibble_lut, "nibble_lut", s.data(), @@ -729,7 +729,7 @@ TEST(ExcessPositions128Experimental, DeinterleavedSseBoundaryAndTieCases) { int case_id = 0; for (const auto& s : cases) { - for (const auto [left, right] : ranges) { + for (const auto& [left, right] : ranges) { check_min_matches_naive(excess_min_128_deinterleaved_sse, "deinterleaved_sse", s.data(), left, right, case_id); diff --git a/src/tests/rank_select_tests.cpp b/src/tests/rank_select_tests.cpp index 9dedb89..d89cc7d 100644 --- a/src/tests/rank_select_tests.cpp +++ b/src/tests/rank_select_tests.cpp @@ -1,15 +1,76 @@ #include +#include +#include #include +#include #include #include #include +#include +#include #include +#include #include #include namespace { +void overwrite_u64(std::vector& bytes, + std::size_t offset, + std::uint64_t value) { + for (std::size_t byte = 0; byte < sizeof(value); ++byte) { + bytes[offset + byte] = + static_cast((value >> (byte * 8)) & 0xffu); + } +} + +void overwrite_u32(std::vector& bytes, + std::size_t offset, + std::uint32_t value) { + for (std::size_t byte = 0; byte < sizeof(value); ++byte) { + bytes[offset + byte] = + static_cast((value >> (byte * 8)) & 0xffu); + } +} + +void overwrite_u16(std::vector& bytes, + std::size_t offset, + std::uint16_t value) { + for (std::size_t byte = 0; byte < sizeof(value); ++byte) { + bytes[offset + byte] = + static_cast((value >> (byte * 8)) & 0xffu); + } +} + +struct RankSelectSampleLayout { + std::size_t select1_begin; + std::size_t select1_count; + std::size_t select0_begin; + std::size_t select0_count; + std::size_t storage_offset; +}; + +RankSelectSampleLayout locate_rank_select_samples( + std::span artifact) { + pixie::BinaryReader reader(artifact); + reader.skip(3 * sizeof(std::uint64_t)); + RankSelectSampleLayout result; + result.select1_begin = reader.read_size(); + result.select1_count = reader.read_size(); + result.select0_begin = reader.read_size(); + result.select0_count = reader.read_size(); + reader.skip(2 * sizeof(std::uint32_t) + 8 * sizeof(std::uint64_t) + + 32 * sizeof(std::uint16_t)); + for (std::size_t storage = 0; storage < 2; ++storage) { + reader.skip(reader.read_size()); + } + const std::size_t sample_bytes = reader.read_size(); + result.storage_offset = reader.position(); + reader.skip(sample_bytes); + return result; +} + template class RankSelectSpecificationTest : public testing::Test {}; @@ -83,4 +144,251 @@ TEST(RankSelectSupportTest, AcceptsStorageSourceWithoutCopying) { EXPECT_EQ(support[2], 0); } +TEST(RankSelectSupportTest, OwningMetadataDeserializationRoundTrips) { + constexpr std::size_t kBitCount = 4097; + std::vector words((kBitCount + 63) / 64); + std::mt19937_64 rng(20260718); + for (std::uint64_t& word : words) { + word = rng(); + } + + const pixie::RankSelectSupport<> original(words, kBitCount); + pixie::VectorOutputSink output; + pixie::BinaryWriter writer(output); + original.serialize(writer); + writer.finish(); + std::vector artifact = output.take(); + pixie::BinaryReader reader(artifact); + const pixie::RankSelectSupport<> restored = + pixie::RankSelectSupport<>::deserialize(words, reader); + EXPECT_TRUE(reader.empty()); + + artifact.clear(); + artifact.shrink_to_fit(); + for (std::size_t position = 0; position <= kBitCount; ++position) { + EXPECT_EQ(restored.rank(position), original.rank(position)); + EXPECT_EQ(restored.rank0(position), original.rank0(position)); + } + const std::size_t ones = original.rank(kBitCount); + const std::size_t zeros = original.rank0(kBitCount); + for (std::size_t rank = 1; rank <= ones + 1; ++rank) { + EXPECT_EQ(restored.select(rank), original.select(rank)); + } + for (std::size_t rank = 1; rank <= zeros + 1; ++rank) { + EXPECT_EQ(restored.select0(rank), original.select0(rank)); + } +} + +TEST(RankSelectSupportTest, QuickAndFullValidationAcceptValidMetadata) { + constexpr std::size_t kBitCount = 65537; + std::vector words((kBitCount + 63) / 64); + std::mt19937_64 rng(20260816); + for (std::uint64_t& word : words) { + word = rng(); + } + const pixie::RankSelectSupport<> original(words, kBitCount); + pixie::VectorOutputSink output; + pixie::BinaryWriter writer(output); + original.serialize(writer); + writer.finish(); + const std::vector artifact = output.take(); + + for (const auto validation : {pixie::DeserializationValidation::kQuick, + pixie::DeserializationValidation::kFull}) { + pixie::BinaryReader reader(artifact); + const auto restored = + pixie::RankSelectSupport<>::deserialize(words, reader, validation); + EXPECT_TRUE(reader.empty()); + EXPECT_EQ(restored.rank(kBitCount), original.rank(kBitCount)); + } +} + +TEST(RankSelectSupportTest, + FullValidationRejectsSourceAndMetadataMismatchesTransactionally) { + constexpr std::size_t kBitCount = 4097; + std::vector words((kBitCount + 63) / 64, + 0xaaaaaaaaaaaaaaaaULL); + const pixie::RankSelectSupport<> original(words, kBitCount); + pixie::VectorOutputSink output; + pixie::BinaryWriter writer(output); + original.serialize(writer); + writer.finish(); + const std::vector valid = output.take(); + + std::vector different_words = words; + different_words[0] ^= 1; + pixie::BinaryReader quick_reader(valid); + EXPECT_NO_THROW((void)pixie::RankSelectSupport<>::deserialize( + different_words, quick_reader, pixie::DeserializationValidation::kQuick)); + EXPECT_TRUE(quick_reader.empty()); + + pixie::BinaryReader full_reader(valid); + EXPECT_THROW((void)pixie::RankSelectSupport<>::deserialize( + different_words, full_reader, + pixie::DeserializationValidation::kFull), + std::invalid_argument); + EXPECT_EQ(full_reader.position(), 0u); + + std::vector bad_rank = valid; + overwrite_u64(bad_rank, 2 * sizeof(std::uint64_t), + original.rank(kBitCount) + 1); + pixie::BinaryReader metadata_quick_reader(bad_rank); + EXPECT_NO_THROW((void)pixie::RankSelectSupport<>::deserialize( + words, metadata_quick_reader, pixie::DeserializationValidation::kQuick)); + EXPECT_TRUE(metadata_quick_reader.empty()); + + pixie::BinaryReader metadata_full_reader(bad_rank); + EXPECT_THROW( + (void)pixie::RankSelectSupport<>::deserialize( + words, metadata_full_reader, pixie::DeserializationValidation::kFull), + std::invalid_argument); + EXPECT_EQ(metadata_full_reader.position(), 0u); +} + +TEST(RankSelectSupportTest, SerializesDirectlyToMappedFile) { + constexpr std::size_t kBitCount = 1025; + std::vector words((kBitCount + 63) / 64); + std::mt19937_64 rng(20260721); + for (std::uint64_t& word : words) { + word = rng(); + } + const pixie::RankSelectSupport<> original(words, kBitCount); + const auto path = std::filesystem::temp_directory_path() / + "pixie_rank_select_serialization_test.bin"; + std::filesystem::remove(path); + { + pixie::io::FileOutputSink output(path); + std::array staging{}; + pixie::BinaryWriter writer(output, staging); + original.serialize(writer); + writer.finish(); + } + + pixie::io::MappedFile file(path); + for (const auto validation : {pixie::DeserializationValidation::kQuick, + pixie::DeserializationValidation::kFull}) { + pixie::BinaryReader reader(file.as_bytes()); + const auto restored = + pixie::RankSelectSupport::deserialize( + words, reader, validation); + EXPECT_TRUE(reader.empty()); + for (std::size_t position = 0; position <= kBitCount; position += 17) { + EXPECT_EQ(restored.rank(position), original.rank(position)); + EXPECT_EQ(restored.rank0(position), original.rank0(position)); + } + } + std::filesystem::remove(path); +} + +TEST(RankSelectSupportTest, + DeserializationRejectsOutOfRangeSelectSamplesTransactionally) { + constexpr std::size_t kBitCount = 4097; + std::vector words((kBitCount + 63) / 64, + 0xaaaaaaaaaaaaaaaaULL); + const pixie::RankSelectSupport<> original(words, kBitCount); + pixie::VectorOutputSink output; + pixie::BinaryWriter writer(output); + original.serialize(writer); + writer.finish(); + const std::vector valid = output.take(); + const RankSelectSampleLayout layout = locate_rank_select_samples(valid); + ASSERT_NE(layout.select1_count, 0u); + ASSERT_NE(layout.select0_count, 0u); + + const auto expect_rejected = [&](std::size_t sample) { + std::vector artifact = valid; + overwrite_u64(artifact, + layout.storage_offset + sample * sizeof(std::uint64_t), + std::numeric_limits::max()); + + for (const auto validation : {pixie::DeserializationValidation::kQuick, + pixie::DeserializationValidation::kFull}) { + pixie::BinaryReader owning_reader(artifact); + EXPECT_THROW((void)pixie::RankSelectSupport<>::deserialize( + words, owning_reader, validation), + std::invalid_argument); + EXPECT_EQ(owning_reader.position(), 0u); + + pixie::BinaryReader view_reader(artifact); + EXPECT_THROW( + (void) + pixie::RankSelectSupport::deserialize( + words, view_reader, validation), + std::invalid_argument); + EXPECT_EQ(view_reader.position(), 0u); + } + }; + + expect_rejected(layout.select1_begin); + expect_rejected(layout.select0_begin); +} + +TEST(RankSelectSupportTest, + DeserializationRejectsMalformedFixedMetadataTransactionally) { + constexpr std::size_t kBitCount = 4097; + std::vector words((kBitCount + 63) / 64, + 0xaaaaaaaaaaaaaaaaULL); + const pixie::RankSelectSupport<> original(words, kBitCount); + pixie::VectorOutputSink output; + pixie::BinaryWriter writer(output); + original.serialize(writer); + writer.finish(); + const std::vector valid = output.take(); + + const auto expect_rejected = [&](std::vector artifact) { + for (const auto validation : {pixie::DeserializationValidation::kQuick, + pixie::DeserializationValidation::kFull}) { + pixie::BinaryReader reader(artifact); + EXPECT_THROW((void)pixie::RankSelectSupport<>::deserialize(words, reader, + validation), + std::invalid_argument); + EXPECT_EQ(reader.position(), 0u); + } + }; + + auto bad_padded_size = valid; + overwrite_u64(bad_padded_size, sizeof(std::uint64_t), 0); + expect_rejected(std::move(bad_padded_size)); + + auto excessive_rank = valid; + overwrite_u64(excessive_rank, 2 * sizeof(std::uint64_t), kBitCount + 1); + expect_rejected(std::move(excessive_rank)); + + auto invalid_select1_begin = valid; + overwrite_u64(invalid_select1_begin, 3 * sizeof(std::uint64_t), + std::numeric_limits::max()); + expect_rejected(std::move(invalid_select1_begin)); + + auto missing_select1_samples = valid; + overwrite_u64(missing_select1_samples, 4 * sizeof(std::uint64_t), 0); + expect_rejected(std::move(missing_select1_samples)); + + auto invalid_select0_begin = valid; + overwrite_u64(invalid_select0_begin, 5 * sizeof(std::uint64_t), + std::numeric_limits::max()); + expect_rejected(std::move(invalid_select0_begin)); + + auto missing_select0_samples = valid; + overwrite_u64(missing_select0_samples, 6 * sizeof(std::uint64_t), 0); + expect_rejected(std::move(missing_select0_samples)); + + auto invalid_support = valid; + overwrite_u32(invalid_support, 7 * sizeof(std::uint64_t), 4); + expect_rejected(std::move(invalid_support)); + + auto invalid_boolean = valid; + overwrite_u32(invalid_boolean, + 7 * sizeof(std::uint64_t) + sizeof(std::uint32_t), 2); + expect_rejected(std::move(invalid_boolean)); + + auto invalid_super_delta = valid; + overwrite_u64(invalid_super_delta, 9 * sizeof(std::uint64_t), 0); + expect_rejected(std::move(invalid_super_delta)); + + auto invalid_basic_delta = valid; + overwrite_u16(invalid_basic_delta, + 16 * sizeof(std::uint64_t) + sizeof(std::uint16_t), 0); + expect_rejected(std::move(invalid_basic_delta)); +} + } // namespace diff --git a/src/tests/rmq_tests.cpp b/src/tests/rmq_tests.cpp index f31434d..19e2ec0 100644 --- a/src/tests/rmq_tests.cpp +++ b/src/tests/rmq_tests.cpp @@ -1,17 +1,24 @@ #include +#include +#include #include #include #include #include +#include +#include +#include #include #include +#include #include #include #include #include #include #include +#include #include #include @@ -206,6 +213,506 @@ struct HybridBTreeCase { } // namespace +template +concept HasRmqSerialization = + requires(const Rmq& rmq, pixie::BinaryWriter& writer) { + rmq.serialize(writer); + }; + +template +concept HasRmqDeserialization = requires(std::span values, + pixie::BinaryReader& reader) { + { Rmq::deserialize(values, reader) } -> std::same_as; +}; + +using SerializableRmq = pixie::rmq::CartesianHybridBTree; +static_assert(HasRmqSerialization); +static_assert(HasRmqDeserialization); +static_assert( + !HasRmqSerialization>); +static_assert(!HasRmqSerialization< + pixie::rmq::CartesianHybridBTree>>); +static_assert(!HasRmqSerialization< + pixie::rmq::CartesianHybridBTree, + std::uint32_t>>); +static_assert(!HasRmqSerialization< + pixie::rmq::CartesianHybridBTree, + std::size_t, + 1024>>); +static_assert(!HasRmqSerialization>); + +static std::vector serialize_rmq(const SerializableRmq& rmq) { + pixie::VectorOutputSink output; + pixie::BinaryWriter writer(output); + rmq.serialize(writer); + writer.finish(); + return output.take(); +} + +static void overwrite_u64(std::vector& bytes, + std::size_t offset, + std::uint64_t value) { + for (std::size_t byte = 0; byte < sizeof(value); ++byte) { + bytes[offset + byte] = + static_cast((value >> (byte * 8)) & 0xffu); + } +} + +struct RmqArtifactLayout { + std::size_t bp_bit_count; + std::size_t top_block_size; + std::size_t top_block_count; + std::size_t top_sparse_levels; + std::size_t bp_storage_size; + std::size_t candidate_count; + std::size_t candidates; + std::size_t has_rank_index; + std::size_t depth_selectors; + std::size_t depth_selector_count; + std::size_t depth_min_depths; + std::size_t candidate_size; + std::size_t bp_storage_bytes; +}; + +static RmqArtifactLayout locate_rmq_artifact( + std::span artifact) { + pixie::BinaryReader reader(artifact); + reader.skip(6 * sizeof(std::uint64_t)); + RmqArtifactLayout result; + result.bp_bit_count = reader.position(); + reader.skip(sizeof(std::uint64_t)); + result.top_block_size = reader.position(); + reader.skip(sizeof(std::uint64_t)); + result.top_block_count = reader.position(); + reader.skip(sizeof(std::uint64_t)); + result.top_sparse_levels = reader.position(); + reader.skip(sizeof(std::uint64_t)); + result.bp_storage_size = reader.position(); + result.bp_storage_bytes = reader.read_size(); + reader.skip(result.bp_storage_bytes); + result.candidate_count = reader.position(); + result.candidate_size = reader.read_size(); + result.candidates = reader.position(); + reader.skip(result.candidate_size * sizeof(std::uint64_t)); + result.has_rank_index = reader.position(); + const bool has_rank_index = reader.read_u8() != 0; + if (has_rank_index) { + reader.skip(7 * sizeof(std::uint64_t) + 2 * sizeof(std::uint32_t) + + 8 * sizeof(std::uint64_t) + 32 * sizeof(std::uint16_t)); + for (std::size_t storage = 0; storage < 3; ++storage) { + reader.skip(reader.read_size()); + } + } + reader.skip(sizeof(std::uint64_t)); + result.depth_selector_count = reader.read_size(); + result.depth_selectors = reader.position(); + reader.skip(result.depth_selector_count * 8 * sizeof(std::uint64_t)); + reader.skip(reader.read_size() * sizeof(std::uint64_t)); + const std::size_t min_depth_count = reader.read_size(); + result.depth_min_depths = reader.position(); + reader.skip(min_depth_count * sizeof(std::int64_t)); + return result; +} + +static void expect_serialized_rmq_ranges(const SerializableRmq& original, + const SerializableRmq& restored, + std::span values, + bool exhaustive) { + if (exhaustive) { + check_all_ranges(restored, values, std::less()); + return; + } + + const std::array, 10> boundaries = { + std::pair{std::size_t{0}, values.size()}, + std::pair{std::size_t{0}, std::min(512, values.size())}, + std::pair{std::min(511, values.size() - 1), + std::min(513, values.size())}, + std::pair{std::min(4095, values.size() - 1), + std::min(4097, values.size())}, + std::pair{values.size() / 3, values.size()}, + std::pair{values.size() / 4, 3 * values.size() / 4}, + std::pair{values.size() - 1, values.size()}, + std::pair{std::size_t{1}, values.size()}, + std::pair{values.size() / 2, values.size() / 2 + 1}, + std::pair{std::size_t{0}, std::size_t{1}}}; + for (const auto& [left, right] : boundaries) { + if (left >= right) { + continue; + } + const std::size_t expected = + naive_arg_min(values, left, right, std::less()); + EXPECT_EQ(original.arg_min(left, right), expected); + EXPECT_EQ(restored.arg_min(left, right), expected); + EXPECT_EQ(restored.range_min(left, right), values[expected]); + } + + std::mt19937_64 rng(20260718); + std::uniform_int_distribution position(0, values.size() - 1); + for (std::size_t query = 0; query < 2000; ++query) { + std::size_t left = position(rng); + std::size_t right = position(rng); + if (left > right) { + std::swap(left, right); + } + ++right; + const std::size_t expected = + naive_arg_min(values, left, right, std::less()); + EXPECT_EQ(restored.arg_min(left, right), expected); + } +} + +static void expect_rmq_serialization_round_trip( + std::size_t size, + pixie::DeserializationValidation validation) { + std::vector values(size); + for (std::size_t i = 0; i < size; ++i) { + values[i] = static_cast((i * 37 + i / 5 + i / 97) % 31) - 15; + } + if (size > 3) { + values[size / 3] = -100; + values[size / 3 + 1] = -100; + } + + const SerializableRmq original(values); + std::vector artifact = serialize_rmq(original); + pixie::BinaryReader reader(artifact); + const SerializableRmq restored = + SerializableRmq::deserialize(values, reader, validation); + + EXPECT_TRUE(reader.empty()); + EXPECT_EQ(restored.size(), values.size()); + EXPECT_EQ(restored.bp_bit_count(), original.bp_bit_count()); + EXPECT_EQ(restored.bp_words().size(), original.bp_words().size()); + EXPECT_EQ(serialize_rmq(restored), artifact); + + artifact.clear(); + artifact.shrink_to_fit(); + if (values.empty()) { + EXPECT_TRUE(restored.empty()); + EXPECT_EQ(restored.arg_min(0, 0), SerializableRmq::npos); + return; + } + expect_serialized_rmq_ranges(original, restored, values, size <= 513); +} + +TEST(RmqSerializationTest, RoundTripsOwningMetadataAtBoundaries) { + for (const std::size_t size : std::array{ + 0, 1, 255, 256, 257, 511, 512, 513, 4095, 4096, 4097}) { + for (const auto validation : {pixie::DeserializationValidation::kQuick, + pixie::DeserializationValidation::kFull}) { + SCOPED_TRACE(::testing::Message() << "size=" << size << " validation=" + << static_cast(validation)); + expect_rmq_serialization_round_trip(size, validation); + } + } +} + +TEST(RmqSerializationTest, AdvancesAcrossConcatenatedArtifacts) { + std::vector values(33); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] = static_cast((i * 11) % 17); + } + const SerializableRmq original(values); + pixie::VectorOutputSink output; + pixie::BinaryWriter writer(output); + original.serialize(writer); + original.serialize(writer); + writer.finish(); + const auto artifacts = output.take(); + pixie::BinaryReader reader(artifacts); + + const auto first = SerializableRmq::deserialize(values, reader); + EXPECT_FALSE(reader.empty()); + const auto second = SerializableRmq::deserialize(values, reader); + EXPECT_TRUE(reader.empty()); + EXPECT_EQ(first.arg_min(0, values.size()), second.arg_min(0, values.size())); +} + +TEST(RmqSerializationTest, SerializesDirectlyToMappedFile) { + std::vector values(65); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] = static_cast((i * 19) % 23) - 11; + } + const SerializableRmq original(values); + const auto path = std::filesystem::temp_directory_path() / + "pixie_rmq_serialization_test.bin"; + std::filesystem::remove(path); + { + pixie::io::FileOutputSink output(path); + std::array staging{}; + pixie::BinaryWriter writer(output, staging); + original.serialize(writer); + writer.finish(); + } + + pixie::io::MappedFile file(path); + for (const auto validation : {pixie::DeserializationValidation::kQuick, + pixie::DeserializationValidation::kFull}) { + pixie::BinaryReader reader(file.as_bytes()); + const SerializableRmq restored = + SerializableRmq::deserialize(values, reader, validation); + EXPECT_TRUE(reader.empty()); + check_all_ranges(restored, std::span(values), + std::less()); + } + std::filesystem::remove(path); +} + +TEST(RmqSerializationTest, RejectsCorruptionWithoutAdvancingInput) { + std::vector values(40); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] = static_cast((i * 13) % 23); + } + const SerializableRmq original(values); + const auto valid = serialize_rmq(original); + + const auto expect_rejected = [&](std::vector artifact) { + for (const auto validation : {pixie::DeserializationValidation::kQuick, + pixie::DeserializationValidation::kFull}) { + pixie::BinaryReader reader(artifact); + EXPECT_THROW( + (void)SerializableRmq::deserialize(values, reader, validation), + std::exception); + EXPECT_EQ(reader.position(), 0u); + } + }; + const auto overwrite = [](std::vector& artifact, + std::size_t offset, auto value) { + using Value = decltype(value); + using Unsigned = std::make_unsigned_t; + const Unsigned encoded = static_cast(value); + for (std::size_t byte = 0; byte < sizeof(Value); ++byte) { + artifact[offset + byte] = + static_cast((encoded >> (byte * 8)) & Unsigned{0xff}); + } + }; + + auto bad_magic = valid; + bad_magic[0] ^= std::byte{1}; + expect_rejected(std::move(bad_magic)); + + auto bad_version = valid; + overwrite(bad_version, 8, std::uint32_t{2}); + expect_rejected(std::move(bad_version)); + + auto bad_leaf_size = valid; + overwrite(bad_leaf_size, 32, std::uint64_t{1024}); + expect_rejected(std::move(bad_leaf_size)); + + auto bad_bp_count = valid; + overwrite(bad_bp_count, 48, std::uint64_t{0}); + expect_rejected(std::move(bad_bp_count)); + + auto impossible_storage = valid; + overwrite(impossible_storage, 80, std::numeric_limits::max()); + expect_rejected(std::move(impossible_storage)); + + std::span truncated = + std::span(valid).first(valid.size() - 1); + pixie::BinaryReader truncated_reader(truncated); + EXPECT_THROW((void)SerializableRmq::deserialize(values, truncated_reader), + std::invalid_argument); + EXPECT_EQ(truncated_reader.position(), 0u); + + std::span short_values(values.data(), values.size() - 1); + pixie::BinaryReader reader(valid); + EXPECT_THROW((void)SerializableRmq::deserialize(short_values, reader), + std::invalid_argument); + EXPECT_EQ(reader.position(), 0u); + + std::vector different_values(values.size(), -1); + pixie::BinaryReader different_reader(valid); + EXPECT_NO_THROW( + (void)SerializableRmq::deserialize(different_values, different_reader)); + EXPECT_TRUE(different_reader.empty()); + + pixie::BinaryReader different_full_reader(valid); + EXPECT_THROW((void)SerializableRmq::deserialize( + different_values, different_full_reader, + pixie::DeserializationValidation::kFull), + std::invalid_argument); + EXPECT_EQ(different_full_reader.position(), 0u); +} + +TEST(RmqSerializationTest, RejectsMalformedBpEncodingAndPadding) { + constexpr std::size_t kBpDataOffset = 88; + const auto set_bit = [](std::vector& artifact, + std::size_t position) { + artifact[kBpDataOffset + position / 8] |= + static_cast(std::uint8_t{1} << (position % 8)); + }; + const auto clear_bit = [](std::vector& artifact, + std::size_t position) { + artifact[kBpDataOffset + position / 8] &= + static_cast(~(std::uint8_t{1} << (position % 8))); + }; + + for (const std::size_t size : + {std::size_t{63}, std::size_t{64}, std::size_t{65}}) { + SCOPED_TRACE(::testing::Message() << "size=" << size); + std::vector values(size); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] = static_cast((i * 13 + i / 3) % 17); + } + const SerializableRmq original(values); + const std::vector valid = serialize_rmq(original); + const auto expect_rejected = [&](std::vector artifact) { + pixie::BinaryReader reader(artifact); + EXPECT_THROW((void)SerializableRmq::deserialize( + values, reader, pixie::DeserializationValidation::kFull), + std::invalid_argument); + EXPECT_EQ(reader.position(), 0u); + }; + + auto negative_first_prefix = valid; + clear_bit(negative_first_prefix, 0); + expect_rejected(std::move(negative_first_prefix)); + + auto nonzero_final_excess = valid; + set_bit(nonzero_final_excess, 2 * size - 1); + expect_rejected(std::move(nonzero_final_excess)); + + auto nonzero_padding = valid; + set_bit(nonzero_padding, 2 * size); + expect_rejected(std::move(nonzero_padding)); + } + + constexpr std::size_t kLateViolationSize = 129; + std::vector values(kLateViolationSize); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] = static_cast((i * 19 + i / 7) % 23); + } + const SerializableRmq original(values); + std::vector negative_later = serialize_rmq(original); + for (std::size_t position = 128; position < 2 * values.size(); ++position) { + clear_bit(negative_later, position); + } + pixie::BinaryReader reader(negative_later); + EXPECT_THROW((void)SerializableRmq::deserialize( + values, reader, pixie::DeserializationValidation::kFull), + std::invalid_argument); + EXPECT_EQ(reader.position(), 0u); +} + +TEST(RmqSerializationTest, RejectsMalformedIndexMetadataTransactionally) { + std::vector values(4097); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] = static_cast((i * 17 + i / 11) % 29); + } + const SerializableRmq original(values); + const std::vector valid = serialize_rmq(original); + const RmqArtifactLayout layout = locate_rmq_artifact(valid); + ASSERT_EQ(layout.candidate_size, 4u); + ASSERT_EQ(layout.depth_selector_count, 1u); + + const auto expect_rejected = [&](std::vector artifact) { + for (const auto validation : {pixie::DeserializationValidation::kQuick, + pixie::DeserializationValidation::kFull}) { + pixie::BinaryReader reader(artifact); + EXPECT_THROW( + (void)SerializableRmq::deserialize(values, reader, validation), + std::invalid_argument); + EXPECT_EQ(reader.position(), 0u); + } + }; + + auto misaligned_storage = valid; + overwrite_u64(misaligned_storage, layout.bp_storage_size, + layout.bp_storage_bytes - 1); + expect_rejected(std::move(misaligned_storage)); + + auto insufficient_storage = valid; + overwrite_u64(insufficient_storage, layout.bp_bit_count, + layout.bp_storage_bytes * 8 + 64); + expect_rejected(std::move(insufficient_storage)); + + auto wrong_bp_bit_count = valid; + overwrite_u64(wrong_bp_bit_count, layout.bp_bit_count, 2 * values.size() - 1); + expect_rejected(std::move(wrong_bp_bit_count)); + + auto wrong_top_block_size = valid; + overwrite_u64(wrong_top_block_size, layout.top_block_size, 1); + expect_rejected(std::move(wrong_top_block_size)); + + auto wrong_top_block_count = valid; + overwrite_u64(wrong_top_block_count, layout.top_block_count, 0); + expect_rejected(std::move(wrong_top_block_count)); + + auto wrong_top_sparse_levels = valid; + overwrite_u64(wrong_top_sparse_levels, layout.top_sparse_levels, 0); + expect_rejected(std::move(wrong_top_sparse_levels)); + + auto invalid_marker = valid; + invalid_marker[layout.has_rank_index] = std::byte{2}; + expect_rejected(std::move(invalid_marker)); + + auto invalid_candidate = valid; + overwrite_u64(invalid_candidate, layout.candidates, values.size()); + expect_rejected(std::move(invalid_candidate)); + + auto invalid_sparse_padding = valid; + overwrite_u64(invalid_sparse_padding, + layout.candidates + 3 * sizeof(std::uint64_t), 0); + expect_rejected(std::move(invalid_sparse_padding)); + + auto wrong_exact_block_minimum = valid; + const std::size_t original_candidate = + pixie::BinaryReader(std::span(valid).subspan( + layout.candidates, sizeof(std::uint64_t))) + .read_size(); + const std::size_t alternate_candidate = + original_candidate == 0 ? 1 : original_candidate - 1; + overwrite_u64(wrong_exact_block_minimum, layout.candidates, + alternate_candidate); + pixie::BinaryReader exact_quick_reader(wrong_exact_block_minimum); + EXPECT_NO_THROW((void)SerializableRmq::deserialize( + values, exact_quick_reader, pixie::DeserializationValidation::kQuick)); + EXPECT_TRUE(exact_quick_reader.empty()); + pixie::BinaryReader exact_full_reader(wrong_exact_block_minimum); + EXPECT_THROW( + (void)SerializableRmq::deserialize( + values, exact_full_reader, pixie::DeserializationValidation::kFull), + std::invalid_argument); + EXPECT_EQ(exact_full_reader.position(), 0u); + + auto wrong_depth_selector = valid; + wrong_depth_selector[layout.depth_selectors] ^= std::byte{1}; + pixie::BinaryReader selector_quick_reader(wrong_depth_selector); + EXPECT_NO_THROW((void)SerializableRmq::deserialize( + values, selector_quick_reader, pixie::DeserializationValidation::kQuick)); + EXPECT_TRUE(selector_quick_reader.empty()); + pixie::BinaryReader selector_full_reader(wrong_depth_selector); + EXPECT_THROW((void)SerializableRmq::deserialize( + values, selector_full_reader, + pixie::DeserializationValidation::kFull), + std::invalid_argument); + EXPECT_EQ(selector_full_reader.position(), 0u); + + auto wrong_depth_minimum = valid; + overwrite_u64(wrong_depth_minimum, layout.depth_min_depths, + std::numeric_limits::max()); + pixie::BinaryReader depth_full_reader(wrong_depth_minimum); + EXPECT_THROW( + (void)SerializableRmq::deserialize( + values, depth_full_reader, pixie::DeserializationValidation::kFull), + std::invalid_argument); + EXPECT_EQ(depth_full_reader.position(), 0u); + + const std::vector empty_values; + const SerializableRmq empty(empty_values); + std::vector invalid_empty = serialize_rmq(empty); + const RmqArtifactLayout empty_layout = locate_rmq_artifact(invalid_empty); + overwrite_u64(invalid_empty, empty_layout.top_block_size, 1); + pixie::BinaryReader empty_reader(invalid_empty); + EXPECT_THROW((void)SerializableRmq::deserialize(empty_values, empty_reader), + std::invalid_argument); + EXPECT_EQ(empty_reader.position(), 0u); +} + template class ValueRmqSpecificationTest : public ::testing::Test {}; @@ -501,7 +1008,7 @@ TEST(RmqHybridBTree, BoundaryAndFallbackRanges) { {0, values.size()}, }; - for (const auto [left, right] : ranges) { + for (const auto& [left, right] : ranges) { const std::size_t expected = naive_arg_min(std::span(values), left, right, std::less()); EXPECT_EQ(rmq.arg_min(left, right), expected) @@ -610,7 +1117,7 @@ TEST(RmqHybridBTree, LeafSelectorEnumVariants) { {3000, 4099}, }; - for (const auto [left, right] : ranges) { + for (const auto& [left, right] : ranges) { const std::size_t expected = naive_arg_min(std::span(values), left, right, std::less()); EXPECT_EQ(mask_rmq.arg_min(left, right), expected) @@ -650,7 +1157,7 @@ TEST(RmqHybridBTree, LeafSelectorEnumVariants) { {kLeaf, 2 * kLeaf}, {2 * kLeaf, 3 * kLeaf}, {1, small_values.size() - 1}, {kLeaf + 1, kLeaf + 2}, }; - for (const auto [left, right] : small_ranges) { + for (const auto& [left, right] : small_ranges) { const std::size_t expected = naive_arg_min( std::span(small_values), left, right, std::less()); EXPECT_EQ(rmq.arg_min(left, right), expected) @@ -741,7 +1248,7 @@ TEST(RmqHybridBTree, MiddleFanoutBoundaryRanges) { {values.size() - 3 * kLeaf, values.size()}, }; - for (const auto [left, right] : ranges) { + for (const auto& [left, right] : ranges) { const std::size_t expected = naive_arg_min(std::span(values), left, right, std::less()); EXPECT_EQ(rmq.arg_min(left, right), expected) @@ -775,7 +1282,7 @@ TEST(RmqHybridBTree, TopSparseOverlayBoundaryRanges) { {0, values.size()}, }; - for (const auto [left, right] : ranges) { + for (const auto& [left, right] : ranges) { const std::size_t expected = naive_arg_min(std::span(values), left, right, std::less()); EXPECT_EQ(rmq.arg_min(left, right), expected) @@ -832,7 +1339,7 @@ TEST(RmqHybridBTree, TopSparseOverlayComparatorMaximum) { {0, values.size()}, }; - for (const auto [left, right] : ranges) { + for (const auto& [left, right] : ranges) { const std::size_t expected = naive_arg_min( std::span(values), left, right, std::greater()); EXPECT_EQ(rmq.arg_min(left, right), expected) @@ -921,7 +1428,7 @@ TEST(RmqCartesianHybridBTree, BoundarySizesAndBpEncoding) { {size / 3, std::min(size, size / 3 + 19)}, {size / 2, std::min(size, size / 2 + 37)}, }; - for (const auto [left, right] : ranges) { + for (const auto& [left, right] : ranges) { ASSERT_LT(left, right); EXPECT_EQ(rmq.arg_min(left, right), naive_arg_min(std::span(values), left, right, @@ -1033,7 +1540,7 @@ TEST(RmqCartesianHybridBTree, DepthBackendDirectPaths) { {700, kDepthCount - 3}, {kDepthCount - 65, kDepthCount}, }; - for (const auto [left, right] : ranges) { + for (const auto& [left, right] : ranges) { const std::size_t expected = naive_depth_arg_min(std::span(depths), left, right); EXPECT_EQ(high_sparse.arg_min(left, right), expected) @@ -1112,7 +1619,7 @@ TEST(RmqCartesianHybridBTree, TopSparseOverlayBoundaryRanges) { {0, values.size()}, }; - for (const auto [left, right] : ranges) { + for (const auto& [left, right] : ranges) { ASSERT_LT(left, right); const std::size_t expected = naive_arg_min(std::span(values), left, right, std::less()); @@ -1171,7 +1678,7 @@ TEST(RmqCartesianHybridBTree, TopSparseOverlayComparatorMaximum) { {0, values.size()}, }; - for (const auto [left, right] : ranges) { + for (const auto& [left, right] : ranges) { ASSERT_LT(left, right); EXPECT_EQ(rmq.arg_min(left, right), naive_arg_min(std::span(values), left, right, diff --git a/src/tests/serialization_tests.cpp b/src/tests/serialization_tests.cpp new file mode 100644 index 0000000..67d143d --- /dev/null +++ b/src/tests/serialization_tests.cpp @@ -0,0 +1,458 @@ +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +TEST(BinarySerializationTest, UsesCanonicalLittleEndianIntegerEncoding) { + pixie::VectorOutputSink output; + std::array staging{}; + pixie::BinaryWriter writer(output, staging); + writer.write_u8(0x12); + writer.write_u16(0x3456); + writer.write_u32(0x789abcde); + writer.write_u64(0x0123456789abcdef); + writer.write_i32(-2); + writer.finish(); + + const std::array expected = { + std::byte{0x12}, std::byte{0x56}, std::byte{0x34}, std::byte{0xde}, + std::byte{0xbc}, std::byte{0x9a}, std::byte{0x78}, std::byte{0xef}, + std::byte{0xcd}, std::byte{0xab}, std::byte{0x89}, std::byte{0x67}, + std::byte{0x45}, std::byte{0x23}, std::byte{0x01}, std::byte{0xfe}, + std::byte{0xff}, std::byte{0xff}, std::byte{0xff}}; + EXPECT_TRUE(std::ranges::equal(output.bytes(), expected)); + + pixie::BinaryReader reader(output.bytes()); + EXPECT_EQ(reader.read_u8(), 0x12); + EXPECT_EQ(reader.read_u16(), 0x3456); + EXPECT_EQ(reader.read_u32(), 0x789abcdeu); + EXPECT_EQ(reader.read_u64(), 0x0123456789abcdefu); + EXPECT_EQ(reader.read_i32(), -2); + EXPECT_TRUE(reader.empty()); +} + +TEST(BinarySerializationTest, RoundTripsEverySignedIntegerWidth) { + pixie::VectorOutputSink output; + pixie::BinaryWriter writer(output); + writer.write_i8(-2); + writer.write_i16(-3); + writer.write_i32(-4); + writer.write_i64(-5); + writer.finish(); + + pixie::BinaryReader reader(output.bytes()); + EXPECT_EQ(reader.read_i8(), -2); + EXPECT_EQ(reader.read_i16(), -3); + EXPECT_EQ(reader.read_i32(), -4); + EXPECT_EQ(reader.read_i64(), -5); + EXPECT_TRUE(reader.empty()); +} + +TEST(BinarySerializationTest, WritesBytesFramesAndPadding) { + const std::array payload = {std::byte{1}, std::byte{2}, std::byte{3}}; + std::array storage{}; + std::array staging{}; + pixie::SpanOutputSink output(storage); + pixie::BinaryWriter writer(output, staging); + const std::size_t size_position = writer.write_u64_placeholder(); + writer.write_bytes(payload); + writer.align_to(8); + writer.patch_u64(size_position, writer.size_bytes()); + writer.finish(); + + ASSERT_EQ(writer.size_bytes(), 16u); + EXPECT_EQ(output.size_bytes(), 16u); + pixie::BinaryReader reader(output.bytes()); + EXPECT_EQ(reader.read_u64(), 16u); + EXPECT_TRUE(std::ranges::equal(reader.read_bytes(payload.size()), payload)); + reader.require_zero_padding(5); + EXPECT_TRUE(reader.empty()); +} + +TEST(BinarySerializationTest, BackpatchesAFieldSplitByABufferBoundary) { + pixie::VectorOutputSink output; + std::array staging{}; + pixie::BinaryWriter writer(output, staging); + writer.write_u8(0xaa); + const std::size_t size_position = writer.write_u64_placeholder(); + writer.write_u16(0x1234); + writer.patch_u64(size_position, writer.size_bytes()); + writer.finish(); + + pixie::BinaryReader reader(output.bytes()); + EXPECT_EQ(reader.read_u8(), 0xaa); + EXPECT_EQ(reader.read_u64(), 11u); + EXPECT_EQ(reader.read_u16(), 0x1234); + EXPECT_TRUE(reader.empty()); +} + +TEST(BinarySerializationTest, ExplicitVectorSinkCanBeTakenAndReused) { + pixie::VectorOutputSink output; + { + pixie::BinaryWriter writer(output); + writer.write_u32(42); + writer.finish(); + } + const std::vector first = output.take(); + EXPECT_EQ(first.size(), sizeof(std::uint32_t)); + EXPECT_TRUE(output.empty()); + + { + pixie::BinaryWriter writer(output); + writer.write_u8(7); + writer.finish(); + } + const std::vector second = output.take(); + ASSERT_EQ(second.size(), 1u); + EXPECT_EQ(second[0], std::byte{7}); +} + +TEST(BinarySerializationTest, ReaderReportsOffsetsWithoutPartialPrimitiveRead) { + const std::array bytes = {std::byte{0}, std::byte{0}, std::byte{0}, + std::byte{0}, std::byte{1}, std::byte{2}}; + pixie::BinaryReader reader(bytes); + reader.skip(4); + try { + (void)reader.read_u32(); + FAIL() << "Expected a truncated-input error"; + } catch (const pixie::SerializationError& error) { + EXPECT_EQ(error.byte_offset(), 4u); + } + EXPECT_EQ(reader.position(), 4u); +} + +TEST(BinarySerializationTest, SubreaderIsBoundedAndTracksAbsoluteOffsets) { + pixie::VectorOutputSink output; + pixie::BinaryWriter writer(output); + writer.write_u32(1); + writer.write_u16(2); + writer.write_u16(3); + writer.finish(); + pixie::BinaryReader reader(output.bytes()); + reader.skip(sizeof(std::uint32_t)); + pixie::BinaryReader child = reader.read_subreader(sizeof(std::uint16_t)); + + EXPECT_EQ(child.read_u16(), 2); + EXPECT_TRUE(child.empty()); + EXPECT_EQ(reader.position(), 6u); + EXPECT_EQ(reader.read_u16(), 3); + EXPECT_TRUE(reader.empty()); +} + +TEST(BinarySerializationTest, RejectsBadPaddingAndInvalidPatch) { + const std::array bytes = {std::byte{0}, std::byte{1}}; + pixie::BinaryReader reader(bytes); + EXPECT_THROW(reader.require_zero_padding(2), pixie::SerializationError); + EXPECT_EQ(reader.position(), 0u); + + pixie::VectorOutputSink output; + pixie::BinaryWriter writer(output); + writer.write_u32(0); + EXPECT_THROW(writer.patch_u64(0, 1), std::out_of_range); + EXPECT_THROW(writer.align_to(0), std::invalid_argument); + writer.finish(); +} + +TEST(BinarySerializationTest, RejectsExcessivePaddingTransactionally) { + const std::array bytes = {std::byte{0}, std::byte{0}}; + pixie::BinaryReader reader(bytes); + EXPECT_THROW(reader.require_zero_padding(1), pixie::SerializationError); + EXPECT_EQ(reader.position(), 0u); +} + +TEST(BinarySerializationTest, OutputSinksCheckDirectWritesAndPatches) { + const std::array initial = {std::byte{1}, std::byte{2}, std::byte{3}}; + const std::array patch = {std::byte{7}, std::byte{8}}; + + pixie::VectorOutputSink vector_output; + vector_output.reserve_bytes(16); + vector_output.write(initial); + vector_output.write_at(1, patch); + vector_output.write_at(vector_output.size_bytes(), {}); + EXPECT_THROW(vector_output.write_at(2, patch), std::out_of_range); + EXPECT_THROW(vector_output.write_at(4, {}), std::out_of_range); + vector_output.finish(); + EXPECT_EQ(vector_output.bytes()[0], std::byte{1}); + EXPECT_EQ(vector_output.bytes()[1], std::byte{7}); + EXPECT_EQ(vector_output.bytes()[2], std::byte{8}); + + std::array storage{}; + pixie::SpanOutputSink span_output(storage); + EXPECT_TRUE(span_output.empty()); + EXPECT_EQ(span_output.capacity_bytes(), storage.size()); + span_output.write(initial); + span_output.write_at(1, patch); + EXPECT_THROW(span_output.write(patch), std::length_error); + EXPECT_THROW(span_output.write_at(2, patch), std::out_of_range); + span_output.finish(); + EXPECT_EQ(span_output.bytes()[1], std::byte{7}); + EXPECT_EQ(span_output.bytes()[2], std::byte{8}); +} + +TEST(BinarySerializationTest, RejectsEmptyStagingBuffers) { + pixie::VectorOutputSink output; + EXPECT_THROW((void)pixie::BinaryWriter(output, std::size_t{0}), + std::invalid_argument); + EXPECT_THROW((void)pixie::BinaryWriter(output, std::span{}), + std::invalid_argument); +} + +TEST(BinarySerializationTest, FixedSpanReportsCapacityExhaustion) { + std::array storage{}; + std::array staging{}; + pixie::SpanOutputSink output(storage); + pixie::BinaryWriter writer(output, staging); + writer.write_u32(42); + writer.write_u8(7); + EXPECT_THROW(writer.finish(), std::length_error); + EXPECT_EQ(output.size_bytes(), sizeof(std::uint32_t)); + EXPECT_THROW(writer.write_u8(1), std::logic_error); +} + +class RecordingOutputSink { + public: + void write(std::span bytes) { + maximum_write_size = std::max(maximum_write_size, bytes.size()); + size_bytes += bytes.size(); + } + + void write_at(std::size_t position, std::span bytes) { + if (position > size_bytes || bytes.size() > size_bytes - position) { + throw std::out_of_range("Recording sink patch is outside the output"); + } + } + + void finish() { finished = true; } + + std::size_t size_bytes = 0; + std::size_t maximum_write_size = 0; + bool finished = false; +}; + +static_assert(pixie::SeekableBinaryOutputSink); + +TEST(BinarySerializationTest, UsesOnlyTheCallerProvidedBufferForGeneratedData) { + RecordingOutputSink output; + std::array staging{}; + pixie::BinaryWriter writer(output, staging); + writer.write_zeros(1024 * 1024); + writer.finish(); + + EXPECT_EQ(writer.buffer_size_bytes(), staging.size()); + EXPECT_EQ(output.size_bytes, 1024u * 1024u); + EXPECT_LE(output.maximum_write_size, staging.size()); + EXPECT_TRUE(output.finished); +} + +class FailingOutputSink { + public: + void write(std::span) { + throw std::runtime_error("injected write failure"); + } + + void write_at(std::size_t, std::span) { + throw std::runtime_error("injected patch failure"); + } + + void finish() {} +}; + +class PatchFailingOutputSink { + public: + void write(std::span bytes) { size_bytes += bytes.size(); } + + void write_at(std::size_t, std::span) { + throw std::runtime_error("injected patch failure"); + } + + void finish() {} + + std::size_t size_bytes = 0; +}; + +class FinishFailingOutputSink { + public: + void write(std::span bytes) { size_bytes += bytes.size(); } + + void write_at(std::size_t, std::span) {} + + void finish() { throw std::runtime_error("injected finish failure"); } + + std::size_t size_bytes = 0; +}; + +TEST(BinarySerializationTest, BecomesUnusableAfterSinkFailure) { + FailingOutputSink output; + std::array staging{}; + pixie::BinaryWriter writer(output, staging); + EXPECT_THROW(writer.write_u8(1), std::runtime_error); + EXPECT_THROW(writer.write_u8(2), std::logic_error); + EXPECT_THROW(writer.finish(), std::logic_error); +} + +TEST(BinarySerializationTest, BecomesUnusableAfterPatchOrFinishFailure) { + PatchFailingOutputSink patch_output; + std::array patch_staging{}; + pixie::BinaryWriter patch_writer(patch_output, patch_staging); + patch_writer.write_u64(0); + ASSERT_EQ(patch_output.size_bytes, sizeof(std::uint64_t)); + EXPECT_THROW(patch_writer.patch_u64(0, 1), std::runtime_error); + EXPECT_THROW(patch_writer.flush(), std::logic_error); + + FinishFailingOutputSink finish_output; + pixie::BinaryWriter finish_writer(finish_output); + finish_writer.write_u8(1); + EXPECT_THROW(finish_writer.finish(), std::runtime_error); + EXPECT_EQ(finish_output.size_bytes, 1u); + EXPECT_THROW(finish_writer.finish(), std::logic_error); +} + +TEST(BinarySerializationTest, FlushDeliversBufferedDataAndKeepsWriterOpen) { + pixie::VectorOutputSink output; + std::array staging{}; + pixie::BinaryWriter writer(output, staging); + EXPECT_TRUE(writer.empty()); + writer.write_u16(0x1234); + EXPECT_TRUE(output.empty()); + writer.flush(); + EXPECT_EQ(output.size_bytes(), sizeof(std::uint16_t)); + writer.flush(); + writer.write_u8(5); + writer.finish(); + EXPECT_EQ(output.size_bytes(), 3u); +} + +TEST(BinarySerializationTest, FinishIsIdempotentAndClosesTheWriter) { + pixie::VectorOutputSink output; + pixie::BinaryWriter writer(output); + writer.write_u8(1); + writer.finish(); + EXPECT_NO_THROW(writer.finish()); + EXPECT_THROW(writer.write_u8(2), std::logic_error); +} + +TEST(BinarySerializationTest, WritesAndBackpatchesAFileWithBoundedMemory) { + const auto path = std::filesystem::temp_directory_path() / + "pixie_file_output_sink_test.bin"; + std::filesystem::remove(path); + { + pixie::io::FileOutputSink output(path); + std::array staging{}; + pixie::BinaryWriter writer(output, staging); + const std::size_t size_position = writer.write_u64_placeholder(); + writer.write_u32(0x12345678); + writer.align_to(sizeof(std::uint64_t)); + writer.patch_u64(size_position, writer.size_bytes()); + writer.finish(); + EXPECT_EQ(output.size_bytes(), 16u); + EXPECT_THROW(output.write(std::span{}), std::logic_error); + } + + pixie::io::MappedFile file(path); + pixie::BinaryReader reader(file.as_bytes()); + EXPECT_EQ(reader.read_u64(), 16u); + EXPECT_EQ(reader.read_u32(), 0x12345678u); + reader.require_zero_padding(4); + std::filesystem::remove(path); +} + +TEST(BinarySerializationTest, FileSinkSupportsMovesAndChecksItsState) { + const auto first_path = std::filesystem::temp_directory_path() / + "pixie_file_output_sink_move_test.bin"; + const auto second_path = std::filesystem::temp_directory_path() / + "pixie_file_output_sink_assignment_test.bin"; + std::filesystem::remove(first_path); + std::filesystem::remove(second_path); + const std::array bytes = {std::byte{1}, std::byte{2}, std::byte{3}}; + const std::array patch = {std::byte{9}}; + { + pixie::io::FileOutputSink first(first_path); + first.write(bytes); + pixie::io::FileOutputSink moved(std::move(first)); + EXPECT_THROW(first.write({}), std::logic_error); + + pixie::io::FileOutputSink assigned(second_path); + assigned = std::move(moved); + EXPECT_EQ(assigned.size_bytes(), bytes.size()); + assigned.write_at(1, patch); + EXPECT_THROW(assigned.write_at(3, patch), std::out_of_range); + assigned.finish(); + EXPECT_NO_THROW(assigned.finish()); + EXPECT_THROW(assigned.write({}), std::logic_error); + } + + pixie::io::MappedFile file(first_path); + ASSERT_EQ(file.as_bytes().size(), bytes.size()); + EXPECT_EQ(file.as_bytes()[1], std::byte{9}); + std::filesystem::remove(first_path); + std::filesystem::remove(second_path); +} + +TEST(BinarySerializationTest, FileSinkReportsOpenFailure) { + const auto missing_directory = + std::filesystem::temp_directory_path() / "pixie_missing_output_directory"; + std::filesystem::remove_all(missing_directory); + EXPECT_THROW( + (void)pixie::io::FileOutputSink(missing_directory / "artifact.bin"), + std::system_error); +} + +bool packed_bit(std::span words, std::size_t position) { + return ((words[position / 64] >> (position % 64)) & 1u) != 0; +} + +TEST(PackedBitBuilderTest, WritesAcrossEveryWordOffset) { + constexpr std::uint64_t kPattern = 0xfedcba9876543210; + for (std::size_t offset = 0; offset < 64; ++offset) { + for (const std::size_t width : std::array{8, 16, 32, 64}) { + SCOPED_TRACE(::testing::Message() + << "offset=" << offset << ", width=" << width); + pixie::PackedBitBuilder builder; + for (std::size_t bit = 0; bit < offset; ++bit) { + builder.write_bit(false); + } + builder.write_bits(kPattern, width); + EXPECT_EQ(builder.size_bits(), offset + width); + const std::vector words = builder.take_words(); + for (std::size_t bit = 0; bit < offset; ++bit) { + EXPECT_FALSE(packed_bit(words, bit)); + } + for (std::size_t bit = 0; bit < width; ++bit) { + EXPECT_EQ(packed_bit(words, offset + bit), + ((kPattern >> bit) & 1u) != 0); + } + EXPECT_EQ(builder.size_bits(), 0u); + } + } +} + +TEST(PackedBitBuilderTest, SupportsPartialFieldsAndSafeReuse) { + pixie::PackedBitBuilder builder; + builder.reserve_bits(129); + builder.write_bits(0xff, 4); + builder.write_bit(false); + const auto first = builder.take_words(); + ASSERT_EQ(first.size(), 1u); + EXPECT_EQ(first[0], 0xfu); + + builder.write_bit(true); + builder.write_bits(42, 0); + const auto second = builder.take_words(); + ASSERT_EQ(second.size(), 1u); + EXPECT_EQ(second[0], 1u); + EXPECT_THROW(builder.write_bits(0, 65), std::invalid_argument); +} + +} // namespace diff --git a/src/tests/storage_tests.cpp b/src/tests/storage_tests.cpp index aecee51..22b5528 100644 --- a/src/tests/storage_tests.cpp +++ b/src/tests/storage_tests.cpp @@ -7,7 +7,6 @@ #include #include #include -#include #include #include #include @@ -77,24 +76,29 @@ TYPED_TEST(StorageSpecificationTest, ProvidesAlignedWordViewsWhenValid) { TEST(StorageSerializationTest, OwningStorageAndViewSerializeIdentically) { pixie::AlignedStorage storage(1); storage.writable_bytes()[0] = std::byte{42}; - pixie::OutputBitStream owning_stream; - pixie::OutputBitStream view_stream; - storage.serialize(owning_stream); - storage.view().serialize(view_stream); - EXPECT_EQ(owning_stream.extract(), view_stream.extract()); + pixie::VectorOutputSink owning_output; + pixie::VectorOutputSink view_output; + pixie::BinaryWriter owning_writer(owning_output); + pixie::BinaryWriter view_writer(view_output); + storage.serialize(owning_writer); + storage.view().serialize(view_writer); + owning_writer.finish(); + view_writer.finish(); + EXPECT_EQ(owning_output.take(), view_output.take()); } TEST(StorageSerializationTest, ReadOnlyViewRoundTripsAndAdvancesInput) { pixie::AlignedStorage storage(1); storage.writable_bytes()[0] = std::byte{42}; - pixie::OutputBitStream stream; - storage.serialize(stream); - const auto serialized_words = stream.extract(); - std::span input = - std::as_bytes(std::span(serialized_words)); - const auto restored = pixie::ReadOnlyStorageView::deserialize(input); + pixie::VectorOutputSink output; + pixie::BinaryWriter writer(output); + storage.serialize(writer); + writer.finish(); + const auto serialized_data = output.take(); + pixie::BinaryReader reader(serialized_data); + const auto restored = pixie::ReadOnlyStorageView::deserialize(reader); EXPECT_TRUE(std::ranges::equal(restored.as_bytes(), storage.as_bytes())); - EXPECT_TRUE(input.empty()); + EXPECT_TRUE(reader.empty()); } TEST(AlignedStorageTest, PadsResizesAndProvidesWritableStorage) { @@ -117,12 +121,15 @@ TEST(ReadOnlyStorageViewTest, MutatingOperationsAreNotAvailable) { } TEST(ReadOnlyStorageViewTest, DeserializeRejectsTruncatedInput) { - std::array bytes{}; - const std::size_t payload_size = 1; - std::memcpy(bytes.data(), &payload_size, sizeof(payload_size)); - std::span input(bytes); - EXPECT_THROW(pixie::ReadOnlyStorageView::deserialize(input), + pixie::VectorOutputSink output; + pixie::BinaryWriter writer(output); + writer.write_size(1); + writer.finish(); + const auto bytes = output.take(); + pixie::BinaryReader reader(bytes); + EXPECT_THROW(pixie::ReadOnlyStorageView::deserialize(reader), std::invalid_argument); + EXPECT_EQ(reader.position(), 0u); } TEST(MappedFileTest, MapsContentsAndIsMoveOnly) { diff --git a/src/tests/test_rmm.cpp b/src/tests/test_rmm.cpp index f61beb9..9f00a10 100644 --- a/src/tests/test_rmm.cpp +++ b/src/tests/test_rmm.cpp @@ -1,16 +1,21 @@ #include +#include +#include #include #include #include #include +#include #include +#include #include #include #include #include #include #include +#include #include using std::size_t; @@ -24,15 +29,6 @@ static std::string bits_to_parens(const std::string& bits) { return s; } -static std::string vecbits_to_string(const std::vector& v) { - std::string s; - s.resize(v.size()); - for (size_t i = 0; i < v.size(); ++i) { - s[i] = v[i] ? '1' : '0'; - } - return s; -} - static std::vector pack_words_lsb_first( const std::string& bits) { const size_t n = bits.size(); @@ -540,6 +536,276 @@ static void expect_range_ops_equal(const pixie::RmMTree& rm, } } +static std::vector serialize_rmm(const pixie::RmMTree& tree) { + pixie::VectorOutputSink output; + pixie::BinaryWriter writer(output); + tree.serialize(writer); + writer.finish(); + return output.take(); +} + +static void overwrite_i32(std::vector& bytes, + std::size_t offset, + std::int32_t value) { + const std::uint32_t encoded = static_cast(value); + for (std::size_t byte = 0; byte < sizeof(value); ++byte) { + bytes[offset + byte] = + static_cast((encoded >> (byte * 8)) & 0xffu); + } +} + +static void expect_rmm_serialization_round_trip(const std::string& bits) { + auto words = pack_words_lsb_first(bits); + const pixie::RmMTree original(std::span(words), + bits.size(), + /*leaf_block_bits=*/128); + std::vector artifact = serialize_rmm(original); + pixie::BinaryReader reader(artifact); + const pixie::RmMTree restored = pixie::RmMTree::deserialize(words, reader); + + EXPECT_TRUE(reader.empty()); + EXPECT_EQ(serialize_rmm(restored), artifact); + artifact.clear(); + artifact.shrink_to_fit(); + + const NaiveRmM naive(bits); + expect_rank_select_equal(restored, naive, bits.size()); + expect_range_ops_equal(restored, naive, bits.size()); + if (!bits.empty()) { + const std::array deltas = {-9, -2, -1, 0, 1, 2, 9}; + const std::size_t step = std::max(1, bits.size() / 17); + for (std::size_t position = 0; position < bits.size(); position += step) { + EXPECT_EQ(restored.close(position), naive.close(position)); + EXPECT_EQ(restored.open(position), naive.open(position)); + EXPECT_EQ(restored.enclose(position), naive.enclose(position)); + for (const int delta : deltas) { + EXPECT_EQ(restored.fwdsearch(position, delta), + naive.fwdsearch(position, delta)); + EXPECT_EQ(restored.bwdsearch(position + 1, delta), + naive.bwdsearch(position + 1, delta)); + } + } + } +} + +TEST(RmMSerializationTest, RoundTripsOwningMetadataAtBoundaries) { + std::mt19937_64 rng(20260718); + for (const std::size_t size : + std::array{0, 1, 63, 64, 65, 127, 128, 129, 256, 777}) { + SCOPED_TRACE(::testing::Message() << "size=" << size); + expect_rmm_serialization_round_trip(random_bits(rng, size)); + } +} + +TEST(RmMSerializationTest, AdvancesAcrossConcatenatedArtifacts) { + const std::string bits = "11101001011000"; + auto words = pack_words_lsb_first(bits); + const pixie::RmMTree original(words, bits.size(), 64); + pixie::VectorOutputSink output; + pixie::BinaryWriter writer(output); + original.serialize(writer); + original.serialize(writer); + writer.finish(); + const auto artifacts = output.take(); + pixie::BinaryReader reader(artifacts); + + const auto first = pixie::RmMTree::deserialize(words, reader); + EXPECT_FALSE(reader.empty()); + const auto second = pixie::RmMTree::deserialize(words, reader); + EXPECT_TRUE(reader.empty()); + EXPECT_EQ(first.range_min_query_pos(0, bits.size() - 1), + second.range_min_query_pos(0, bits.size() - 1)); +} + +TEST(RmMSerializationTest, SerializesDirectlyToMappedFile) { + const std::string bits = "1110100101100011010010110010"; + auto words = pack_words_lsb_first(bits); + const pixie::RmMTree original(words, bits.size(), 64); + const auto path = std::filesystem::temp_directory_path() / + "pixie_rmm_serialization_test.bin"; + std::filesystem::remove(path); + { + pixie::io::FileOutputSink output(path); + std::array staging{}; + pixie::BinaryWriter writer(output, staging); + original.serialize(writer); + writer.finish(); + } + + pixie::io::MappedFile file(path); + for (const auto validation : {pixie::DeserializationValidation::kQuick, + pixie::DeserializationValidation::kFull}) { + pixie::BinaryReader reader(file.as_bytes()); + const pixie::RmMTree restored = + pixie::RmMTree::deserialize(words, reader, validation); + EXPECT_TRUE(reader.empty()); + EXPECT_EQ(restored.range_min_query_pos(0, bits.size() - 1), + original.range_min_query_pos(0, bits.size() - 1)); + EXPECT_EQ(restored.range_max_query_pos(0, bits.size() - 1), + original.range_max_query_pos(0, bits.size() - 1)); + } + std::filesystem::remove(path); +} + +TEST(RmMSerializationTest, + FullValidationRejectsSameLengthSourceMismatchTransactionally) { + const std::string bits = "1110100101100011010010110010"; + auto words = pack_words_lsb_first(bits); + const pixie::RmMTree original(words, bits.size(), 64); + const std::vector artifact = serialize_rmm(original); + std::vector different_words = words; + different_words[0] ^= std::uint64_t{1} << 7; + + pixie::BinaryReader quick_reader(artifact); + EXPECT_NO_THROW((void)pixie::RmMTree::deserialize( + different_words, quick_reader, pixie::DeserializationValidation::kQuick)); + EXPECT_TRUE(quick_reader.empty()); + + pixie::BinaryReader full_reader(artifact); + EXPECT_THROW((void)pixie::RmMTree::deserialize( + different_words, full_reader, + pixie::DeserializationValidation::kFull), + std::invalid_argument); + EXPECT_EQ(full_reader.position(), 0u); +} + +TEST(RmMSerializationTest, + FullValidationRejectsExactLeafCorruptionTransactionally) { + const std::string bits(64, '1'); + const auto words = pack_words_lsb_first(bits); + const pixie::RmMTree original(words, bits.size(), 64); + std::vector artifact = serialize_rmm(original); + + pixie::BinaryReader layout_reader(artifact); + layout_reader.skip(8 * sizeof(std::uint64_t)); + const std::size_t segment_count = layout_reader.read_size(); + layout_reader.skip(segment_count * sizeof(std::uint32_t)); + const std::size_t total_count = layout_reader.read_size(); + ASSERT_EQ(total_count, 2u); + const std::size_t total_data = layout_reader.position(); + overwrite_i32(artifact, total_data + sizeof(std::int32_t), 62); + + pixie::BinaryReader quick_reader(artifact); + EXPECT_NO_THROW((void)pixie::RmMTree::deserialize( + words, quick_reader, pixie::DeserializationValidation::kQuick)); + EXPECT_TRUE(quick_reader.empty()); + + pixie::BinaryReader full_reader(artifact); + EXPECT_THROW((void)pixie::RmMTree::deserialize( + words, full_reader, pixie::DeserializationValidation::kFull), + std::invalid_argument); + EXPECT_EQ(full_reader.position(), 0u); +} + +TEST(RmMSerializationTest, + FullValidationCoversBinaryUnaryAndEmptyInternalNodes) { + constexpr std::size_t kBitCount = 257; + const std::array inputs = { + std::string(kBitCount, '1'), std::string(kBitCount, '0'), [] { + std::string bits(kBitCount, '0'); + for (std::size_t position = 0; position < bits.size(); position += 2) { + bits[position] = '1'; + } + return bits; + }()}; + + for (const std::string& bits : inputs) { + const auto words = pack_words_lsb_first(bits); + const pixie::RmMTree original(words, bits.size(), 64); + const std::vector artifact = serialize_rmm(original); + pixie::BinaryReader reader(artifact); + const pixie::RmMTree restored = pixie::RmMTree::deserialize( + words, reader, pixie::DeserializationValidation::kFull); + EXPECT_TRUE(reader.empty()); + EXPECT_EQ(restored.range_min_query_pos(0, bits.size() - 1), + original.range_min_query_pos(0, bits.size() - 1)); + } +} + +TEST(RmMSerializationTest, + FullValidationRejectsExactInternalCorruptionTransactionally) { + const std::string bits(257, '1'); + const auto words = pack_words_lsb_first(bits); + const pixie::RmMTree original(words, bits.size(), 64); + std::vector artifact = serialize_rmm(original); + + pixie::BinaryReader layout_reader(artifact); + layout_reader.skip(8 * sizeof(std::uint64_t)); + const std::size_t segment_count = layout_reader.read_size(); + layout_reader.skip(segment_count * sizeof(std::uint32_t)); + const std::size_t total_count = layout_reader.read_size(); + ASSERT_EQ(total_count, 13u); + const std::size_t total_data = layout_reader.position(); + overwrite_i32(artifact, total_data + sizeof(std::int32_t), 255); + + pixie::BinaryReader quick_reader(artifact); + EXPECT_NO_THROW((void)pixie::RmMTree::deserialize( + words, quick_reader, pixie::DeserializationValidation::kQuick)); + EXPECT_TRUE(quick_reader.empty()); + + pixie::BinaryReader full_reader(artifact); + EXPECT_THROW((void)pixie::RmMTree::deserialize( + words, full_reader, pixie::DeserializationValidation::kFull), + std::invalid_argument); + EXPECT_EQ(full_reader.position(), 0u); +} + +TEST(RmMSerializationTest, RejectsCorruptionWithoutAdvancingInput) { + const std::string bits(257, '1'); + auto words = pack_words_lsb_first(bits); + const pixie::RmMTree original(words, bits.size(), 128); + const auto valid = serialize_rmm(original); + + const auto expect_rejected = [&](std::vector artifact) { + for (const auto validation : {pixie::DeserializationValidation::kQuick, + pixie::DeserializationValidation::kFull}) { + pixie::BinaryReader reader(artifact); + EXPECT_THROW((void)pixie::RmMTree::deserialize(words, reader, validation), + std::exception); + EXPECT_EQ(reader.position(), 0u); + } + }; + const auto overwrite = [](std::vector& artifact, + std::size_t offset, auto value) { + using Value = decltype(value); + using Unsigned = std::make_unsigned_t; + const Unsigned encoded = static_cast(value); + for (std::size_t byte = 0; byte < sizeof(Value); ++byte) { + artifact[offset + byte] = + static_cast((encoded >> (byte * 8)) & Unsigned{0xff}); + } + }; + + auto bad_magic = valid; + bad_magic[0] ^= std::byte{1}; + expect_rejected(std::move(bad_magic)); + + auto bad_version = valid; + overwrite(bad_version, 8, std::uint32_t{2}); + expect_rejected(std::move(bad_version)); + + auto bad_source_count = valid; + overwrite(bad_source_count, 24, std::uint64_t{bits.size() + 1}); + expect_rejected(std::move(bad_source_count)); + + auto impossible_vector = valid; + overwrite(impossible_vector, 64, std::numeric_limits::max()); + expect_rejected(std::move(impossible_vector)); + + std::span truncated = + std::span(valid).first(valid.size() - 1); + pixie::BinaryReader truncated_reader(truncated); + EXPECT_THROW((void)pixie::RmMTree::deserialize(words, truncated_reader), + std::invalid_argument); + EXPECT_EQ(truncated_reader.position(), 0u); + + std::span short_words; + pixie::BinaryReader reader(valid); + EXPECT_THROW((void)pixie::RmMTree::deserialize(short_words, reader), + std::invalid_argument); + EXPECT_EQ(reader.position(), 0u); +} + TEST(RmMEdgeCases, MultiwordPattern10AcrossWordBoundaries) { const size_t n = 640; std::string bits(n, '1'); diff --git a/src/tests/wavelet_tree_tests.cpp b/src/tests/wavelet_tree_tests.cpp index 03e900e..d9b18e7 100644 --- a/src/tests/wavelet_tree_tests.cpp +++ b/src/tests/wavelet_tree_tests.cpp @@ -1,11 +1,94 @@ #include +#include +#include #include #include +#include +#include +#include +#include +#include +#include #include +#include +#include using pixie::WaveletTree; +namespace { + +void overwrite_u64(std::vector& bytes, + std::size_t offset, + std::uint64_t value) { + for (std::size_t byte = 0; byte < sizeof(value); ++byte) { + bytes[offset + byte] = + static_cast((value >> (byte * 8)) & 0xffu); + } +} + +struct WaveletNodeOffsets { + std::size_t parent; + std::size_t left_child; + std::size_t right_child; + std::size_t middle; + std::size_t rank_num_bits; +}; + +struct WaveletArtifactOffsets { + std::size_t alphabet_size; + std::size_t data_size; + std::size_t root; + std::size_t node_count; + std::vector nodes; + std::size_t leaves; + std::size_t permutation; +}; + +WaveletArtifactOffsets locate_wavelet_artifact( + std::span artifact) { + pixie::BinaryReader reader(artifact); + reader.skip(3 * sizeof(std::uint64_t)); + const std::size_t alphabet_size = reader.position(); + const std::size_t alphabet_count = reader.read_size(); + const std::size_t data_size = reader.position(); + reader.skip(sizeof(std::uint64_t)); + const std::size_t root = reader.position(); + reader.skip(sizeof(std::uint64_t)); + const std::size_t node_count_offset = reader.position(); + const std::size_t node_count = reader.read_size(); + + const auto skip_storage = [&reader] { reader.skip(reader.read_size()); }; + std::vector nodes; + nodes.reserve(node_count); + for (std::size_t node = 0; node < node_count; ++node) { + const std::size_t parent = reader.position(); + reader.skip(sizeof(std::uint64_t)); + const std::size_t left_child = reader.position(); + reader.skip(sizeof(std::uint64_t)); + const std::size_t right_child = reader.position(); + reader.skip(sizeof(std::uint64_t)); + const std::size_t middle = reader.position(); + reader.skip(sizeof(std::uint64_t)); + nodes.push_back({parent, left_child, right_child, middle, 0}); + + skip_storage(); + nodes.back().rank_num_bits = reader.position(); + reader.skip(7 * sizeof(std::uint64_t) + 2 * sizeof(std::uint32_t) + + 8 * sizeof(std::uint64_t) + 32 * sizeof(std::uint16_t)); + for (std::size_t storage = 0; storage < 3; ++storage) { + skip_storage(); + } + } + const std::size_t leaves = reader.position(); + const std::size_t permutation = + leaves + alphabet_count * sizeof(std::uint64_t); + return {alphabet_size, data_size, root, node_count_offset, + std::move(nodes), leaves, permutation}; +} + +} // namespace + TEST(WaveletTreeTest, BasicSelect) { const std::vector data = {3, 2, 0, 3, 1, 1, 2}; size_t data_size = 7, alphabet_size = 4; @@ -156,34 +239,317 @@ TEST(WaveletTreeTest, SerializationSmoke) { pixie::WaveletTreeBuildType::Huffman}) { WaveletTree orig_tree(alphabet_size, data, build_type); - pixie::OutputBitStream bs; - orig_tree.serialize(bs); - std::vector serialized_data = bs.extract(); + pixie::VectorOutputSink output; + pixie::BinaryWriter writer(output); + orig_tree.serialize(writer); + writer.finish(); + std::vector serialized_data = output.take(); + for (const auto validation : {pixie::DeserializationValidation::kQuick, + pixie::DeserializationValidation::kFull}) { + pixie::BinaryReader reader(serialized_data); + auto view_tree = pixie::WaveletTreeView::deserialize(reader, validation); + EXPECT_TRUE(reader.empty()); + + for (size_t i = 0; i <= data_size; i += 16) { + uint64_t symb = data[i == data_size ? 0 : i]; + EXPECT_EQ(orig_tree.rank(symb, i), view_tree.rank(symb, i)); + } - std::span byte_span( - reinterpret_cast(serialized_data.data()), - serialized_data.size() * sizeof(uint64_t)); + std::vector count(alphabet_size, 0); + for (auto symb : data) { + count[symb]++; + } - auto view_tree = pixie::WaveletTreeView::deserialize(byte_span); + for (uint64_t symb = 0; symb < alphabet_size; symb++) { + for (uint64_t rank = 1; rank <= count[symb]; rank++) { + EXPECT_EQ(orig_tree.select(symb, rank), view_tree.select(symb, rank)); + } + } - for (size_t i = 0; i <= data_size; i += 16) { - uint64_t symb = data[i == data_size ? 0 : i]; - EXPECT_EQ(orig_tree.rank(symb, i), view_tree.rank(symb, i)); + auto orig_segment = orig_tree.get_segment(0, data_size); + auto view_segment = view_tree.get_segment(0, data_size); + EXPECT_EQ(orig_segment, view_segment); } + } +} - std::vector count(alphabet_size, 0); - for (auto symb : data) { - count[symb]++; - } +TEST(WaveletTreeTest, SerializationAdvancesAcrossFramedArtifacts) { + const std::vector data = {3, 2, 0, 3, 1, 1, 2}; + const WaveletTree original(4, data); + pixie::VectorOutputSink output; + pixie::BinaryWriter writer(output); + original.serialize(writer); + original.serialize(writer); + writer.finish(); + const std::vector artifacts = output.take(); + pixie::BinaryReader reader(artifacts); + + const auto first = pixie::WaveletTreeView::deserialize(reader); + EXPECT_FALSE(reader.empty()); + const auto second = pixie::WaveletTreeView::deserialize(reader); + EXPECT_TRUE(reader.empty()); + EXPECT_EQ(first.get_segment(0, data.size()), data); + EXPECT_EQ(second.get_segment(0, data.size()), data); +} - for (uint64_t symb = 0; symb < alphabet_size; symb++) { - for (uint64_t rank = 1; rank <= count[symb]; rank++) { - EXPECT_EQ(orig_tree.select(symb, rank), view_tree.select(symb, rank)); - } - } +TEST(WaveletTreeTest, SerializationRoundTripsAnEmptyTree) { + const std::vector data; + const WaveletTree original(0, data); + pixie::VectorOutputSink output; + pixie::BinaryWriter writer(output); + original.serialize(writer); + writer.finish(); + const std::vector artifact = output.take(); + pixie::BinaryReader reader(artifact); + const auto restored = pixie::WaveletTreeView::deserialize(reader); + + EXPECT_TRUE(reader.empty()); + EXPECT_TRUE(restored.empty()); + EXPECT_EQ(restored.rank(0, 0), 0u); + EXPECT_EQ(restored.select(0, 1), 0u); + EXPECT_TRUE(restored.get_segment(0, 0).empty()); +} + +TEST(WaveletTreeTest, SerializesDirectlyToMappedFile) { + const std::vector data = {3, 2, 0, 3, 1, 1, 2}; + const WaveletTree original(4, data); + const auto path = std::filesystem::temp_directory_path() / + "pixie_wavelet_serialization_test.bin"; + std::filesystem::remove(path); + { + pixie::io::FileOutputSink output(path); + std::array staging{}; + pixie::BinaryWriter writer(output, staging); + original.serialize(writer); + writer.finish(); + } + + pixie::io::MappedFile file(path); + for (const auto validation : {pixie::DeserializationValidation::kQuick, + pixie::DeserializationValidation::kFull}) { + pixie::BinaryReader reader(file.as_bytes()); + const auto restored = + pixie::WaveletTreeView::deserialize(reader, validation); + EXPECT_TRUE(reader.empty()); + EXPECT_EQ(restored.get_segment(0, data.size()), data); + } + std::filesystem::remove(path); +} + +TEST(WaveletTreeTest, + FullValidationRejectsLeafAndChildLengthCorruptionTransactionally) { + const std::vector data = {0, 1, 2, 3, 4, 5, 6, 7}; + const WaveletTree original(8, data); + pixie::VectorOutputSink output; + pixie::BinaryWriter writer(output); + original.serialize(writer); + writer.finish(); + const std::vector valid = output.take(); + const WaveletArtifactOffsets layout = locate_wavelet_artifact(valid); + + std::vector bad_leaf = valid; + overwrite_u64(bad_leaf, layout.leaves, 3); + pixie::BinaryReader leaf_quick_reader(bad_leaf); + EXPECT_NO_THROW((void)pixie::WaveletTreeView::deserialize( + leaf_quick_reader, pixie::DeserializationValidation::kQuick)); + EXPECT_TRUE(leaf_quick_reader.empty()); + pixie::BinaryReader leaf_full_reader(bad_leaf); + EXPECT_THROW((void)pixie::WaveletTreeView::deserialize( + leaf_full_reader, pixie::DeserializationValidation::kFull), + std::invalid_argument); + EXPECT_EQ(leaf_full_reader.position(), 0u); + + std::vector bad_child_length = valid; + overwrite_u64(bad_child_length, layout.nodes[1].rank_num_bits, 3); + pixie::BinaryReader child_full_reader(bad_child_length); + EXPECT_THROW((void)pixie::WaveletTreeView::deserialize( + child_full_reader, pixie::DeserializationValidation::kFull), + std::invalid_argument); + EXPECT_EQ(child_full_reader.position(), 0u); +} - auto orig_segment = orig_tree.get_segment(0, data_size); - auto view_segment = view_tree.get_segment(0, data_size); - EXPECT_EQ(orig_segment, view_segment); +TEST(WaveletTreeTest, SerializationRejectsEveryTruncatedPrefixTransactionally) { + const std::vector data = {3, 2, 0, 3, 1, 1, 2}; + const WaveletTree original(4, data); + pixie::VectorOutputSink output; + pixie::BinaryWriter writer(output); + original.serialize(writer); + writer.finish(); + const std::vector artifact = output.take(); + + for (std::size_t size = 0; size < artifact.size(); ++size) { + SCOPED_TRACE(::testing::Message() << "size=" << size); + pixie::BinaryReader reader( + std::span(artifact).first(size)); + EXPECT_THROW((void)pixie::WaveletTreeView::deserialize(reader), + std::invalid_argument); + EXPECT_EQ(reader.position(), 0u); } } + +TEST(WaveletTreeTest, SerializationRejectsUnalignedZeroCopyArtifacts) { + const std::vector data = {3, 2, 0, 3, 1, 1, 2}; + const WaveletTree original(4, data); + + pixie::VectorOutputSink unaligned_output; + pixie::BinaryWriter unaligned_writer(unaligned_output); + unaligned_writer.write_u8(0); + EXPECT_THROW(original.serialize(unaligned_writer), std::invalid_argument); + EXPECT_EQ(unaligned_writer.size_bytes(), 1u); + + pixie::VectorOutputSink output; + pixie::BinaryWriter writer(output); + original.serialize(writer); + writer.finish(); + const std::vector artifact = output.take(); + std::vector unaligned_artifact(artifact.size() + 1); + std::ranges::copy(artifact, unaligned_artifact.begin() + 1); + pixie::BinaryReader reader( + std::span(unaligned_artifact).subspan(1)); + EXPECT_THROW((void)pixie::WaveletTreeView::deserialize(reader), + std::invalid_argument); + EXPECT_EQ(reader.position(), 0u); +} + +TEST(WaveletTreeTest, SerializationRejectsMalformedTopologyTransactionally) { + constexpr std::uint64_t kNoNode = std::numeric_limits::max(); + const std::vector data = {0, 1, 2, 3, 4, 5, 6, 7}; + const WaveletTree original(8, data); + pixie::VectorOutputSink output; + pixie::BinaryWriter writer(output); + original.serialize(writer); + writer.finish(); + const std::vector valid = output.take(); + const auto layout = locate_wavelet_artifact(valid); + const auto& nodes = layout.nodes; + ASSERT_EQ(nodes.size(), 7u); + + const auto expect_rejected = [&](std::vector artifact) { + for (const auto validation : {pixie::DeserializationValidation::kQuick, + pixie::DeserializationValidation::kFull}) { + pixie::BinaryReader reader(artifact); + EXPECT_THROW( + (void)pixie::WaveletTreeView::deserialize(reader, validation), + std::invalid_argument); + EXPECT_EQ(reader.position(), 0u); + } + }; + + auto root_with_parent = valid; + overwrite_u64(root_with_parent, nodes[0].parent, 0); + expect_rejected(std::move(root_with_parent)); + + auto self_loop = valid; + overwrite_u64(self_loop, nodes[2].left_child, 2); + expect_rejected(std::move(self_loop)); + + auto inconsistent_parent = valid; + overwrite_u64(inconsistent_parent, nodes[1].parent, kNoNode); + expect_rejected(std::move(inconsistent_parent)); + + auto duplicate_parent = valid; + overwrite_u64(duplicate_parent, nodes[0].right_child, 1); + expect_rejected(std::move(duplicate_parent)); + + auto detached_node = valid; + overwrite_u64(detached_node, nodes[0].left_child, kNoNode); + overwrite_u64(detached_node, nodes[1].parent, kNoNode); + expect_rejected(std::move(detached_node)); + + auto disconnected_cycle = valid; + overwrite_u64(disconnected_cycle, nodes[0].left_child, kNoNode); + overwrite_u64(disconnected_cycle, nodes[1].parent, 2); + overwrite_u64(disconnected_cycle, nodes[2].left_child, 1); + expect_rejected(std::move(disconnected_cycle)); + + auto invalid_middle = valid; + overwrite_u64(invalid_middle, nodes[0].middle, 8); + expect_rejected(std::move(invalid_middle)); + + auto zero_middle = valid; + overwrite_u64(zero_middle, nodes[0].middle, 0); + expect_rejected(std::move(zero_middle)); + + auto invalid_parent = valid; + overwrite_u64(invalid_parent, nodes[1].parent, nodes.size()); + expect_rejected(std::move(invalid_parent)); + + auto invalid_left_child = valid; + overwrite_u64(invalid_left_child, nodes[0].left_child, nodes.size()); + expect_rejected(std::move(invalid_left_child)); + + auto invalid_right_child = valid; + overwrite_u64(invalid_right_child, nodes[0].right_child, nodes.size()); + expect_rejected(std::move(invalid_right_child)); +} + +TEST(WaveletTreeTest, + SerializationRejectsMalformedFrameMetadataTransactionally) { + constexpr std::size_t kReservedOffset = + sizeof(std::uint64_t) + sizeof(std::uint32_t); + const std::vector data = {0, 1, 2, 3, 4, 5, 6, 7}; + const WaveletTree original(8, data); + pixie::VectorOutputSink output; + pixie::BinaryWriter writer(output); + original.serialize(writer); + writer.finish(); + const std::vector valid = output.take(); + const auto layout = locate_wavelet_artifact(valid); + ASSERT_EQ(layout.nodes.size(), 7u); + + const auto expect_rejected = [&](std::vector artifact) { + for (const auto validation : {pixie::DeserializationValidation::kQuick, + pixie::DeserializationValidation::kFull}) { + pixie::BinaryReader reader(artifact); + EXPECT_THROW( + (void)pixie::WaveletTreeView::deserialize(reader, validation), + std::exception); + EXPECT_EQ(reader.position(), 0u); + } + }; + + auto incompatible_header = valid; + incompatible_header[kReservedOffset] = std::byte{1}; + expect_rejected(std::move(incompatible_header)); + + auto unpadded_size = valid; + overwrite_u64(unpadded_size, 2 * sizeof(std::uint64_t), + 3 * sizeof(std::uint64_t) + 1); + expect_rejected(std::move(unpadded_size)); + + auto excessive_node_count = valid; + overwrite_u64(excessive_node_count, layout.node_count, + std::numeric_limits::max()); + expect_rejected(std::move(excessive_node_count)); + + auto truncated_nodes = valid; + overwrite_u64(truncated_nodes, layout.node_count, 1000); + expect_rejected(std::move(truncated_nodes)); + + auto excessive_alphabet = valid; + overwrite_u64(excessive_alphabet, layout.alphabet_size, + std::numeric_limits::max()); + expect_rejected(std::move(excessive_alphabet)); + + auto duplicate_permutation = valid; + overwrite_u64(duplicate_permutation, layout.permutation, 0); + overwrite_u64(duplicate_permutation, + layout.permutation + sizeof(std::uint64_t), 0); + expect_rejected(std::move(duplicate_permutation)); + + auto out_of_range_permutation = valid; + overwrite_u64(out_of_range_permutation, layout.permutation, 8); + expect_rejected(std::move(out_of_range_permutation)); + + auto invalid_root = valid; + overwrite_u64(invalid_root, layout.root, layout.nodes.size()); + expect_rejected(std::move(invalid_root)); + + auto invalid_leaf = valid; + overwrite_u64(invalid_leaf, layout.leaves, layout.nodes.size()); + expect_rejected(std::move(invalid_leaf)); + + auto wrong_root_length = valid; + overwrite_u64(wrong_root_length, layout.data_size, data.size() + 1); + expect_rejected(std::move(wrong_root_length)); +}