From e8d1990eb563b462782dab7acb746a8372f1cf1b Mon Sep 17 00:00:00 2001 From: Googler Date: Wed, 21 Oct 2026 02:07:56 -0700 Subject: [PATCH] Add `ZipFile::ReadInto(span)`, `ZipFile::Skip(n)` and `ZipArchive::TryClone()`. Adds ZipFile::ReadInto(span), which decompresses into a caller-owned buffer without allocating, ZipFile::Skip(n), which discards bytes without allocating, and ZipArchive::TryClone(), which returns an independent handle onto a file-backed archive that shares the parsed central directory and owns its own file descriptor. PiperOrigin-RevId: 1003764260 --- zip/converters.cc | 46 ++++ zip/converters.h | 35 ++- zip/file.cc | 106 ++++++++ zip/file.h | 139 ++++++++++- zip/read.cc | 234 ++++++++++++++++- zip/read.h | 160 +++++++++++- zip/rust/Cargo.toml | 5 +- zip/rust/entry_metadata.rs | 173 +++++++++++++ zip/rust/error.rs | 16 ++ zip/rust/file.rs | 174 ++++++++++++- zip/rust/lib.rs | 14 +- zip/rust/read.rs | 498 ++++++++++++++++++++++++++++++++++++- zip/rust/write.rs | 386 +++++++++++++++++++++++++++- zip/write.cc | 265 ++++++++++++++++++++ zip/write.h | 90 +++++++ zip/zip_benchmark.cc | 231 +++++++++++++++++ 16 files changed, 2546 insertions(+), 26 deletions(-) create mode 100644 zip/rust/entry_metadata.rs create mode 100644 zip/zip_benchmark.cc diff --git a/zip/converters.cc b/zip/converters.cc index faac231a..2051fff1 100644 --- a/zip/converters.cc +++ b/zip/converters.cc @@ -1,5 +1,6 @@ #include "converters.h" +#include #include #include @@ -51,6 +52,14 @@ absl::Status FromRustResultUnit( return absl::OkStatus(); } +absl::StatusOr FromRustResultU64( + rs_std::Result result_u64) { + if (!result_u64.has_value()) { + return ZipErrorToStatus(std::move(result_u64).err()); + } + return std::move(result_u64).value(); +} + absl::StatusOr FromRustBufferedZipArchive( rs_std::Result result_buffered_zip_archive) { @@ -87,6 +96,15 @@ absl::StatusOr FromRustFsZipFile( return std::move(result_fs_zip_file).value(); } +absl::StatusOr FromRustZipEntryMetadata( + rs_std::Result + result_zip_entry_metadata) { + if (!result_zip_entry_metadata.has_value()) { + return ZipErrorToStatus(std::move(result_zip_entry_metadata).err()); + } + return std::move(result_zip_entry_metadata).value(); +} + absl::StatusOr FromRustBufferedZipWriter( rs_std::Result result_buffered_zip_writer) { @@ -105,4 +123,32 @@ absl::StatusOr FromRustFsZipWriter( return std::move(result_fs_zip_writer).value(); } +absl::StatusOr +FromRustBufferedZipStreamReader( + rs_std::Result + result_buffered_zip_stream_reader) { + if (!result_buffered_zip_stream_reader.has_value()) { + return ZipErrorToStatus(std::move(result_buffered_zip_stream_reader).err()); + } + return std::move(result_buffered_zip_stream_reader).value(); +} + +absl::StatusOr FromRustFsZipStreamReader( + rs_std::Result + result_fs_zip_stream_reader) { + if (!result_fs_zip_stream_reader.has_value()) { + return ZipErrorToStatus(std::move(result_fs_zip_stream_reader).err()); + } + return std::move(result_fs_zip_stream_reader).value(); +} + +absl::StatusOr FromRustFsZipStreamWriter( + rs_std::Result + result_fs_zip_stream_writer) { + if (!result_fs_zip_stream_writer.has_value()) { + return ZipErrorToStatus(std::move(result_fs_zip_stream_writer).err()); + } + return std::move(result_fs_zip_stream_writer).value(); +} + } // namespace security::zip diff --git a/zip/converters.h b/zip/converters.h index a75bb805..a985969f 100644 --- a/zip/converters.h +++ b/zip/converters.h @@ -1,6 +1,7 @@ #ifndef SECURITY_ZIP_CONVERTERS_H_ #define SECURITY_ZIP_CONVERTERS_H_ +#include #include #include "crubit_helpers/string_conversions.h" @@ -49,11 +50,24 @@ class RustVecU8Wrapper { // // Consuming inside the full-expression is safe: // std::string owned(f.GetFileData()->AsStringView()); // OK - absl::string_view AsStringView() const& ABSL_ATTRIBUTE_LIFETIME_BOUND { + [[nodiscard]] absl::string_view AsStringView() + const& ABSL_ATTRIBUTE_LIFETIME_BOUND { return security::crubit_helpers::StringViewFromVecU8(vec_u8_); } absl::string_view AsStringView() const&& = delete; + [[nodiscard]] const char* data() const ABSL_ATTRIBUTE_LIFETIME_BOUND { + return AsStringView().data(); + } + + [[nodiscard]] size_t size() const { return AsStringView().size(); } + + [[nodiscard]] bool empty() const { return AsStringView().empty(); } + + explicit operator absl::string_view() const ABSL_ATTRIBUTE_LIFETIME_BOUND { + return AsStringView(); + } + private: rust::VecU8 vec_u8_; }; @@ -62,6 +76,10 @@ absl::StatusOr FromRustResultVecU8( rs_std::Result result_vec_u8); absl::Status FromRustResultUnit( rs_std::Result result_unit); +// Converts a byte count, as returned by the reads that write into a buffer the +// caller owns rather than returning one. +absl::StatusOr FromRustResultU64( + rs_std::Result result_u64); absl::StatusOr FromRustBufferedZipArchive( rs_std::Result @@ -76,6 +94,9 @@ absl::StatusOr FromRustBufferedZipFile( absl::StatusOr FromRustFsZipFile( rs_std::Result result_fs_zip_file); +absl::StatusOr FromRustZipEntryMetadata( + rs_std::Result + result_zip_entry_metadata); absl::StatusOr FromRustBufferedZipWriter( rs_std::Result @@ -84,6 +105,18 @@ absl::StatusOr FromRustFsZipWriter( rs_std::Result result_fs_zip_writer); +absl::StatusOr +FromRustBufferedZipStreamReader( + rs_std::Result + result_buffered_zip_stream_reader); +absl::StatusOr FromRustFsZipStreamReader( + rs_std::Result + result_fs_zip_stream_reader); + +absl::StatusOr FromRustFsZipStreamWriter( + rs_std::Result + result_fs_zip_stream_writer); + } // namespace security::zip #endif // SECURITY_ZIP_CONVERTERS_H_ diff --git a/zip/file.cc b/zip/file.cc index 92604a77..ddccf9ad 100644 --- a/zip/file.cc +++ b/zip/file.cc @@ -11,6 +11,8 @@ #include "absl/status/status.h" #include "absl/status/status_macros.h" #include "absl/status/statusor.h" +#include "absl/types/span.h" +#include "support/rs_std/slice_ref.h" namespace security::zip { namespace { @@ -36,8 +38,51 @@ absl::StatusOr ToSecurityZipCompressionMethod( return absl::InvalidArgumentError("Unknown compression method"); } +// Rust writes into the caller's buffer through this; the slice borrows it and +// must not outlive the call it is passed to. +rs_std::SliceRef AsMutableSlice(absl::Span buf) { + return rs_std::SliceRef( + absl::Span(reinterpret_cast(buf.data()), buf.size())); +} + } // namespace +bool ZipEntryMetadata::IsFile() const { return metadata_.is_file(); } +bool ZipEntryMetadata::IsDir() const { return metadata_.is_dir(); } +bool ZipEntryMetadata::IsEncrypted() const { return metadata_.is_encrypted(); } +std::string ZipEntryMetadata::GetFileName() const { + rust::VecU8 name_vec = metadata_.get_file_name(); + return std::string(security::crubit_helpers::StringViewFromVecU8(name_vec)); +} +absl::StatusOr ZipEntryMetadata::GetCompressionMethod() + const { + return ToSecurityZipCompressionMethod(metadata_.get_compression_method()); +} +std::string ZipEntryMetadata::GetComment() const { + rust::VecU8 comment_vec = metadata_.get_comment(); + return std::string( + security::crubit_helpers::StringViewFromVecU8(comment_vec)); +} +uint64_t ZipEntryMetadata::GetUncompressedSize() const { + return metadata_.get_uncompressed_size(); +} +uint64_t ZipEntryMetadata::GetCompressedSize() const { + return metadata_.get_compressed_size(); +} +uint32_t ZipEntryMetadata::GetCrc32() const { return metadata_.get_crc32(); } +uint16_t ZipEntryMetadata::GetLastModifiedDate() const { + return metadata_.get_last_modified_date(); +} +uint16_t ZipEntryMetadata::GetLastModifiedTime() const { + return metadata_.get_last_modified_time(); +} +uint32_t ZipEntryMetadata::GetUnixMode() const { + return metadata_.get_unix_mode(); +} +uint64_t ZipEntryMetadata::GetHeaderStart() const { + return metadata_.get_header_start(); +} + absl::Status BufferedZipFile::CheckNone() const { if (IsNone()) { return absl::InternalError("ZipFile object used after being moved-out"); @@ -98,6 +143,14 @@ absl::StatusOr BufferedZipFile::GetExtraData() const { ABSL_RETURN_IF_ERROR(CheckNone()); return RustVecU8Wrapper(zip_.get_extra_data()); } +absl::StatusOr BufferedZipFile::GetHeaderStart() const { + ABSL_RETURN_IF_ERROR(CheckNone()); + return zip_.get_header_start(); +} +absl::StatusOr BufferedZipFile::GetDataStart() const { + ABSL_RETURN_IF_ERROR(CheckNone()); + return zip_.get_data_start(); +} absl::StatusOr BufferedZipFile::GetFileData() { ABSL_RETURN_IF_ERROR(CheckNone()); return FromRustResultVecU8(zip_.get_file_data()); @@ -108,6 +161,20 @@ absl::StatusOr BufferedZipFile::GetFileData(size_t max_size) { } return FromRustResultVecU8(zip_.get_file_data_with_max_size(max_size)); } +absl::StatusOr BufferedZipFile::ReadBytes(size_t max_bytes) { + ABSL_RETURN_IF_ERROR(CheckNone()); + return FromRustResultVecU8(zip_.read_bytes(max_bytes)); +} +absl::StatusOr BufferedZipFile::ReadInto(absl::Span buf) { + ABSL_RETURN_IF_ERROR(CheckNone()); + ABSL_ASSIGN_OR_RETURN(const uint64_t read, + FromRustResultU64(zip_.read_into(AsMutableSlice(buf)))); + return static_cast(read); +} +absl::StatusOr BufferedZipFile::Skip(uint64_t max_bytes) { + ABSL_RETURN_IF_ERROR(CheckNone()); + return FromRustResultU64(zip_.skip(max_bytes)); +} absl::Status FsZipFile::CheckNone() const { if (IsNone()) { @@ -168,6 +235,14 @@ absl::StatusOr FsZipFile::GetExtraData() const { ABSL_RETURN_IF_ERROR(CheckNone()); return RustVecU8Wrapper(zip_.get_extra_data()); } +absl::StatusOr FsZipFile::GetHeaderStart() const { + ABSL_RETURN_IF_ERROR(CheckNone()); + return zip_.get_header_start(); +} +absl::StatusOr FsZipFile::GetDataStart() const { + ABSL_RETURN_IF_ERROR(CheckNone()); + return zip_.get_data_start(); +} absl::StatusOr FsZipFile::GetFileData() { ABSL_RETURN_IF_ERROR(CheckNone()); return FromRustResultVecU8(zip_.get_file_data()); @@ -178,6 +253,20 @@ absl::StatusOr FsZipFile::GetFileData(size_t max_size) { } return FromRustResultVecU8(zip_.get_file_data_with_max_size(max_size)); } +absl::StatusOr FsZipFile::ReadBytes(size_t max_bytes) { + ABSL_RETURN_IF_ERROR(CheckNone()); + return FromRustResultVecU8(zip_.read_bytes(max_bytes)); +} +absl::StatusOr FsZipFile::ReadInto(absl::Span buf) { + ABSL_RETURN_IF_ERROR(CheckNone()); + ABSL_ASSIGN_OR_RETURN(const uint64_t read, + FromRustResultU64(zip_.read_into(AsMutableSlice(buf)))); + return static_cast(read); +} +absl::StatusOr FsZipFile::Skip(uint64_t max_bytes) { + ABSL_RETURN_IF_ERROR(CheckNone()); + return FromRustResultU64(zip_.skip(max_bytes)); +} absl::StatusOr ZipFile::IsFile() const { return std::visit([](const auto& zip) { return zip.IsFile(); }, zip_); @@ -223,6 +312,12 @@ absl::StatusOr ZipFile::GetUnixMode() const { absl::StatusOr ZipFile::GetExtraData() const { return std::visit([](const auto& zip) { return zip.GetExtraData(); }, zip_); } +absl::StatusOr ZipFile::GetHeaderStart() const { + return std::visit([](const auto& zip) { return zip.GetHeaderStart(); }, zip_); +} +absl::StatusOr ZipFile::GetDataStart() const { + return std::visit([](const auto& zip) { return zip.GetDataStart(); }, zip_); +} absl::StatusOr ZipFile::GetFileData() { return std::visit([](auto& zip) { return zip.GetFileData(); }, zip_); } @@ -230,5 +325,16 @@ absl::StatusOr ZipFile::GetFileData(size_t max_size) { return std::visit([max_size](auto& zip) { return zip.GetFileData(max_size); }, zip_); } +absl::StatusOr ZipFile::ReadBytes(size_t max_bytes) { + return std::visit([max_bytes](auto& zip) { return zip.ReadBytes(max_bytes); }, + zip_); +} +absl::StatusOr ZipFile::ReadInto(absl::Span buf) { + return std::visit([buf](auto& zip) { return zip.ReadInto(buf); }, zip_); +} +absl::StatusOr ZipFile::Skip(uint64_t max_bytes) { + return std::visit([max_bytes](auto& zip) { return zip.Skip(max_bytes); }, + zip_); +} } // namespace security::zip diff --git a/zip/file.h b/zip/file.h index 4fa278c2..d654f1b4 100644 --- a/zip/file.h +++ b/zip/file.h @@ -11,11 +11,15 @@ #include "crubit/rust.h" #include "absl/status/status.h" #include "absl/status/statusor.h" +#include "absl/types/span.h" namespace security::zip { class BufferedZipWriter; class FsZipWriter; +class BufferedZipStreamWriter; +class FsZipStreamWriter; +class ZipStreamWriter; enum class CompressionMethod : int32_t { kStored, @@ -26,13 +30,64 @@ enum class CompressionMethod : int32_t { kXz, }; +// What an archive's central directory records about one entry. +// +// Obtaining this reads nothing: the central directory is parsed when the +// archive is opened. Contrast a ZipFile, which additionally reads the entry's +// local file header to find where its data begins, so that listing an archive +// entry by entry costs one read apiece. Prefer this whenever the entry's +// contents are not wanted; over a network the difference is one round trip +// against one per entry. +// +// This is a snapshot and does not borrow the archive, so it stays valid after +// the archive is destroyed and several may be held at once. +// +// Correspondingly there is no GetDataStart(): where an entry's data begins is +// the one thing the central directory does not record. Open the entry for it. +// Nor is there GetExtraData(): the zip crate exposes an entry's extra fields +// only once the entry is opened, so open it (ZipFile::GetExtraData()) for them. +class ZipEntryMetadata final { + public: + explicit ZipEntryMetadata(rust::ZipEntryMetadata metadata) + : metadata_(std::move(metadata)) {} + + // Returns whether the entry is a file, that is, neither a directory nor a + // symbolic link. + bool IsFile() const; + bool IsDir() const; + // Returns whether the entry's data is encrypted. This library does not + // decrypt, so such an entry can be listed but not read. + bool IsEncrypted() const; + std::string GetFileName() const; + absl::StatusOr GetCompressionMethod() const; + std::string GetComment() const; + uint64_t GetUncompressedSize() const; + uint64_t GetCompressedSize() const; + uint32_t GetCrc32() const; + // Returns the last-modified date encoded as an MS-DOS FAT date bitfield + // (bits 15-9: year offset from 1980, bits 8-5: month [1-12], bits 4-0: day + // [1-31]), or 0 if the entry records none. + uint16_t GetLastModifiedDate() const; + // Returns the last-modified time encoded as an MS-DOS FAT time bitfield + // (bits 15-11: hour [0-23], bits 10-5: minute [0-59], bits 4-0: second / 2 + // [0-29]), or 0 if the entry records none. + uint16_t GetLastModifiedTime() const; + // Returns the entry's Unix mode, or 0 if it records none. + uint32_t GetUnixMode() const; + // Returns the offset of this entry's local file header within the archive. + uint64_t GetHeaderStart() const; + + private: + rust::ZipEntryMetadata metadata_; +}; + class BufferedZipFile final { public: explicit BufferedZipFile(rust::BufferedZipFile zip) : zip_(std::move(zip)) {} absl::StatusOr IsFile() const; absl::StatusOr IsDir() const; - bool IsNone() const; + [[nodiscard]] bool IsNone() const; // WARNING: Path Traversal ("Zip Slip") Hazard: The returned file name is // extracted directly from the archive header without sanitization. If using // this path for filesystem operations, callers MUST sanitize and validate @@ -68,12 +123,35 @@ class BufferedZipFile final { // omits ExtendedTimestamp (0x5455) and Ntfs (0x000a) extra fields when // re-serializing `extra_data()`). absl::StatusOr GetExtraData() const; + // Returns the offset of this entry's local file header within the archive. + absl::StatusOr GetHeaderStart() const; + // Returns the offset of this entry's data within the archive, or 0 if it is + // unknown. Together with GetCompressedSize() this delimits the entry's raw + // bytes, so a caller that only needs a kStored entry can read that byte range + // directly rather than through this object. The offset is only known once the + // local file header has been parsed, which happens when an archive opens the + // entry; entries produced by a *ZipStreamReader always report 0. + absl::StatusOr GetDataStart() const; // WARNING: Decompression Bomb Hazard: This overload decompresses the entire // entry into an unbounded heap buffer. For untrusted zip archives, prefer // `GetFileData(size_t max_size)` below to cap memory allocation and defend // against zip bomb memory exhaustion (OOM) attacks. absl::StatusOr GetFileData(); + absl::StatusOr ReadBytes(size_t max_bytes); + // Decompresses into `buf` and returns how many bytes were written, which is + // less than `buf.size()` only at end of entry. + // + // Unlike ReadBytes(), nothing is allocated and the bytes are not copied + // again on the way out, so this is what a caller reading an entry of any + // size should use. + absl::StatusOr ReadInto(absl::Span buf); + // Discards up to `max_bytes` and returns how many were discarded, which is + // less than `max_bytes` only at end of entry. + // + // A zip entry is a forward-only stream, so this is how a caller seeks + // within one. Nothing is allocated, whatever the distance. + absl::StatusOr Skip(uint64_t max_bytes); // Decompresses at most `max_size` bytes from the entry, returning // `absl::StatusCode::kOutOfRange` if the decompressed size exceeds @@ -85,6 +163,9 @@ class BufferedZipFile final { rust::BufferedZipFile zip_; friend class BufferedZipWriter; friend class FsZipWriter; + friend class BufferedZipStreamWriter; + friend class FsZipStreamWriter; + friend class ZipStreamWriter; }; class FsZipFile final { @@ -92,7 +173,7 @@ class FsZipFile final { explicit FsZipFile(rust::FsZipFile zip) : zip_(std::move(zip)) {} absl::StatusOr IsFile() const; absl::StatusOr IsDir() const; - bool IsNone() const; + [[nodiscard]] bool IsNone() const; // WARNING: Path Traversal ("Zip Slip") Hazard: The returned file name is // extracted directly from the archive header without sanitization. If using @@ -129,12 +210,35 @@ class FsZipFile final { // omits ExtendedTimestamp (0x5455) and Ntfs (0x000a) extra fields when // re-serializing `extra_data()`). absl::StatusOr GetExtraData() const; + // Returns the offset of this entry's local file header within the archive. + absl::StatusOr GetHeaderStart() const; + // Returns the offset of this entry's data within the archive, or 0 if it is + // unknown. Together with GetCompressedSize() this delimits the entry's raw + // bytes, so a caller that only needs a kStored entry can read that byte range + // directly rather than through this object. The offset is only known once the + // local file header has been parsed, which happens when an archive opens the + // entry; entries produced by a *ZipStreamReader always report 0. + absl::StatusOr GetDataStart() const; // WARNING: Decompression Bomb Hazard: This overload decompresses the entire // entry into an unbounded heap buffer. For untrusted zip archives, prefer // `GetFileData(size_t max_size)` below to cap memory allocation and defend // against zip bomb memory exhaustion (OOM) attacks. absl::StatusOr GetFileData(); + absl::StatusOr ReadBytes(size_t max_bytes); + // Decompresses into `buf` and returns how many bytes were written, which is + // less than `buf.size()` only at end of entry. + // + // Unlike ReadBytes(), nothing is allocated and the bytes are not copied + // again on the way out, so this is what a caller reading an entry of any + // size should use. + absl::StatusOr ReadInto(absl::Span buf); + // Discards up to `max_bytes` and returns how many were discarded, which is + // less than `max_bytes` only at end of entry. + // + // A zip entry is a forward-only stream, so this is how a caller seeks + // within one. Nothing is allocated, whatever the distance. + absl::StatusOr Skip(uint64_t max_bytes); // Decompresses at most `max_size` bytes from the entry, returning // `absl::StatusCode::kOutOfRange` if the decompressed size exceeds @@ -146,6 +250,9 @@ class FsZipFile final { rust::FsZipFile zip_; friend class BufferedZipWriter; friend class FsZipWriter; + friend class BufferedZipStreamWriter; + friend class FsZipStreamWriter; + friend class ZipStreamWriter; }; class ZipFile final { @@ -160,7 +267,7 @@ class ZipFile final { absl::StatusOr IsFile() const; absl::StatusOr IsDir() const; - bool IsNone() const; + [[nodiscard]] bool IsNone() const; // WARNING: Path Traversal ("Zip Slip") Hazard: The returned file name is // extracted directly from the archive header without sanitization. If using @@ -197,12 +304,35 @@ class ZipFile final { // omits ExtendedTimestamp (0x5455) and Ntfs (0x000a) extra fields when // re-serializing `extra_data()`). absl::StatusOr GetExtraData() const; + // Returns the offset of this entry's local file header within the archive. + absl::StatusOr GetHeaderStart() const; + // Returns the offset of this entry's data within the archive, or 0 if it is + // unknown. Together with GetCompressedSize() this delimits the entry's raw + // bytes, so a caller that only needs a kStored entry can read that byte range + // directly rather than through this object. The offset is only known once the + // local file header has been parsed, which happens when an archive opens the + // entry; entries produced by a *ZipStreamReader always report 0. + absl::StatusOr GetDataStart() const; // WARNING: Decompression Bomb Hazard: This overload decompresses the entire // entry into an unbounded heap buffer. For untrusted zip archives, prefer // `GetFileData(size_t max_size)` below to cap memory allocation and defend // against zip bomb memory exhaustion (OOM) attacks. absl::StatusOr GetFileData(); + absl::StatusOr ReadBytes(size_t max_bytes); + // Decompresses into `buf` and returns how many bytes were written, which is + // less than `buf.size()` only at end of entry. + // + // Unlike ReadBytes(), nothing is allocated and the bytes are not copied + // again on the way out, so this is what a caller reading an entry of any + // size should use. + absl::StatusOr ReadInto(absl::Span buf); + // Discards up to `max_bytes` and returns how many were discarded, which is + // less than `max_bytes` only at end of entry. + // + // A zip entry is a forward-only stream, so this is how a caller seeks + // within one. Nothing is allocated, whatever the distance. + absl::StatusOr Skip(uint64_t max_bytes); // Decompresses at most `max_size` bytes from the entry, returning // `absl::StatusCode::kOutOfRange` if the decompressed size exceeds @@ -215,6 +345,9 @@ class ZipFile final { explicit ZipFile(BackendType b) : zip_(std::move(b)) {} friend class BufferedZipWriter; friend class FsZipWriter; + friend class BufferedZipStreamWriter; + friend class FsZipStreamWriter; + friend class ZipStreamWriter; }; } // namespace security::zip diff --git a/zip/read.cc b/zip/read.cc index f130f3c8..68ca8701 100644 --- a/zip/read.cc +++ b/zip/read.cc @@ -1,6 +1,7 @@ #include "read.h" #include +#include #include #include #include @@ -9,6 +10,7 @@ #include "converters.h" #include "file.h" #include "crubit/rust.h" +#include "absl/functional/overload.h" #include "absl/status/status.h" #include "absl/status/status_macros.h" #include "absl/status/statusor.h" @@ -30,14 +32,14 @@ absl::StatusOr BufferedZipArchive::NewFromData( absl::StatusOr BufferedZipArchive::GetLength() const { if (archive_.is_none()) { - return absl::InternalError("ZipArchive object used after being moved-out"); + return absl::FailedPreconditionError("Zip archive is not open"); } return archive_.get_length(); } absl::StatusOr BufferedZipArchive::GetComment() const { if (archive_.is_none()) { - return absl::InternalError("ZipArchive object used after being moved-out"); + return absl::FailedPreconditionError("Zip archive is not open"); } rust::VecU8 comment_vec = archive_.get_comment(); return std::string( @@ -45,6 +47,9 @@ absl::StatusOr BufferedZipArchive::GetComment() const { } absl::StatusOr BufferedZipArchive::GetFileByIndex(uintptr_t index) { + if (archive_.is_none()) { + return absl::FailedPreconditionError("Zip archive is not open"); + } ABSL_ASSIGN_OR_RETURN( rust::BufferedZipFile file, FromRustBufferedZipFile(archive_.get_file_by_index(index))); @@ -52,12 +57,28 @@ absl::StatusOr BufferedZipArchive::GetFileByIndex(uintptr_t index) { } absl::StatusOr BufferedZipArchive::GetFileByIndexRaw(uintptr_t index) { + if (archive_.is_none()) { + return absl::FailedPreconditionError("Zip archive is not open"); + } ABSL_ASSIGN_OR_RETURN( rust::BufferedZipFile file, FromRustBufferedZipFile(archive_.get_file_by_index_raw(index))); return ZipFile::FromBuffer(std::move(file)); } +absl::StatusOr BufferedZipArchive::GetEntryMetadata( + uintptr_t index) const { + if (archive_.is_none()) { + return absl::FailedPreconditionError("Zip archive is not open"); + } + ABSL_ASSIGN_OR_RETURN( + rust::ZipEntryMetadata metadata, + FromRustZipEntryMetadata(archive_.get_entry_metadata(index))); + return ZipEntryMetadata(std::move(metadata)); +} + +bool BufferedZipArchive::IsNone() const { return archive_.is_none(); } + absl::StatusOr FsZipArchive::NewFromPath(absl::string_view path) { ABSL_ASSIGN_OR_RETURN( rust::FsZipArchive archive, @@ -67,16 +88,25 @@ absl::StatusOr FsZipArchive::NewFromPath(absl::string_view path) { return FsZipArchive(std::move(archive)); } +absl::StatusOr FsZipArchive::TryClone() const { + if (archive_.is_none()) { + return absl::FailedPreconditionError("Zip archive is not open"); + } + ABSL_ASSIGN_OR_RETURN(rust::FsZipArchive archive, + FromRustFsZipArchive(archive_.try_clone())); + return FsZipArchive(std::move(archive)); +} + absl::StatusOr FsZipArchive::GetLength() const { if (archive_.is_none()) { - return absl::InternalError("ZipArchive object used after being moved-out"); + return absl::FailedPreconditionError("Zip archive is not open"); } return archive_.get_length(); } absl::StatusOr FsZipArchive::GetComment() const { if (archive_.is_none()) { - return absl::InternalError("ZipArchive object used after being moved-out"); + return absl::FailedPreconditionError("Zip archive is not open"); } rust::VecU8 comment_vec = archive_.get_comment(); return std::string( @@ -84,18 +114,37 @@ absl::StatusOr FsZipArchive::GetComment() const { } absl::StatusOr FsZipArchive::GetFileByIndex(uintptr_t index) { + if (archive_.is_none()) { + return absl::FailedPreconditionError("Zip archive is not open"); + } ABSL_ASSIGN_OR_RETURN(rust::FsZipFile file, FromRustFsZipFile(archive_.get_file_by_index(index))); return ZipFile::FromFile(std::move(file)); } absl::StatusOr FsZipArchive::GetFileByIndexRaw(uintptr_t index) { + if (archive_.is_none()) { + return absl::FailedPreconditionError("Zip archive is not open"); + } ABSL_ASSIGN_OR_RETURN( rust::FsZipFile file, FromRustFsZipFile(archive_.get_file_by_index_raw(index))); return ZipFile::FromFile(std::move(file)); } +absl::StatusOr FsZipArchive::GetEntryMetadata( + uintptr_t index) const { + if (archive_.is_none()) { + return absl::FailedPreconditionError("Zip archive is not open"); + } + ABSL_ASSIGN_OR_RETURN( + rust::ZipEntryMetadata metadata, + FromRustZipEntryMetadata(archive_.get_entry_metadata(index))); + return ZipEntryMetadata(std::move(metadata)); +} + +bool FsZipArchive::IsNone() const { return archive_.is_none(); } + absl::StatusOr ZipArchive::FromFile(absl::string_view path) { ABSL_ASSIGN_OR_RETURN(FsZipArchive archive, FsZipArchive::NewFromPath(path)); return ZipArchive(std::move(archive)); @@ -107,6 +156,20 @@ absl::StatusOr ZipArchive::FromBuffer(absl::string_view data) { return ZipArchive(std::move(archive)); } +absl::StatusOr ZipArchive::TryClone() const { + return std::visit( + absl::Overload( + [](const BufferedZipArchive&) -> absl::StatusOr { + return absl::UnimplementedError( + "TryClone is not supported for archives opened from a buffer"); + }, + [](const auto& archive) -> absl::StatusOr { + ABSL_ASSIGN_OR_RETURN(auto clone, archive.TryClone()); + return ZipArchive(std::move(clone)); + }), + archive_); +} + absl::StatusOr ZipArchive::GetLength() const { return std::visit([](const auto& arg) { return arg.GetLength(); }, archive_); } @@ -125,4 +188,167 @@ absl::StatusOr ZipArchive::GetFileByIndexRaw(uintptr_t index) { archive_); } +absl::StatusOr ZipArchive::GetEntryMetadata( + uintptr_t index) const { + return std::visit( + [index](const auto& arg) { return arg.GetEntryMetadata(index); }, + archive_); +} + +bool ZipArchive::IsNone() const { + return std::visit([](const auto& arg) { return arg.IsNone(); }, archive_); +} + +absl::StatusOr BufferedZipStreamReader::NewFromData( + absl::string_view data) { + rust::VecU8 input_data = + rust::VecU8::copy_from_slice(absl::Span( + reinterpret_cast(data.data()), data.size())); + ABSL_ASSIGN_OR_RETURN( + rust::BufferedZipStreamReader reader, + FromRustBufferedZipStreamReader( + rust::BufferedZipStreamReader::new_from_data(input_data))); + return BufferedZipStreamReader(std::move(reader)); +} + +absl::StatusOr> BufferedZipStreamReader::ReadNextFile() { + if (reader_.is_none()) { + return absl::FailedPreconditionError("Zip stream reader is not open"); + } + ABSL_ASSIGN_OR_RETURN(rust::BufferedZipFile file, + FromRustBufferedZipFile(reader_.read_next_file())); + if (file.is_none()) { + return std::nullopt; + } + return ZipFile::FromBuffer(std::move(file)); +} + +absl::StatusOr> +BufferedZipStreamReader::ReadNextFileWithCompressedSize( + uint64_t compressed_size) { + if (reader_.is_none()) { + return absl::FailedPreconditionError("Zip stream reader is not open"); + } + ABSL_ASSIGN_OR_RETURN( + rust::BufferedZipFile file, + FromRustBufferedZipFile( + reader_.read_next_file_with_compressed_size(compressed_size))); + if (file.is_none()) { + return std::nullopt; + } + return ZipFile::FromBuffer(std::move(file)); +} + +bool BufferedZipStreamReader::IsFinished() const { + return reader_.is_finished(); +} + +bool BufferedZipStreamReader::IsNone() const { return reader_.is_none(); } + +absl::StatusOr FsZipStreamReader::NewFromPath( + absl::string_view path) { + ABSL_ASSIGN_OR_RETURN( + rust::FsZipStreamReader reader, + FromRustFsZipStreamReader(rust::FsZipStreamReader::new_from_path( + absl::Span( + reinterpret_cast(path.data()), path.size())))); + return FsZipStreamReader(std::move(reader)); +} + +absl::StatusOr> FsZipStreamReader::ReadNextFile() { + if (reader_.is_none()) { + return absl::FailedPreconditionError("Zip stream reader is not open"); + } + ABSL_ASSIGN_OR_RETURN(rust::FsZipFile file, + FromRustFsZipFile(reader_.read_next_file())); + if (file.is_none()) { + return std::nullopt; + } + return ZipFile::FromFile(std::move(file)); +} + +absl::StatusOr> +FsZipStreamReader::ReadNextFileWithCompressedSize(uint64_t compressed_size) { + if (reader_.is_none()) { + return absl::FailedPreconditionError("Zip stream reader is not open"); + } + ABSL_ASSIGN_OR_RETURN( + rust::FsZipFile file, + FromRustFsZipFile( + reader_.read_next_file_with_compressed_size(compressed_size))); + if (file.is_none()) { + return std::nullopt; + } + return ZipFile::FromFile(std::move(file)); +} + +bool FsZipStreamReader::IsFinished() const { return reader_.is_finished(); } + +bool FsZipStreamReader::IsNone() const { return reader_.is_none(); } + +absl::StatusOr ZipStreamReader::FromFile( + absl::string_view path) { + ABSL_ASSIGN_OR_RETURN(FsZipStreamReader reader, + FsZipStreamReader::NewFromPath(path)); + return ZipStreamReader(std::move(reader)); +} + +absl::StatusOr ZipStreamReader::FromBuffer( + absl::string_view data) { + ABSL_ASSIGN_OR_RETURN(BufferedZipStreamReader reader, + BufferedZipStreamReader::NewFromData(data)); + return ZipStreamReader(std::move(reader)); +} + +absl::StatusOr> ZipStreamReader::ReadNextFile() { + return std::visit([](auto& arg) { return arg.ReadNextFile(); }, reader_); +} + +absl::StatusOr> +ZipStreamReader::ReadNextFileWithCompressedSize(uint64_t compressed_size) { + return std::visit( + [compressed_size](auto& arg) { + return arg.ReadNextFileWithCompressedSize(compressed_size); + }, + reader_); +} + +bool ZipStreamReader::IsFinished() const { + return std::visit([](const auto& arg) { return arg.IsFinished(); }, reader_); +} + +bool ZipStreamReader::IsNone() const { + return std::visit([](const auto& arg) { return arg.IsNone(); }, reader_); +} + +absl::StatusOr> ReadZipFileFromStream( + BufferedZipStreamReader& reader) { + return reader.ReadNextFile(); +} + +absl::StatusOr> ReadZipFileFromStream( + FsZipStreamReader& reader) { + return reader.ReadNextFile(); +} + +absl::StatusOr> ReadZipFileFromStream( + ZipStreamReader& reader) { + return reader.ReadNextFile(); +} + +absl::StatusOr> ReadZipFileFromStreamWithCompressedSize( + BufferedZipStreamReader& reader, uint64_t compressed_size) { + return reader.ReadNextFileWithCompressedSize(compressed_size); +} + +absl::StatusOr> ReadZipFileFromStreamWithCompressedSize( + FsZipStreamReader& reader, uint64_t compressed_size) { + return reader.ReadNextFileWithCompressedSize(compressed_size); +} + +absl::StatusOr> ReadZipFileFromStreamWithCompressedSize( + ZipStreamReader& reader, uint64_t compressed_size) { + return reader.ReadNextFileWithCompressedSize(compressed_size); +} + } // namespace security::zip diff --git a/zip/read.h b/zip/read.h index 40da10ed..f149aefe 100644 --- a/zip/read.h +++ b/zip/read.h @@ -2,12 +2,14 @@ #define SECURITY_ZIP_READ_H_ #include +#include #include #include #include #include "file.h" #include "crubit/rust.h" +#include "absl/base/attributes.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" @@ -20,12 +22,20 @@ class BufferedZipArchive final { absl::StatusOr GetComment() const; // Returns a ZipFile by index. The result file returns decompressed data when // read. - absl::StatusOr GetFileByIndex(uintptr_t index); + absl::StatusOr GetFileByIndex(uintptr_t index) + ABSL_ATTRIBUTE_LIFETIME_BOUND; // Returns a ZipFile by index. The result file returns compressed data as-is // without decompressing when read. // Warning: Writers in zip-rs do not support writing compressed data as-is. // This data will be compressed again when written through a `*ZipWriter`. - absl::StatusOr GetFileByIndexRaw(uintptr_t index); + absl::StatusOr GetFileByIndexRaw(uintptr_t index) + ABSL_ATTRIBUTE_LIFETIME_BOUND; + // Returns what the central directory records about the entry at `index`, + // reading nothing. Prefer this to GetFileByIndex when the entry's contents + // are not wanted: that one also reads the entry's local file header, so + // listing an archive with it costs one read per entry. + absl::StatusOr GetEntryMetadata(uintptr_t index) const; + [[nodiscard]] bool IsNone() const; private: rust::BufferedZipArchive archive_; @@ -36,16 +46,40 @@ class BufferedZipArchive final { class FsZipArchive final { public: static absl::StatusOr NewFromPath(absl::string_view path); + // Returns an independent handle onto the same archive. + // + // The new handle shares the central directory that NewFromPath already + // parsed, so this costs one open(2) and is O(1) in the number of entries, + // whereas NewFromPath is O(entries). Use it when several entries must be read + // at once: GetFileByIndex keeps the archive borrowed for as long as the + // returned ZipFile lives, so N concurrently open entries need N handles, and + // reopening by path for each one is quadratic on large archives. + // + // Each handle owns its own file descriptor and therefore its own file + // position, so handles may be read independently. + // + // The caller must ensure the file has not been modified since it was opened; + // the shared central directory would otherwise describe byte ranges that no + // longer hold the entries it names. + absl::StatusOr TryClone() const; absl::StatusOr GetLength() const; absl::StatusOr GetComment() const; // Returns a ZipFile by index. The result file returns decompressed data when // read. - absl::StatusOr GetFileByIndex(uintptr_t index); + absl::StatusOr GetFileByIndex(uintptr_t index) + ABSL_ATTRIBUTE_LIFETIME_BOUND; // Returns a ZipFile by index. The result file returns compressed data as-is // without decompressing when read. // Warning: Writers in zip-rs do not support writing compressed data as-is. // This data will be compressed again when written through a `*ZipWriter`. - absl::StatusOr GetFileByIndexRaw(uintptr_t index); + absl::StatusOr GetFileByIndexRaw(uintptr_t index) + ABSL_ATTRIBUTE_LIFETIME_BOUND; + // Returns what the central directory records about the entry at `index`, + // reading nothing. Prefer this to GetFileByIndex when the entry's contents + // are not wanted: that one also reads the entry's local file header, so + // listing an archive with it costs one read per entry. + absl::StatusOr GetEntryMetadata(uintptr_t index) const; + [[nodiscard]] bool IsNone() const; private: rust::FsZipArchive archive_; @@ -57,16 +91,32 @@ class ZipArchive final { public: static absl::StatusOr FromFile(absl::string_view path); static absl::StatusOr FromBuffer(absl::string_view data); + // Returns an independent handle onto the same archive, sharing the central + // directory this one already parsed. See FsZipArchive::TryClone for the cost + // model and the caller's obligations. + // + // Not supported for archives opened with FromBuffer, which return + // absl::UnimplementedError, because their reader cannot currently be shared + // and cloning one would copy the whole archive. + absl::StatusOr TryClone() const; absl::StatusOr GetLength() const; absl::StatusOr GetComment() const; // Returns a ZipFile by index. The result file returns decompressed data when // read. - absl::StatusOr GetFileByIndex(uintptr_t index); + absl::StatusOr GetFileByIndex(uintptr_t index) + ABSL_ATTRIBUTE_LIFETIME_BOUND; // Returns a ZipFile by index. The result file returns compressed data as-is // without decompressing when read. // Warning: Writers in zip-rs do not support writing compressed data as-is. // This data will be compressed again when written through a `*ZipWriter`. - absl::StatusOr GetFileByIndexRaw(uintptr_t index); + absl::StatusOr GetFileByIndexRaw(uintptr_t index) + ABSL_ATTRIBUTE_LIFETIME_BOUND; + // Returns what the central directory records about the entry at `index`, + // reading nothing. Prefer this to GetFileByIndex when the entry's contents + // are not wanted: that one also reads the entry's local file header, so + // listing an archive with it costs one read per entry. + absl::StatusOr GetEntryMetadata(uintptr_t index) const; + [[nodiscard]] bool IsNone() const; private: using BackendType = std::variant; @@ -75,6 +125,104 @@ class ZipArchive final { explicit ZipArchive(BackendType archive) : archive_(std::move(archive)) {} }; +// Streaming readers that read sequential zip entries from a stream without +// seeking. +class BufferedZipStreamReader final { + public: + static absl::StatusOr NewFromData( + absl::string_view data); + // Reads the next file in the stream. Returns std::nullopt when end of archive + // is reached. + // IMPORTANT: The returned ZipFile borrows the underlying stream and drains + // any unread compressed bytes when destroyed so the stream is positioned at + // the next local file header. Callers MUST destroy the returned ZipFile + // before calling ReadNextFile* again. + absl::StatusOr> ReadNextFile() + ABSL_ATTRIBUTE_LIFETIME_BOUND; + // Reads the next file in the stream with an assumed compressed size. + // Callers MUST destroy the returned ZipFile before calling ReadNextFile* + // again. + absl::StatusOr> ReadNextFileWithCompressedSize( + uint64_t compressed_size) ABSL_ATTRIBUTE_LIFETIME_BOUND; + [[nodiscard]] bool IsFinished() const; + [[nodiscard]] bool IsNone() const; + + private: + rust::BufferedZipStreamReader reader_; + explicit BufferedZipStreamReader(rust::BufferedZipStreamReader reader) + : reader_(std::move(reader)) {} +}; + +class FsZipStreamReader final { + public: + static absl::StatusOr NewFromPath(absl::string_view path); + // Reads the next file in the stream. Returns std::nullopt when end of archive + // is reached. + // IMPORTANT: The returned ZipFile borrows the underlying stream and drains + // any unread compressed bytes when destroyed so the stream is positioned at + // the next local file header. Callers MUST destroy the returned ZipFile + // before calling ReadNextFile* again. + absl::StatusOr> ReadNextFile() + ABSL_ATTRIBUTE_LIFETIME_BOUND; + // Reads the next file in the stream with an assumed compressed size. + // Callers MUST destroy the returned ZipFile before calling ReadNextFile* + // again. + absl::StatusOr> ReadNextFileWithCompressedSize( + uint64_t compressed_size) ABSL_ATTRIBUTE_LIFETIME_BOUND; + [[nodiscard]] bool IsFinished() const; + [[nodiscard]] bool IsNone() const; + + private: + rust::FsZipStreamReader reader_; + explicit FsZipStreamReader(rust::FsZipStreamReader reader) + : reader_(std::move(reader)) {} +}; + +class ZipStreamReader final { + public: + static absl::StatusOr FromFile(absl::string_view path); + static absl::StatusOr FromBuffer(absl::string_view data); + // Reads the next file in the stream. Returns std::nullopt when end of archive + // is reached. + // IMPORTANT: The returned ZipFile borrows the underlying stream and drains + // any unread compressed bytes when destroyed so the stream is positioned at + // the next local file header. Callers MUST destroy the returned ZipFile + // before calling ReadNextFile* again. + absl::StatusOr> ReadNextFile() + ABSL_ATTRIBUTE_LIFETIME_BOUND; + // Reads the next file in the stream with an assumed compressed size. + // Callers MUST destroy the returned ZipFile before calling ReadNextFile* + // again. + absl::StatusOr> ReadNextFileWithCompressedSize( + uint64_t compressed_size) ABSL_ATTRIBUTE_LIFETIME_BOUND; + [[nodiscard]] bool IsFinished() const; + [[nodiscard]] bool IsNone() const; + + private: + using BackendType = std::variant; + BackendType reader_; + + explicit ZipStreamReader(BackendType reader) : reader_(std::move(reader)) {} +}; + +// Free function helpers mirroring zip::read::read_zipfile_from_stream. +absl::StatusOr> ReadZipFileFromStream( + BufferedZipStreamReader& reader ABSL_ATTRIBUTE_LIFETIME_BOUND); +absl::StatusOr> ReadZipFileFromStream( + FsZipStreamReader& reader ABSL_ATTRIBUTE_LIFETIME_BOUND); +absl::StatusOr> ReadZipFileFromStream( + ZipStreamReader& reader ABSL_ATTRIBUTE_LIFETIME_BOUND); + +absl::StatusOr> ReadZipFileFromStreamWithCompressedSize( + BufferedZipStreamReader& reader ABSL_ATTRIBUTE_LIFETIME_BOUND, + uint64_t compressed_size); +absl::StatusOr> ReadZipFileFromStreamWithCompressedSize( + FsZipStreamReader& reader ABSL_ATTRIBUTE_LIFETIME_BOUND, + uint64_t compressed_size); +absl::StatusOr> ReadZipFileFromStreamWithCompressedSize( + ZipStreamReader& reader ABSL_ATTRIBUTE_LIFETIME_BOUND, + uint64_t compressed_size); + } // namespace security::zip #endif // SECURITY_ZIP_READ_H_ diff --git a/zip/rust/Cargo.toml b/zip/rust/Cargo.toml index ff7b9bf6..91d6ba42 100644 --- a/zip/rust/Cargo.toml +++ b/zip/rust/Cargo.toml @@ -9,4 +9,7 @@ path = "lib.rs" doctest = false [dependencies] -zip = { version = "9.0", features = ["deflate", "bzip2", "time", "zstd", "lzma", "xz"] } \ No newline at end of file +zip = { version = "9.0", features = ["deflate", "bzip2", "time", "zstd", "lzma", "xz"] } + +[features] +deflate-zopfli = ["zip/deflate-zopfli"] diff --git a/zip/rust/entry_metadata.rs b/zip/rust/entry_metadata.rs new file mode 100644 index 00000000..e734c8e1 --- /dev/null +++ b/zip/rust/entry_metadata.rs @@ -0,0 +1,173 @@ +//! A zip entry's central directory record, readable without any I/O. +//! +//! Crubit does not support generic type parameters, so the archive types that +//! hand these out are written out one by one; this type itself owns its data +//! and so is shared by all of them. + +use crate::write::CompressionMethod; +use crate::VecU8; +use zip::read::ZipArchiveMetadata; +use zip::CompressionMethod as ZipCrateCompressionMethod; + +/// What a zip archive's central directory records about one entry. +/// +/// Every field here is read when the archive is opened, so obtaining this +/// costs no I/O. Note the contrast with `*ZipFile`, which additionally seeks +/// to the entry's local file header in order to find where its data begins: +/// listing an archive through `get_file_by_index` therefore costs one read per +/// entry, which is ruinous when the archive is behind a network. Prefer this +/// whenever the entry's contents are not wanted. +/// +/// Correspondingly, this deliberately has no `get_data_start()`: that offset +/// is the one piece of per-entry information that is not in the central +/// directory. Open the entry if you need it. +#[derive(Debug)] +pub struct ZipEntryMetadata { + name: Vec, + comment: Vec, + uncompressed_size: u64, + compressed_size: u64, + header_start: u64, + crc32: u32, + unix_mode: u32, + last_modified_date: u16, + last_modified_time: u16, + compression_method: CompressionMethod, + is_dir: bool, + is_file: bool, + is_encrypted: bool, +} + +impl Default for ZipEntryMetadata { + /// The default value exists only so that Crubit can generate a movable + /// type; it describes no entry. + fn default() -> Self { + Self { + name: Vec::new(), + comment: Vec::new(), + uncompressed_size: 0, + compressed_size: 0, + header_start: 0, + crc32: 0, + unix_mode: 0, + last_modified_date: 0, + last_modified_time: 0, + compression_method: CompressionMethod::Unsupported, + is_dir: false, + is_file: false, + is_encrypted: false, + } + } +} + +impl ZipEntryMetadata { + /// Copies out what `metadata` records about the entry at `index`, or + /// `None` if there is no such entry. + /// + /// This function won't be generated by Crubit and is only called from + /// rust. (b/259749095) + pub(crate) fn from_central_directory( + metadata: &ZipArchiveMetadata, + index: usize, + ) -> Option { + let entry = metadata.entry(index).ok()?; + let last_modified = entry.last_modified(); + Some(Self { + // Decoded the same way as `*ZipFile::get_file_name`, so that listing + // and opening an entry agree on its name (UTF-8, else CP437). + name: match entry.name() { + Ok(name) => name.as_bytes().to_vec(), + Err(_) => entry.name_raw().to_vec(), + }, + comment: entry.comment().as_bytes().to_vec(), + uncompressed_size: entry.size(), + compressed_size: entry.compressed_size(), + header_start: entry.header_start(), + crc32: entry.crc32(), + unix_mode: entry.unix_mode().unwrap_or(0), + last_modified_date: last_modified.map(|dt| dt.datepart()).unwrap_or(0), + last_modified_time: last_modified.map(|dt| dt.timepart()).unwrap_or(0), + compression_method: match entry.compression() { + ZipCrateCompressionMethod::STORE => CompressionMethod::Stored, + ZipCrateCompressionMethod::DEFLATE => CompressionMethod::Deflated, + ZipCrateCompressionMethod::BZIP2 => CompressionMethod::Bzip2, + ZipCrateCompressionMethod::ZSTD => CompressionMethod::Zstd, + ZipCrateCompressionMethod::LZMA => CompressionMethod::Lzma, + ZipCrateCompressionMethod::XZ => CompressionMethod::Xz, + _ => CompressionMethod::Unsupported, + }, + is_dir: entry.is_dir(), + is_file: entry.is_file(), + is_encrypted: entry.encrypted(), + }) + } + + /// Returns the name of the entry. + pub fn get_file_name(&self) -> VecU8 { + VecU8::copy_from_slice(&self.name) + } + + /// Returns whether the entry is a file, that is, neither a directory nor a + /// symbolic link. + pub fn is_file(&self) -> bool { + self.is_file + } + + /// Returns whether the entry is a directory. + pub fn is_dir(&self) -> bool { + self.is_dir + } + + /// Returns whether the entry's data is encrypted. This library does not + /// decrypt, so such an entry can be listed but not read. + pub fn is_encrypted(&self) -> bool { + self.is_encrypted + } + + /// Returns the compression method of the entry. + pub fn get_compression_method(&self) -> CompressionMethod { + self.compression_method + } + + /// Returns the comment on the entry. + pub fn get_comment(&self) -> VecU8 { + VecU8::copy_from_slice(&self.comment) + } + + /// Returns the uncompressed size of the entry. + pub fn get_uncompressed_size(&self) -> u64 { + self.uncompressed_size + } + + /// Returns the compressed size of the entry. + pub fn get_compressed_size(&self) -> u64 { + self.compressed_size + } + + /// Returns the CRC32 of the entry. + pub fn get_crc32(&self) -> u32 { + self.crc32 + } + + /// Returns the last modified date of the entry in DOS format, or 0 if it + /// does not record one. + pub fn get_last_modified_date(&self) -> u16 { + self.last_modified_date + } + + /// Returns the last modified time of the entry in DOS format, or 0 if it + /// does not record one. + pub fn get_last_modified_time(&self) -> u16 { + self.last_modified_time + } + + /// Returns the Unix mode of the entry, or 0 if it does not record one. + pub fn get_unix_mode(&self) -> u32 { + self.unix_mode + } + + /// Returns the offset of the entry's local file header in the archive. + pub fn get_header_start(&self) -> u64 { + self.header_start + } +} diff --git a/zip/rust/error.rs b/zip/rust/error.rs index 79599674..9838fd48 100644 --- a/zip/rust/error.rs +++ b/zip/rust/error.rs @@ -37,3 +37,19 @@ impl ZipError { Self::new(message, ZipErrorCode::Internal) } } + +impl ZipError { + /// Converts a `zip` crate error. Deliberately not a `From` impl: a public + /// `From` would make Crubit generate C++ bindings for + /// the `zip` crate itself (and include them from zip_wrapper.h), which + /// clients neither need nor want. + pub(crate) fn from_zip_error(err: zip::result::ZipError) -> Self { + match err { + zip::result::ZipError::Io(e) => ZipError::internal(e.to_string()), + zip::result::ZipError::InvalidArchive(e) => ZipError::invalid_argument(e.as_ref()), + zip::result::ZipError::UnsupportedArchive(e) => ZipError::failed_precondition(e), + zip::result::ZipError::FileNotFound => ZipError::out_of_range("File not found"), + _ => ZipError::internal(err.to_string()), + } + } +} diff --git a/zip/rust/file.rs b/zip/rust/file.rs index 5d8cf466..71592516 100644 --- a/zip/rust/file.rs +++ b/zip/rust/file.rs @@ -112,6 +112,26 @@ impl<'a> BufferedZipFile<'a> { get_extra_data_impl(&self.file) } + /// Returns the offset of this entry's local file header within the archive. + /// The result only is valid if `is_none()` returns false. + pub fn get_header_start(&self) -> u64 { + get_header_start_impl(&self.file) + } + + /// Returns the offset of this entry's data within the archive, or 0 if it + /// is unknown. Together with `get_compressed_size()` this delimits the + /// entry's raw bytes, so a caller that only needs a `Stored` entry can read + /// that byte range directly instead of going through this archive. + /// + /// The offset is only known once the local file header has been parsed, + /// which archive-based readers do when the entry is opened. It is never + /// known for entries obtained from a `*ZipStreamReader`, which report 0. + /// + /// The result only is valid if `is_none()` returns false. + pub fn get_data_start(&self) -> u64 { + get_data_start_impl(&self.file) + } + /// Returns the data of the file. /// The result is only valid if `is_none()` returns false. pub fn get_file_data(&mut self) -> Result { @@ -124,6 +144,36 @@ impl<'a> BufferedZipFile<'a> { pub fn get_file_data_with_max_size(&mut self, max_size: usize) -> Result { get_file_data_with_max_size_impl(&mut self.file, max_size) } + + /// Reads up to `max_bytes` decompressed data from the file. + /// Returns an empty VecU8 on EOF. + /// The result only is valid if `is_none()` returns false. + pub fn read_bytes(&mut self, max_bytes: usize) -> Result { + read_bytes_impl(&mut self.file, max_bytes) + } + + /// Decompresses into `buf` and returns how many bytes were written, which + /// is less than `buf.len()` only at end of entry. + /// + /// Unlike `read_bytes`, nothing is allocated and the bytes are not copied + /// again on the way out, so this is what a caller reading an entry of any + /// size should use. + /// + /// The result only is valid if `is_none()` returns false. + pub fn read_into(&mut self, buf: &mut [u8]) -> Result { + read_into_impl(&mut self.file, buf) + } + + /// Discards up to `max_bytes` and returns how many were discarded, which + /// is less than `max_bytes` only at end of entry. + /// + /// A zip entry is a forward-only stream, so this is how a caller seeks + /// within one. Nothing is allocated, whatever the distance. + /// + /// The result only is valid if `is_none()` returns false. + pub fn skip(&mut self, max_bytes: u64) -> Result { + skip_impl(&mut self.file, max_bytes) + } } impl<'a> Read for BufferedZipFile<'a> { @@ -238,6 +288,26 @@ impl<'a> FsZipFile<'a> { get_extra_data_impl(&self.file) } + /// Returns the offset of this entry's local file header within the archive. + /// The result only is valid if `is_none()` returns false. + pub fn get_header_start(&self) -> u64 { + get_header_start_impl(&self.file) + } + + /// Returns the offset of this entry's data within the archive, or 0 if it + /// is unknown. Together with `get_compressed_size()` this delimits the + /// entry's raw bytes, so a caller that only needs a `Stored` entry can read + /// that byte range directly instead of going through this archive. + /// + /// The offset is only known once the local file header has been parsed, + /// which archive-based readers do when the entry is opened. It is never + /// known for entries obtained from a `*ZipStreamReader`, which report 0. + /// + /// The result only is valid if `is_none()` returns false. + pub fn get_data_start(&self) -> u64 { + get_data_start_impl(&self.file) + } + /// Returns the data of the file. /// The result is only valid if `is_none()` returns false. pub fn get_file_data(&mut self) -> Result { @@ -250,6 +320,36 @@ impl<'a> FsZipFile<'a> { pub fn get_file_data_with_max_size(&mut self, max_size: usize) -> Result { get_file_data_with_max_size_impl(&mut self.file, max_size) } + + /// Reads up to `max_bytes` decompressed data from the file. + /// Returns an empty VecU8 on EOF. + /// The result only is valid if `is_none()` returns false. + pub fn read_bytes(&mut self, max_bytes: usize) -> Result { + read_bytes_impl(&mut self.file, max_bytes) + } + + /// Decompresses into `buf` and returns how many bytes were written, which + /// is less than `buf.len()` only at end of entry. + /// + /// Unlike `read_bytes`, nothing is allocated and the bytes are not copied + /// again on the way out, so this is what a caller reading an entry of any + /// size should use. + /// + /// The result only is valid if `is_none()` returns false. + pub fn read_into(&mut self, buf: &mut [u8]) -> Result { + read_into_impl(&mut self.file, buf) + } + + /// Discards up to `max_bytes` and returns how many were discarded, which + /// is less than `max_bytes` only at end of entry. + /// + /// A zip entry is a forward-only stream, so this is how a caller seeks + /// within one. Nothing is allocated, whatever the distance. + /// + /// The result only is valid if `is_none()` returns false. + pub fn skip(&mut self, max_bytes: u64) -> Result { + skip_impl(&mut self.file, max_bytes) + } } impl<'a> Read for FsZipFile<'a> { @@ -317,6 +417,23 @@ fn get_extra_data_impl<'a, R: Read>(file: &Option>>) - file.as_ref().and_then(|f| f.extra_data()).map(VecU8::from).unwrap_or_default() } +fn get_header_start_impl<'a, R: Read>(file: &Option>>) -> u64 { + match file.as_ref() { + Some(file) => file.header_start(), + None => 0, + } +} + +fn get_data_start_impl<'a, R: Read>(file: &Option>>) -> u64 { + match file.as_ref() { + // `data_start` is `None` until the local file header has been parsed, + // and a real entry can never start at offset 0 because the header + // itself occupies at least 30 bytes, so 0 is unambiguously "unknown". + Some(file) => file.data_start().unwrap_or(0), + None => 0, + } +} + fn get_compression_method_impl<'a, R: Read>( file: &Option>>, ) -> CompressionMethod { @@ -345,7 +462,7 @@ fn get_file_data_impl<'a, R: Read>( Err(e) => Err(ZipError::internal(e.to_string())), } } - None => Ok(VecU8::default()), + None => Err(ZipError::internal("ZipFile is not available")), } } @@ -374,3 +491,58 @@ fn get_file_data_with_max_size_impl<'a, R: Read>( None => Ok(VecU8::default()), } } + +fn read_bytes_impl<'a, R: Read>( + file: &mut Option>>, + max_bytes: usize, +) -> Result { + match file.as_mut() { + Some(file) => { + let mut buffer = vec![0u8; max_bytes]; + match file.read(&mut buffer) { + Ok(bytes_read) => { + buffer.truncate(bytes_read); + Ok(buffer.into()) + } + Err(e) => Err(ZipError::internal(e.to_string())), + } + } + None => Err(ZipError::internal("ZipFile is not available")), + } +} + +fn read_into_impl<'a, R: Read>( + file: &mut Option>>, + buf: &mut [u8], +) -> Result { + let Some(file) = file.as_mut() else { + return Err(ZipError::internal("ZipFile is not available")); + }; + // A decompressor returns what the current block happens to yield, which is + // routinely less than was asked for, so fill the buffer here rather than + // making the caller loop -- across a language boundary, in C++'s case. + let mut filled = 0usize; + while filled < buf.len() { + match file.read(&mut buf[filled..]) { + Ok(0) => break, + Ok(bytes_read) => filled += bytes_read, + // Read contracts allow this and it means nothing was consumed. + Err(e) if e.kind() == ErrorKind::Interrupted => {} + Err(e) => return Err(ZipError::internal(e.to_string())), + } + } + Ok(filled as u64) +} + +fn skip_impl<'a, R: Read>( + file: &mut Option>>, + max_bytes: u64, +) -> Result { + let Some(file) = file.as_mut() else { + return Err(ZipError::internal("ZipFile is not available")); + }; + // `io::copy` drives this with a reused stack buffer, so a long skip costs + // no allocation however far it goes. + std::io::copy(&mut (&mut **file).take(max_bytes), &mut std::io::sink()) + .map_err(|e| ZipError::internal(e.to_string())) +} diff --git a/zip/rust/lib.rs b/zip/rust/lib.rs index ce9153f2..207e28d6 100644 --- a/zip/rust/lib.rs +++ b/zip/rust/lib.rs @@ -4,14 +4,24 @@ //! //! WARNING: This crate should never be used from Rust, instead use zip directly. +mod entry_metadata; +pub use entry_metadata::ZipEntryMetadata; + mod file; pub use file::{BufferedZipFile, FsZipFile}; mod read; -pub use read::{BufferedZipArchive, FsZipArchive}; +pub use read::{ + read_buffered_zipfile_from_stream, read_buffered_zipfile_from_stream_with_compressed_size, + read_fs_zipfile_from_stream, read_fs_zipfile_from_stream_with_compressed_size, + BufferedZipArchive, BufferedZipStreamReader, FsZipArchive, FsZipStreamReader, +}; mod write; -pub use write::{BufferedZipWriter, CompressionMethod, FsZipWriter, ZipWriterFileOptions}; +pub use write::{ + new_buffered_zip_stream_writer, new_fs_zip_stream_writer, BufferedZipStreamWriter, + BufferedZipWriter, CompressionMethod, FsZipStreamWriter, FsZipWriter, ZipWriterFileOptions, +}; mod vec_u8; pub use vec_u8::VecU8; diff --git a/zip/rust/read.rs b/zip/rust/read.rs index f38b872c..186f279c 100644 --- a/zip/rust/read.rs +++ b/zip/rust/read.rs @@ -1,7 +1,7 @@ // Crubit does not support generic type parameters, so we need to implement // BufferedZipArchive and FsZipArchive manually. -use crate::{BufferedZipFile, FsZipFile, VecU8, ZipError}; +use crate::{BufferedZipFile, FsZipFile, VecU8, ZipEntryMetadata, ZipError}; use std::fmt::{Debug, Formatter}; use std::fs::File; use std::io::{Cursor, Read, Seek}; @@ -83,7 +83,7 @@ impl BufferedZipArchive { Ok(file) => Ok(BufferedZipFile::new(file)), Err(e) => Err(ZipError::out_of_range(e.to_string())), }, - None => Ok(BufferedZipFile::default()), + None => Err(ZipError::failed_precondition("Zip archive is not open")), } } @@ -100,14 +100,30 @@ impl BufferedZipArchive { Ok(file) => Ok(BufferedZipFile::new(file)), Err(e) => Err(ZipError::out_of_range(e.to_string())), }, - None => Ok(BufferedZipFile::default()), + None => Err(ZipError::failed_precondition("Zip archive is not open")), } } + + /// Returns what the central directory records about the entry at `index`, + /// without reading anything. + /// + /// Prefer this to `get_file_by_index` when the entry's contents are not + /// wanted: that one also reads the entry's local file header, so listing + /// an archive with it costs one read per entry. + /// + /// Returns an error if the archive is not open or `index` is out of + /// bounds. + pub fn get_entry_metadata(&self, index: usize) -> Result { + get_entry_metadata_impl(&self.reader, index) + } } #[derive(Default)] pub struct FsZipArchive { reader: Option>>, + /// Path the archive was opened from, retained so that `try_clone` can open + /// an independent file descriptor onto the same archive. + path: Vec, } impl Debug for FsZipArchive { @@ -154,6 +170,7 @@ impl FsZipArchive { Ok(file) => match WrappedZipArchive::new(file) { Ok(reader) => { self.reader = Some(Box::new(reader)); + self.path = path.to_vec(); Ok(()) } Err(e) => Err(e.to_string()), @@ -162,6 +179,52 @@ impl FsZipArchive { } } + /// Returns an independent handle onto the same archive. + /// + /// The new handle shares the central directory that `new_from_path` already + /// parsed, so this costs one `open(2)` and is O(1) in the number of entries, + /// whereas `new_from_path` is O(entries). Use it when several entries must + /// be read at once: `get_file_by_index` borrows the archive mutably, so a + /// caller that wants N concurrently open entries needs N handles, and + /// reopening by path for each one is quadratic on large archives. + /// + /// Each handle owns its own file descriptor and therefore its own file + /// position, so handles may be read independently. + /// + /// Returns an error if the archive is not open or the path cannot be + /// reopened. The caller must ensure the file has not been modified since it + /// was opened; the shared central directory would otherwise describe byte + /// ranges that no longer hold the entries it names. + pub fn try_clone(&self) -> Result { + let Some(reader) = self.reader.as_ref() else { + return Err(ZipError::failed_precondition("Zip archive is not open")); + }; + let path_str = match std::str::from_utf8(&self.path) { + Ok(s) => s, + Err(e) => return Err(ZipError::invalid_argument(e.to_string())), + }; + let file = match File::open(path_str) { + Ok(file) => file, + Err(e) => { + return Err(ZipError::invalid_argument(format!( + "Failed to reopen zip archive: {}", + e + ))) + } + }; + // SAFETY: `unsafe_new_with_metadata` requires `reader.metadata()` to + // describe the bytes `file` will yield. `file` is a freshly opened + // descriptor for the same path `reader` parsed its central directory + // from, and this function's contract (see the doc comment) makes the + // caller responsible for that file not having been modified in + // between. Should it have been anyway, every subsequent read goes + // through zip-rs's bounds, signature and CRC checks, so a mismatch + // surfaces as an error rather than as memory unsafety. + let archive = + unsafe { WrappedZipArchive::unsafe_new_with_metadata(file, reader.metadata()) }; + Ok(FsZipArchive { reader: Some(Box::new(archive)), path: self.path.clone() }) + } + /// Returns whether the zip archive is none (due to being /// default-constructed or moved-from). pub fn is_none(&self) -> bool { @@ -188,7 +251,7 @@ impl FsZipArchive { Ok(file) => Ok(FsZipFile::new(file)), Err(e) => Err(ZipError::out_of_range(e.to_string())), }, - None => Ok(FsZipFile::default()), + None => Err(ZipError::failed_precondition("Zip archive is not open")), } } @@ -205,9 +268,22 @@ impl FsZipArchive { Ok(file) => Ok(FsZipFile::new(file)), Err(e) => Err(ZipError::out_of_range(e.to_string())), }, - None => Ok(FsZipFile::default()), + None => Err(ZipError::failed_precondition("Zip archive is not open")), } } + + /// Returns what the central directory records about the entry at `index`, + /// without reading anything. + /// + /// Prefer this to `get_file_by_index` when the entry's contents are not + /// wanted: that one also reads the entry's local file header, so listing + /// an archive with it costs one read per entry. + /// + /// Returns an error if the archive is not open or `index` is out of + /// bounds. + pub fn get_entry_metadata(&self, index: usize) -> Result { + get_entry_metadata_impl(&self.reader, index) + } } fn get_length_impl(reader: &Option>>) -> usize { @@ -216,7 +292,417 @@ fn get_length_impl(reader: &Option>>) - None => 0, } } - fn get_comment_impl(reader: &Option>>) -> VecU8 { reader.as_ref().map(|r| VecU8::copy_from_slice(r.comment())).unwrap_or_default() } + +fn get_entry_metadata_impl( + reader: &Option>>, + index: usize, +) -> Result { + let Some(r) = reader.as_ref() else { + return Err(ZipError::failed_precondition("Zip archive is not open")); + }; + // The central directory was parsed when the archive was opened, so this + // reads nothing. + match ZipEntryMetadata::from_central_directory(&r.metadata(), index) { + Some(metadata) => Ok(metadata), + None => Err(ZipError::out_of_range(format!( + "Index {index} is out of bounds for an archive of {} entries", + r.len() + ))), + } +} + +#[derive(Default)] +/// A streaming zip reader that reads from an in-memory buffer without seeking. +pub struct BufferedZipStreamReader { + reader: Option>>>, + finished: bool, +} + +impl Debug for BufferedZipStreamReader { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("BufferedZipStreamReader") + .field("reader", &if self.reader.is_some() { "Some(Cursor>)" } else { "None" }) + .field("finished", &self.finished) + .finish() + } +} + +impl BufferedZipStreamReader { + /// Initializes an empty `BufferedZipStreamReader`. + pub fn new() -> Self { + Self::default() + } + + /// Creates a new `BufferedZipStreamReader` from data. + pub fn new_from_data(data: VecU8) -> Result { + Ok(Self { reader: Some(Box::new(Cursor::new(data.into_vec()))), finished: false }) + } + + /// Returns whether the stream reader is none (due to being default-constructed + /// or moved-from). + pub fn is_none(&self) -> bool { + self.reader.is_none() + } + + /// Returns whether the end of the archive stream has been reached. + pub fn is_finished(&self) -> bool { + self.finished + } + + /// Reads the next zip file from the stream. + /// + /// When there are no more files in the archive (start of central directory + /// is reached), returns a `BufferedZipFile` where `is_none()` returns true. + pub fn read_next_file(&mut self) -> Result, ZipError> { + if self.finished { + return Ok(BufferedZipFile::default()); + } + match self.reader.as_deref_mut() { + Some(reader) => match zip::read::read_zipfile_from_stream(reader) { + Ok(Some(file)) => Ok(BufferedZipFile::new(file)), + Ok(None) => { + self.finished = true; + Ok(BufferedZipFile::default()) + } + Err(e) => Err(ZipError::from_zip_error(e)), + }, + None => Err(ZipError::failed_precondition("Zip stream reader is not open")), + } + } + + /// Reads the next zip file from the stream with an assumed compressed size. + /// + /// When there are no more files in the archive, returns a `BufferedZipFile` + /// where `is_none()` returns true. + pub fn read_next_file_with_compressed_size( + &mut self, + compressed_size: u64, + ) -> Result, ZipError> { + if self.finished { + return Ok(BufferedZipFile::default()); + } + match self.reader.as_deref_mut() { + Some(reader) => { + match zip::read::read_zipfile_from_stream_with_compressed_size( + reader, + compressed_size, + ) { + Ok(Some(file)) => Ok(BufferedZipFile::new(file)), + Ok(None) => { + self.finished = true; + Ok(BufferedZipFile::default()) + } + Err(e) => Err(ZipError::from_zip_error(e)), + } + } + None => Err(ZipError::failed_precondition("Zip stream reader is not open")), + } + } +} + +#[derive(Default)] +/// A streaming zip reader that reads sequentially from a file without seeking. +pub struct FsZipStreamReader { + reader: Option>, + finished: bool, +} + +impl Debug for FsZipStreamReader { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("FsZipStreamReader") + .field("reader", &if self.reader.is_some() { "Some(File)" } else { "None" }) + .field("finished", &self.finished) + .finish() + } +} + +impl FsZipStreamReader { + /// Initializes an empty `FsZipStreamReader`. + pub fn new() -> Self { + Self::default() + } + + /// Creates a new `FsZipStreamReader` from a path. + pub fn new_from_path(path: &[u8]) -> Result { + let path_str = std::str::from_utf8(path) + .map_err(|e| ZipError::invalid_argument(format!("Failed to open zip stream: {}", e)))?; + let file = File::open(path_str) + .map_err(|e| ZipError::invalid_argument(format!("Failed to open zip stream: {}", e)))?; + Ok(Self { reader: Some(Box::new(file)), finished: false }) + } + + /// Returns whether the stream reader is none (due to being default-constructed + /// or moved-from). + pub fn is_none(&self) -> bool { + self.reader.is_none() + } + + /// Returns whether the end of the archive stream has been reached. + pub fn is_finished(&self) -> bool { + self.finished + } + + /// Reads the next zip file from the stream. + /// + /// When there are no more files in the archive (start of central directory + /// is reached), returns a `FsZipFile` where `is_none()` returns true. + pub fn read_next_file(&mut self) -> Result, ZipError> { + if self.finished { + return Ok(FsZipFile::default()); + } + match self.reader.as_deref_mut() { + Some(reader) => match zip::read::read_zipfile_from_stream(reader) { + Ok(Some(file)) => Ok(FsZipFile::new(file)), + Ok(None) => { + self.finished = true; + Ok(FsZipFile::default()) + } + Err(e) => Err(ZipError::from_zip_error(e)), + }, + None => Err(ZipError::failed_precondition("Zip stream reader is not open")), + } + } + + /// Reads the next zip file from the stream with an assumed compressed size. + /// + /// When there are no more files in the archive, returns a `FsZipFile` + /// where `is_none()` returns true. + pub fn read_next_file_with_compressed_size( + &mut self, + compressed_size: u64, + ) -> Result, ZipError> { + if self.finished { + return Ok(FsZipFile::default()); + } + match self.reader.as_deref_mut() { + Some(reader) => { + match zip::read::read_zipfile_from_stream_with_compressed_size( + reader, + compressed_size, + ) { + Ok(Some(file)) => Ok(FsZipFile::new(file)), + Ok(None) => { + self.finished = true; + Ok(FsZipFile::default()) + } + Err(e) => Err(ZipError::from_zip_error(e)), + } + } + None => Err(ZipError::failed_precondition("Zip stream reader is not open")), + } + } +} + +/// Reads the next zip file from a buffered stream reader. +pub fn read_buffered_zipfile_from_stream( + reader: &mut BufferedZipStreamReader, +) -> Result, ZipError> { + reader.read_next_file() +} + +/// Reads the next zip file from a buffered stream reader with an assumed compressed size. +pub fn read_buffered_zipfile_from_stream_with_compressed_size( + reader: &mut BufferedZipStreamReader, + compressed_size: u64, +) -> Result, ZipError> { + reader.read_next_file_with_compressed_size(compressed_size) +} + +/// Reads the next zip file from a filesystem stream reader. +pub fn read_fs_zipfile_from_stream( + reader: &mut FsZipStreamReader, +) -> Result, ZipError> { + reader.read_next_file() +} + +/// Reads the next zip file from a filesystem stream reader with an assumed compressed size. +pub fn read_fs_zipfile_from_stream_with_compressed_size( + reader: &mut FsZipStreamReader, + compressed_size: u64, +) -> Result, ZipError> { + reader.read_next_file_with_compressed_size(compressed_size) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{BufferedZipWriter, CompressionMethod, ZipWriterFileOptions}; + use googletest::prelude::*; + use std::io::Write; + + fn create_test_zip() -> VecU8 { + let mut writer = BufferedZipWriter::new_from_data(VecU8::default(), false).unwrap(); + let options = ZipWriterFileOptions::new().compression_method(CompressionMethod::Stored); + writer.start_file(b"hello.txt", options).unwrap(); + writer.write_data(VecU8::from(b"Hello, world!".to_vec())).unwrap(); + writer.start_file(b"second.txt", options).unwrap(); + writer.write_data(VecU8::from(b"Second file content".to_vec())).unwrap(); + writer.finish().unwrap() + } + + /// Writes `create_test_zip()` to a uniquely named file under the test's + /// temporary directory and returns the path. + fn create_test_zip_file(name: &str) -> String { + let dir = std::env::var("TEST_TMPDIR").unwrap_or_else(|_| "/tmp".to_string()); + let path = format!("{}/{}.zip", dir, name); + let mut file = File::create(&path).unwrap(); + file.write_all(create_test_zip().as_slice()).unwrap(); + file.flush().unwrap(); + path + } + + #[gtest] + fn test_fs_archive_try_clone_reads_the_same_entries() { + let path = create_test_zip_file("try_clone_same_entries"); + let archive = FsZipArchive::new_from_path(path.as_bytes()).unwrap(); + let mut clone = archive.try_clone().unwrap(); + + expect_false!(clone.is_none()); + expect_eq!(clone.get_length(), archive.get_length()); + let mut file = clone.get_file_by_index(0).unwrap(); + expect_eq!(file.get_file_name().as_slice(), b"hello.txt"); + expect_eq!(file.get_file_data().unwrap().as_slice(), b"Hello, world!"); + } + + #[gtest] + fn test_fs_archive_clones_have_independent_positions() { + let path = create_test_zip_file("try_clone_independent"); + let mut archive = FsZipArchive::new_from_path(path.as_bytes()).unwrap(); + let mut clone = archive.try_clone().unwrap(); + + // Each handle owns its own descriptor, so reading one entry to + // completion must not disturb the other handle's entry. This is the + // whole point of `try_clone`: `get_file_by_index` borrows the archive + // mutably, so two entries can only be open at once via two handles. + let mut first = archive.get_file_by_index(0).unwrap(); + expect_eq!(first.get_file_data().unwrap().as_slice(), b"Hello, world!"); + let mut second = clone.get_file_by_index(1).unwrap(); + expect_eq!(second.get_file_data().unwrap().as_slice(), b"Second file content"); + } + + #[gtest] + fn test_fs_archive_try_clone_fails_when_not_open() { + let archive = FsZipArchive::new(); + expect_true!(archive.try_clone().is_err()); + } + + #[gtest] + fn test_entry_offsets_delimit_the_stored_bytes() { + let zip_data = create_test_zip(); + let raw = zip_data.as_slice().to_vec(); + let mut archive = BufferedZipArchive::new_from_data(zip_data).unwrap(); + let file = archive.get_file_by_index(0).unwrap(); + + let header_start = file.get_header_start(); + let data_start = file.get_data_start(); + let compressed_size = file.get_compressed_size(); + + // The first entry's local header sits at the very start of the archive, + // and its data follows the 30-byte fixed header plus the file name. + expect_eq!(header_start, 0); + expect_eq!(data_start, 30 + "hello.txt".len() as u64); + // The entry is Stored, so the raw bytes in that range are the contents. + let begin = data_start as usize; + let end = begin + compressed_size as usize; + expect_eq!(&raw[begin..end], b"Hello, world!"); + } + + #[gtest] + fn test_stream_entries_report_unknown_data_start() { + let zip_data = create_test_zip(); + let mut reader = BufferedZipStreamReader::new_from_data(zip_data).unwrap(); + let file = reader.read_next_file().unwrap(); + + // Stream readers never learn where the entry lives in the archive, so + // the offsets are reported as unknown rather than as bogus values. + expect_eq!(file.get_data_start(), 0); + expect_eq!(file.get_header_start(), 0); + } + + #[gtest] + fn test_buffered_stream_reader_read_all_files() { + let zip_data = create_test_zip(); + let mut reader = BufferedZipStreamReader::new_from_data(zip_data).unwrap(); + expect_false!(reader.is_none()); + expect_false!(reader.is_finished()); + + { + let mut file1 = reader.read_next_file().unwrap(); + expect_false!(file1.is_none()); + expect_eq!(file1.get_file_name().as_slice(), b"hello.txt"); + expect_true!(file1.is_file()); + expect_false!(file1.is_dir()); + expect_eq!(file1.get_file_data().unwrap().as_slice(), b"Hello, world!"); + } + + { + let mut file2 = reader.read_next_file().unwrap(); + expect_false!(file2.is_none()); + expect_eq!(file2.get_file_name().as_slice(), b"second.txt"); + expect_eq!(file2.get_file_data().unwrap().as_slice(), b"Second file content"); + } + + // Third read should return None (end of files) + { + let file3 = reader.read_next_file().unwrap(); + expect_true!(file3.is_none()); + } + expect_true!(reader.is_finished()); + + // Repeated reads after finish should be idempotent + { + let file4 = reader.read_next_file().unwrap(); + expect_true!(file4.is_none()); + } + } + + #[gtest] + fn test_buffered_stream_reader_chunked_read() { + let zip_data = create_test_zip(); + let mut reader = BufferedZipStreamReader::new_from_data(zip_data).unwrap(); + let mut file = reader.read_next_file().unwrap(); + expect_false!(file.is_none()); + + // Read in chunks of 5 bytes + let chunk1 = file.read_bytes(5).unwrap(); + expect_eq!(chunk1.as_slice(), b"Hello"); + let chunk2 = file.read_bytes(5).unwrap(); + expect_eq!(chunk2.as_slice(), b", wor"); + let chunk3 = file.read_bytes(5).unwrap(); + expect_eq!(chunk3.as_slice(), b"ld!"); + let chunk4 = file.read_bytes(5).unwrap(); + expect_true!(chunk4.is_empty()); + } + + #[gtest] + fn test_buffered_stream_reader_free_functions() { + let zip_data = create_test_zip(); + let mut reader = BufferedZipStreamReader::new_from_data(zip_data).unwrap(); + let mut file = read_buffered_zipfile_from_stream(&mut reader).unwrap(); + expect_false!(file.is_none()); + expect_eq!(file.get_file_name().as_slice(), b"hello.txt"); + expect_eq!(file.get_file_data().unwrap().as_slice(), b"Hello, world!"); + } + + #[gtest] + fn test_buffered_stream_reader_with_compressed_size() { + let zip_data = create_test_zip(); + let mut reader = BufferedZipStreamReader::new_from_data(zip_data).unwrap(); + // File 1 is uncompressed "Hello, world!" which is 13 bytes stored + let mut file = reader.read_next_file_with_compressed_size(13).unwrap(); + expect_false!(file.is_none()); + expect_eq!(file.get_file_name().as_slice(), b"hello.txt"); + expect_eq!(file.get_file_data().unwrap().as_slice(), b"Hello, world!"); + } + + #[gtest] + fn test_buffered_stream_reader_invalid_data() { + let mut reader = + BufferedZipStreamReader::new_from_data(VecU8::from(b"not a zip".to_vec())).unwrap(); + let res = reader.read_next_file(); + expect_true!(res.is_err()); + } +} diff --git a/zip/rust/write.rs b/zip/rust/write.rs index 90e77b9f..b6b2228c 100644 --- a/zip/rust/write.rs +++ b/zip/rust/write.rs @@ -6,8 +6,8 @@ use std::fmt::{Debug, Formatter}; use std::fs::{File, OpenOptions}; use std::io::{copy, Cursor, Read, Seek, Write}; use zip::{ - write::FileOptions, CompressionMethod as ZipCrateCompressionMethod, - ZipWriter as WrappedZipWriter, + write::{FileOptions, StreamWriter}, + CompressionMethod as ZipCrateCompressionMethod, ZipWriter as WrappedZipWriter, }; // Expose some of the options for writing files to the zip archive. @@ -264,6 +264,21 @@ impl BufferedZipWriter { pub fn write_file_content(&mut self, path: &[u8]) -> Result<(), ZipError> { write_file_content_impl(&mut self.writer, path) } + + /// Sets the archive comment. + pub fn set_comment(&mut self, comment: &[u8]) -> Result<(), ZipError> { + set_comment_impl(&mut self.writer, comment) + } + + /// Flushes any pending output. + pub fn flush(&mut self) -> Result<(), ZipError> { + flush_impl(&mut self.writer) + } + + /// Returns whether seeking is possible (always true for BufferedZipWriter). + pub fn is_seek_possible(&self) -> bool { + true + } } #[derive(Default)] @@ -388,6 +403,21 @@ impl FsZipWriter { pub fn write_file_content(&mut self, path: &[u8]) -> Result<(), ZipError> { write_file_content_impl(&mut self.writer, path) } + + /// Sets the archive comment. + pub fn set_comment(&mut self, comment: &[u8]) -> Result<(), ZipError> { + set_comment_impl(&mut self.writer, comment) + } + + /// Flushes any pending output. + pub fn flush(&mut self) -> Result<(), ZipError> { + flush_impl(&mut self.writer) + } + + /// Returns whether seeking is possible (always true for FsZipWriter). + pub fn is_seek_possible(&self) -> bool { + true + } } fn start_file_impl( @@ -476,3 +506,355 @@ fn write_file_content_impl( Err(ZipError::failed_precondition("writer is not open")) } } + +fn set_comment_impl( + writer: &mut Option>>, + comment: &[u8], +) -> Result<(), ZipError> { + if let Some(writer) = writer.as_deref_mut() { + match writer.set_raw_comment(Box::from(comment)) { + Ok(_) => Ok(()), + Err(e) => Err(ZipError::invalid_argument(e.to_string())), + } + } else { + Err(ZipError::failed_precondition("writer is not open")) + } +} + +fn flush_impl( + writer: &mut Option>>, +) -> Result<(), ZipError> { + if let Some(writer) = writer.as_deref_mut() { + match writer.flush() { + Ok(_) => Ok(()), + Err(e) => Err(ZipError::internal(e.to_string())), + } + } else { + Err(ZipError::failed_precondition("writer is not open")) + } +} + +type InnerBufferedStreamWriter = WrappedZipWriter>>>; + +#[derive(Default)] +/// A zip writer that streams to an in-memory buffer without seeking. +pub struct BufferedZipStreamWriter { + writer: Option>, +} + +impl Debug for BufferedZipStreamWriter { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("BufferedZipStreamWriter") + .field( + "writer", + &if self.writer.is_some() { + "Some(WrappedZipWriter>>>>)" + } else { + "None" + }, + ) + .finish() + } +} + +impl BufferedZipStreamWriter { + /// Creates a new empty `BufferedZipStreamWriter`. + pub fn new() -> Self { + Self::default() + } + + /// Creates a new streaming `BufferedZipStreamWriter`. + pub fn new_stream() -> Self { + let cursor = Cursor::new(Vec::new()); + Self { writer: Some(Box::new(WrappedZipWriter::new_stream(cursor))) } + } + + /// Returns whether the zip writer is none (due to being + /// default-constructed or moved-from). + pub fn is_none(&self) -> bool { + self.writer.is_none() + } + + /// Returns whether seeking is possible (always false for stream writers). + pub fn is_seek_possible(&self) -> bool { + false + } + + /// Finishes writing the zip archive and returns the buffered data. + pub fn finish(&mut self) -> Result { + if let Some(writer) = self.writer.take() { + match writer.finish() { + Ok(stream_writer) => Ok(stream_writer.into_inner().into_inner().into()), + Err(e) => Err(ZipError::internal(e.to_string())), + } + } else { + Err(ZipError::failed_precondition("writer is not open")) + } + } + + /// Creates a new file in the zip archive and starts writing to it. + pub fn start_file( + &mut self, + name: &[u8], + options: ZipWriterFileOptions, + ) -> Result<(), ZipError> { + start_file_impl(&mut self.writer, name, options) + } + + /// Adds a directory to the zip archive. + pub fn add_directory( + &mut self, + name: &[u8], + options: ZipWriterFileOptions, + ) -> Result<(), ZipError> { + add_directory_impl(&mut self.writer, name, options) + } + + /// Writes data to the current file in the zip archive. + pub fn write_data(&mut self, data: VecU8) -> Result<(), ZipError> { + write_data_impl(&mut self.writer, data) + } + + /// Writes file content from a `BufferedZipFile` to the current file in the zip archive. + pub fn write_buffered_zip_file_content( + &mut self, + file: &mut BufferedZipFile, + ) -> Result<(), ZipError> { + do_copy_impl(&mut self.writer, file) + } + + /// Writes file content from a `FsZipFile` to the current file in the zip archive. + pub fn write_fs_zip_file_content(&mut self, file: &mut FsZipFile) -> Result<(), ZipError> { + do_copy_impl(&mut self.writer, file) + } + + /// Writes file content from a path to the current file in the zip archive. + pub fn write_file_content(&mut self, path: &[u8]) -> Result<(), ZipError> { + write_file_content_impl(&mut self.writer, path) + } + + /// Archive comments are not supported by streaming writers: zip 9.x + /// finalizes a commented archive by seeking back over the comment, which a + /// non-seekable writer cannot do (`zip::write::StreamWriter` only permits + /// no-op seeks). Always returns a FailedPrecondition error. + pub fn set_comment(&mut self, _comment: &[u8]) -> Result<(), ZipError> { + Err(ZipError::failed_precondition( + "archive comments are not supported by streaming zip writers", + )) + } + + /// Flushes any pending output. + pub fn flush(&mut self) -> Result<(), ZipError> { + flush_impl(&mut self.writer) + } +} + +#[derive(Default)] +/// A zip writer that streams to a file on the filesystem without seeking. +pub struct FsZipStreamWriter { + writer: Option>>>, +} + +impl Debug for FsZipStreamWriter { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("FsZipStreamWriter") + .field( + "writer", + &if self.writer.is_some() { + "Some(WrappedZipWriter>)" + } else { + "None" + }, + ) + .finish() + } +} + +impl FsZipStreamWriter { + /// Creates a new `FsZipStreamWriter`. + pub fn new() -> Self { + Self::default() + } + + /// Creates a new streaming `FsZipStreamWriter` writing to `path`. + pub fn new_from_path(path: &[u8]) -> Result { + let mut writer = Self::default(); + if let Err(e) = writer.open(path) { + return Err(ZipError::invalid_argument(format!("Failed to open zip archive: {}", e))); + } + Ok(writer) + } + + fn open(&mut self, path: &[u8]) -> Result<(), String> { + let path_lossy = String::from_utf8_lossy(path); + let path_str = path_lossy.as_ref(); + match OpenOptions::new().write(true).create(true).truncate(true).open(path_str) { + Ok(file) => { + self.writer = Some(Box::new(WrappedZipWriter::new_stream(file))); + Ok(()) + } + Err(e) => Err(e.to_string()), + } + } + + /// Returns whether the zip writer is none (due to being + /// default-constructed or moved-from). + pub fn is_none(&self) -> bool { + self.writer.is_none() + } + + /// Returns whether seeking is possible (always false for stream writers). + pub fn is_seek_possible(&self) -> bool { + false + } + + /// Finishes writing the zip archive to file. + pub fn finish(&mut self) -> Result<(), ZipError> { + if let Some(writer) = self.writer.take() { + match writer.finish() { + Ok(_) => Ok(()), + Err(e) => Err(ZipError::internal(e.to_string())), + } + } else { + Err(ZipError::failed_precondition("writer is not open")) + } + } + + /// Creates a new file in the zip archive and starts writing to it. + pub fn start_file( + &mut self, + name: &[u8], + options: ZipWriterFileOptions, + ) -> Result<(), ZipError> { + start_file_impl(&mut self.writer, name, options) + } + + /// Adds a directory to the zip archive. + pub fn add_directory( + &mut self, + name: &[u8], + options: ZipWriterFileOptions, + ) -> Result<(), ZipError> { + add_directory_impl(&mut self.writer, name, options) + } + + /// Writes data to the current file in the zip archive. + pub fn write_data(&mut self, data: VecU8) -> Result<(), ZipError> { + write_data_impl(&mut self.writer, data) + } + + /// Writes file content from a `BufferedZipFile` to the current file in the zip archive. + pub fn write_buffered_zip_file_content( + &mut self, + file: &mut BufferedZipFile, + ) -> Result<(), ZipError> { + do_copy_impl(&mut self.writer, file) + } + + /// Writes file content from a `FsZipFile` to the current file in the zip archive. + pub fn write_fs_zip_file_content(&mut self, file: &mut FsZipFile) -> Result<(), ZipError> { + do_copy_impl(&mut self.writer, file) + } + + /// Writes file content from a path to the current file in the zip archive. + pub fn write_file_content(&mut self, path: &[u8]) -> Result<(), ZipError> { + write_file_content_impl(&mut self.writer, path) + } + + /// Archive comments are not supported by streaming writers: zip 9.x + /// finalizes a commented archive by seeking back over the comment, which a + /// non-seekable writer cannot do (`zip::write::StreamWriter` only permits + /// no-op seeks). Always returns a FailedPrecondition error. + pub fn set_comment(&mut self, _comment: &[u8]) -> Result<(), ZipError> { + Err(ZipError::failed_precondition( + "archive comments are not supported by streaming zip writers", + )) + } + + /// Flushes any pending output. + pub fn flush(&mut self) -> Result<(), ZipError> { + flush_impl(&mut self.writer) + } +} + +/// Creates a new in-memory streaming zip writer. +pub fn new_buffered_zip_stream_writer() -> BufferedZipStreamWriter { + BufferedZipStreamWriter::new_stream() +} + +/// Creates a new filesystem streaming zip writer. +pub fn new_fs_zip_stream_writer(path: &[u8]) -> Result { + FsZipStreamWriter::new_from_path(path) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::BufferedZipArchive; + use googletest::prelude::*; + + #[gtest] + fn test_buffered_zip_stream_writer() { + let mut writer = BufferedZipStreamWriter::new_stream(); + expect_false!(writer.is_none()); + expect_false!(writer.is_seek_possible()); + + let options = ZipWriterFileOptions::new().compression_method(CompressionMethod::Deflated); + writer.start_file(b"test.txt", options).unwrap(); + writer.write_data(VecU8::from(b"stream writer data".to_vec())).unwrap(); + writer.add_directory(b"mydir", options).unwrap(); + // Comments need a seek on finish, which a stream writer cannot do. + expect_true!(writer.set_comment(b"archive comment").is_err()); + writer.flush().unwrap(); + + let zip_bytes = writer.finish().unwrap(); + expect_false!(zip_bytes.is_empty()); + + // Verify that ZipArchive can read this streaming-written zip archive + let mut archive = BufferedZipArchive::new_from_data(zip_bytes).unwrap(); + expect_eq!(archive.get_length(), 2); + expect_true!(archive.get_comment().is_empty()); + + { + let mut file1 = archive.get_file_by_index(0).unwrap(); + expect_eq!(file1.get_file_name().as_slice(), b"test.txt"); + expect_eq!(file1.get_file_data().unwrap().as_slice(), b"stream writer data"); + } + + { + let file2 = archive.get_file_by_index(1).unwrap(); + expect_true!(file2.is_dir()); + expect_eq!(file2.get_file_name().as_slice(), b"mydir/"); + } + } + + #[gtest] + fn test_free_functions_stream_writer() { + let mut writer = new_buffered_zip_stream_writer(); + expect_false!(writer.is_seek_possible()); + let options = ZipWriterFileOptions::new(); + writer.start_file(b"a.txt", options).unwrap(); + writer.write_data(VecU8::from(b"content".to_vec())).unwrap(); + let zip_bytes = writer.finish().unwrap(); + + let mut archive = BufferedZipArchive::new_from_data(zip_bytes).unwrap(); + expect_eq!(archive.get_length(), 1); + let mut file = archive.get_file_by_index(0).unwrap(); + expect_eq!(file.get_file_data().unwrap().as_slice(), b"content"); + } + + #[gtest] + fn test_regular_writer_is_seek_possible_and_set_comment() { + let mut writer = BufferedZipWriter::new_from_data(VecU8::default(), false).unwrap(); + expect_true!(writer.is_seek_possible()); + let options = ZipWriterFileOptions::new(); + writer.start_file(b"b.txt", options).unwrap(); + writer.write_data(VecU8::from(b"data".to_vec())).unwrap(); + writer.set_comment(b"comment on regular writer").unwrap(); + writer.flush().unwrap(); + let zip_bytes = writer.finish().unwrap(); + + let archive = BufferedZipArchive::new_from_data(zip_bytes).unwrap(); + expect_eq!(archive.get_comment().as_slice(), b"comment on regular writer"); + } +} diff --git a/zip/write.cc b/zip/write.cc index d5b49432..28fd9ff9 100644 --- a/zip/write.cc +++ b/zip/write.cc @@ -91,6 +91,21 @@ absl::Status BufferedZipWriter::WriteFileContent(absl::string_view path) { reinterpret_cast(path.data()), path.size()))); } +absl::Status BufferedZipWriter::SetComment(absl::string_view comment) { + return FromRustResultUnit(writer_.set_comment(absl::Span( + reinterpret_cast(comment.data()), comment.size()))); +} + +absl::Status BufferedZipWriter::Flush() { + return FromRustResultUnit(writer_.flush()); +} + +bool BufferedZipWriter::IsSeekPossible() const { + return writer_.is_seek_possible(); +} + +bool BufferedZipWriter::IsNone() const { return writer_.is_none(); } + absl::StatusOr FsZipWriter::NewFromPath(absl::string_view path, bool append) { ABSL_ASSIGN_OR_RETURN( @@ -148,6 +163,19 @@ absl::Status FsZipWriter::WriteFileContent(absl::string_view path) { reinterpret_cast(path.data()), path.size()))); } +absl::Status FsZipWriter::SetComment(absl::string_view comment) { + return FromRustResultUnit(writer_.set_comment(absl::Span( + reinterpret_cast(comment.data()), comment.size()))); +} + +absl::Status FsZipWriter::Flush() { + return FromRustResultUnit(writer_.flush()); +} + +bool FsZipWriter::IsSeekPossible() const { return writer_.is_seek_possible(); } + +bool FsZipWriter::IsNone() const { return writer_.is_none(); } + absl::StatusOr ZipWriter::FromFile(absl::string_view path, bool append) { ABSL_ASSIGN_OR_RETURN(FsZipWriter writer, @@ -207,4 +235,241 @@ absl::Status ZipWriter::WriteFileContent(absl::string_view path) { writer_); } +absl::Status ZipWriter::SetComment(absl::string_view comment) { + return std::visit([&](auto& writer) { return writer.SetComment(comment); }, + writer_); +} + +absl::Status ZipWriter::Flush() { + return std::visit([&](auto& writer) { return writer.Flush(); }, writer_); +} + +bool ZipWriter::IsSeekPossible() const { + return std::visit([&](const auto& writer) { return writer.IsSeekPossible(); }, + writer_); +} + +bool ZipWriter::IsNone() const { + return std::visit([&](const auto& writer) { return writer.IsNone(); }, + writer_); +} + +absl::StatusOr BufferedZipStreamWriter::New() { + return BufferedZipStreamWriter( + rust::BufferedZipStreamWriter::new_stream()); +} + +absl::StatusOr BufferedZipStreamWriter::Finish() { + return FromRustResultVecU8(writer_.finish()); +} + +absl::Status BufferedZipStreamWriter::StartFile( + absl::string_view file_name, const ZipWriterFileOptions& options) { + return FromRustResultUnit(writer_.start_file( + absl::Span( + reinterpret_cast(file_name.data()), file_name.size()), + options.options_)); +} + +absl::Status BufferedZipStreamWriter::AddDirectory( + absl::string_view file_name, const ZipWriterFileOptions& options) { + return FromRustResultUnit(writer_.add_directory( + absl::Span( + reinterpret_cast(file_name.data()), file_name.size()), + options.options_)); +} + +absl::Status BufferedZipStreamWriter::WriteData(absl::string_view data) { + rust::VecU8 input_data = + rust::VecU8::copy_from_slice(absl::Span( + reinterpret_cast(data.data()), data.size())); + return FromRustResultUnit(writer_.write_data(input_data)); +} + +absl::Status BufferedZipStreamWriter::WriteZipFileContent(ZipFile& file) { + return std::visit( + absl::Overload{[&](BufferedZipFile& file) { + return FromRustResultUnit( + writer_.write_buffered_zip_file_content(file.zip_)); + }, + [&](FsZipFile& file) { + return FromRustResultUnit( + writer_.write_fs_zip_file_content(file.zip_)); + }}, + file.zip_); +} + +absl::Status BufferedZipStreamWriter::WriteFileContent(absl::string_view path) { + return FromRustResultUnit( + writer_.write_file_content(absl::Span( + reinterpret_cast(path.data()), path.size()))); +} + +absl::Status BufferedZipStreamWriter::SetComment(absl::string_view comment) { + return FromRustResultUnit(writer_.set_comment(absl::Span( + reinterpret_cast(comment.data()), comment.size()))); +} + +absl::Status BufferedZipStreamWriter::Flush() { + return FromRustResultUnit(writer_.flush()); +} + +bool BufferedZipStreamWriter::IsSeekPossible() const { + return writer_.is_seek_possible(); +} + +bool BufferedZipStreamWriter::IsNone() const { return writer_.is_none(); } + +absl::StatusOr FsZipStreamWriter::NewFromPath( + absl::string_view path) { + ABSL_ASSIGN_OR_RETURN( + rust::FsZipStreamWriter writer, + FromRustFsZipStreamWriter(rust::FsZipStreamWriter::new_from_path( + absl::Span( + reinterpret_cast(path.data()), path.size())))); + return FsZipStreamWriter(std::move(writer)); +} + +absl::Status FsZipStreamWriter::Finish() { + return FromRustResultUnit(writer_.finish()); +} + +absl::Status FsZipStreamWriter::StartFile(absl::string_view file_name, + const ZipWriterFileOptions& options) { + return FromRustResultUnit(writer_.start_file( + absl::Span( + reinterpret_cast(file_name.data()), file_name.size()), + options.options_)); +} + +absl::Status FsZipStreamWriter::AddDirectory( + absl::string_view file_name, const ZipWriterFileOptions& options) { + return FromRustResultUnit(writer_.add_directory( + absl::Span( + reinterpret_cast(file_name.data()), file_name.size()), + options.options_)); +} + +absl::Status FsZipStreamWriter::WriteData(absl::string_view data) { + rust::VecU8 input_data = + rust::VecU8::copy_from_slice(absl::Span( + reinterpret_cast(data.data()), data.size())); + return FromRustResultUnit(writer_.write_data(input_data)); +} + +absl::Status FsZipStreamWriter::WriteZipFileContent(ZipFile& file) { + return std::visit( + absl::Overload{[&](BufferedZipFile& file) { + return FromRustResultUnit( + writer_.write_buffered_zip_file_content(file.zip_)); + }, + [&](FsZipFile& file) { + return FromRustResultUnit( + writer_.write_fs_zip_file_content(file.zip_)); + }}, + file.zip_); +} + +absl::Status FsZipStreamWriter::WriteFileContent(absl::string_view path) { + return FromRustResultUnit( + writer_.write_file_content(absl::Span( + reinterpret_cast(path.data()), path.size()))); +} + +absl::Status FsZipStreamWriter::SetComment(absl::string_view comment) { + return FromRustResultUnit(writer_.set_comment(absl::Span( + reinterpret_cast(comment.data()), comment.size()))); +} + +absl::Status FsZipStreamWriter::Flush() { + return FromRustResultUnit(writer_.flush()); +} + +bool FsZipStreamWriter::IsSeekPossible() const { + return writer_.is_seek_possible(); +} + +bool FsZipStreamWriter::IsNone() const { return writer_.is_none(); } + +absl::StatusOr ZipStreamWriter::FromFile( + absl::string_view path) { + ABSL_ASSIGN_OR_RETURN(FsZipStreamWriter writer, + FsZipStreamWriter::NewFromPath(path)); + return ZipStreamWriter(std::move(writer)); +} + +absl::StatusOr ZipStreamWriter::FromBuffer() { + ABSL_ASSIGN_OR_RETURN(BufferedZipStreamWriter writer, + BufferedZipStreamWriter::New()); + return ZipStreamWriter(std::move(writer)); +} + +absl::StatusOr ZipStreamWriter::Finish() { + return std::visit( + absl::Overload{ + [](BufferedZipStreamWriter& writer) + -> absl::StatusOr { return writer.Finish(); }, + [](FsZipStreamWriter& writer) -> absl::StatusOr { + ABSL_RETURN_IF_ERROR(writer.Finish()); + return RustVecU8Wrapper(); + }}, + writer_); +} + +absl::Status ZipStreamWriter::StartFile(absl::string_view file_name, + const ZipWriterFileOptions& options) { + return std::visit( + [&](auto& writer) { return writer.StartFile(file_name, options); }, + writer_); +} + +absl::Status ZipStreamWriter::AddDirectory( + absl::string_view file_name, const ZipWriterFileOptions& options) { + return std::visit( + [&](auto& writer) { return writer.AddDirectory(file_name, options); }, + writer_); +} + +absl::Status ZipStreamWriter::WriteData(absl::string_view data) { + return std::visit([&](auto& writer) { return writer.WriteData(data); }, + writer_); +} + +absl::Status ZipStreamWriter::WriteZipFileContent(ZipFile& file) { + return std::visit( + [&](auto& writer) { return writer.WriteZipFileContent(file); }, writer_); +} + +absl::Status ZipStreamWriter::WriteFileContent(absl::string_view path) { + return std::visit([&](auto& writer) { return writer.WriteFileContent(path); }, + writer_); +} + +absl::Status ZipStreamWriter::SetComment(absl::string_view comment) { + return std::visit([&](auto& writer) { return writer.SetComment(comment); }, + writer_); +} + +absl::Status ZipStreamWriter::Flush() { + return std::visit([&](auto& writer) { return writer.Flush(); }, writer_); +} + +bool ZipStreamWriter::IsSeekPossible() const { + return std::visit([&](const auto& writer) { return writer.IsSeekPossible(); }, + writer_); +} + +bool ZipStreamWriter::IsNone() const { + return std::visit([&](const auto& writer) { return writer.IsNone(); }, + writer_); +} + +absl::StatusOr NewBufferedZipStreamWriter() { + return BufferedZipStreamWriter::New(); +} + +absl::StatusOr NewFsZipStreamWriter(absl::string_view path) { + return FsZipStreamWriter::NewFromPath(path); +} + } // namespace security::zip diff --git a/zip/write.h b/zip/write.h index 44775b5a..bc9b40e9 100644 --- a/zip/write.h +++ b/zip/write.h @@ -16,6 +16,9 @@ namespace security::zip { class BufferedZipWriter; class FsZipWriter; +class BufferedZipStreamWriter; +class FsZipStreamWriter; +class ZipStreamWriter; class ZipWriterFileOptions { public: @@ -40,6 +43,9 @@ class ZipWriterFileOptions { rust::ZipWriterFileOptions options_; friend class BufferedZipWriter; friend class FsZipWriter; + friend class BufferedZipStreamWriter; + friend class FsZipStreamWriter; + friend class ZipStreamWriter; }; class BufferedZipWriter final { @@ -54,6 +60,10 @@ class BufferedZipWriter final { absl::Status WriteData(absl::string_view data); absl::Status WriteZipFileContent(ZipFile& file); absl::Status WriteFileContent(absl::string_view path); + absl::Status SetComment(absl::string_view comment); + absl::Status Flush(); + [[nodiscard]] bool IsSeekPossible() const; + [[nodiscard]] bool IsNone() const; private: explicit BufferedZipWriter(rust::BufferedZipWriter writer) @@ -73,6 +83,10 @@ class FsZipWriter final { absl::Status WriteData(absl::string_view data); absl::Status WriteZipFileContent(ZipFile& file); absl::Status WriteFileContent(absl::string_view path); + absl::Status SetComment(absl::string_view comment); + absl::Status Flush(); + [[nodiscard]] bool IsSeekPossible() const; + [[nodiscard]] bool IsNone() const; private: explicit FsZipWriter(rust::FsZipWriter writer) @@ -94,6 +108,10 @@ class ZipWriter final { absl::Status WriteData(absl::string_view data); absl::Status WriteZipFileContent(ZipFile& file); absl::Status WriteFileContent(absl::string_view path); + absl::Status SetComment(absl::string_view comment); + absl::Status Flush(); + [[nodiscard]] bool IsSeekPossible() const; + [[nodiscard]] bool IsNone() const; private: using BackendType = std::variant; @@ -102,6 +120,78 @@ class ZipWriter final { BackendType writer_; }; +// Streaming zip writers write zip archives sequentially without seeking. +class BufferedZipStreamWriter final { + public: + static absl::StatusOr New(); + absl::StatusOr Finish(); + absl::Status StartFile(absl::string_view file_name, + const ZipWriterFileOptions& options); + absl::Status AddDirectory(absl::string_view file_name, + const ZipWriterFileOptions& options); + absl::Status WriteData(absl::string_view data); + absl::Status WriteZipFileContent(ZipFile& file); + absl::Status WriteFileContent(absl::string_view path); + absl::Status SetComment(absl::string_view comment); + absl::Status Flush(); + [[nodiscard]] bool IsSeekPossible() const; + [[nodiscard]] bool IsNone() const; + + private: + explicit BufferedZipStreamWriter(rust::BufferedZipStreamWriter writer) + : writer_(std::move(writer)) {} + rust::BufferedZipStreamWriter writer_; +}; + +class FsZipStreamWriter final { + public: + static absl::StatusOr NewFromPath(absl::string_view path); + absl::Status Finish(); + absl::Status StartFile(absl::string_view file_name, + const ZipWriterFileOptions& options); + absl::Status AddDirectory(absl::string_view file_name, + const ZipWriterFileOptions& options); + absl::Status WriteData(absl::string_view data); + absl::Status WriteZipFileContent(ZipFile& file); + absl::Status WriteFileContent(absl::string_view path); + absl::Status SetComment(absl::string_view comment); + absl::Status Flush(); + [[nodiscard]] bool IsSeekPossible() const; + [[nodiscard]] bool IsNone() const; + + private: + explicit FsZipStreamWriter(rust::FsZipStreamWriter writer) + : writer_(std::move(writer)) {} + rust::FsZipStreamWriter writer_; +}; + +class ZipStreamWriter final { + public: + static absl::StatusOr FromFile(absl::string_view path); + static absl::StatusOr FromBuffer(); + absl::StatusOr Finish(); + absl::Status StartFile(absl::string_view file_name, + const ZipWriterFileOptions& options); + absl::Status AddDirectory(absl::string_view file_name, + const ZipWriterFileOptions& options); + absl::Status WriteData(absl::string_view data); + absl::Status WriteZipFileContent(ZipFile& file); + absl::Status WriteFileContent(absl::string_view path); + absl::Status SetComment(absl::string_view comment); + absl::Status Flush(); + [[nodiscard]] bool IsSeekPossible() const; + [[nodiscard]] bool IsNone() const; + + private: + using BackendType = std::variant; + + explicit ZipStreamWriter(BackendType writer) : writer_(std::move(writer)) {} + BackendType writer_; +}; + +absl::StatusOr NewBufferedZipStreamWriter(); +absl::StatusOr NewFsZipStreamWriter(absl::string_view path); + } // namespace security::zip #endif // SECURITY_ZIP_WRITE_H_ diff --git a/zip/zip_benchmark.cc b/zip/zip_benchmark.cc new file mode 100644 index 00000000..dbd7818c --- /dev/null +++ b/zip/zip_benchmark.cc @@ -0,0 +1,231 @@ +#include +#include +#include +#include + +#include "converters.h" +#include "file.h" +#include "read.h" +#include "write.h" +#include "testing/base/public/benchmark.h" +#include "absl/log/check.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" + +namespace security::zip { +namespace { + +std::string GenerateTestData(size_t size) { + std::string data; + data.reserve(size); + for (size_t i = 0; i < size; ++i) { + data.push_back(static_cast((i * 73 + 19) % 256)); + } + return data; +} + +std::string BuildTestZip(size_t file_count, size_t file_size, + CompressionMethod method) { + absl::StatusOr writer_or = + ZipWriter::FromBuffer("", /*append=*/false); + CHECK_OK(writer_or.status()); + ZipWriter writer = *std::move(writer_or); + ZipWriterFileOptions options; + CHECK_OK(options.SetCompressionMethod(method)); + std::string payload = GenerateTestData(file_size); + for (size_t i = 0; i < file_count; ++i) { + std::string name = absl::StrCat("entry_", i, ".bin"); + CHECK_OK(writer.StartFile(name, options)); + CHECK_OK(writer.WriteData(payload)); + } + absl::StatusOr res_or = writer.Finish(); + CHECK_OK(res_or.status()); + RustVecU8Wrapper res = *std::move(res_or); + return std::string(res.data(), res.size()); +} + +// Benchmark streaming write throughput. +void BM_StreamWriter_Write(benchmark::State& state) { + const size_t file_count = static_cast(state.range(0)); + const size_t file_size = static_cast(state.range(1)); + const std::string payload = GenerateTestData(file_size); + + ZipWriterFileOptions options; + CHECK_OK(options.SetCompressionMethod(CompressionMethod::kStored)); + + for (auto _ : state) { + absl::StatusOr writer_or = ZipStreamWriter::FromBuffer(); + CHECK_OK(writer_or.status()); + ZipStreamWriter writer = *std::move(writer_or); + for (size_t i = 0; i < file_count; ++i) { + std::string name = absl::StrCat("stream_file_", i, ".bin"); + CHECK_OK(writer.StartFile(name, options)); + CHECK_OK(writer.WriteData(payload)); + } + auto result = writer.Finish(); + benchmark::DoNotOptimize(result); + } + + state.SetBytesProcessed(state.iterations() * file_count * file_size); +} +BENCHMARK(BM_StreamWriter_Write) + ->Args({1, 1024}) + ->Args({10, 1024}) + ->Args({1, 65536}) + ->Args({10, 65536}); + +// Benchmark seekable write throughput. +void BM_SeekableWriter_Write(benchmark::State& state) { + const size_t file_count = static_cast(state.range(0)); + const size_t file_size = static_cast(state.range(1)); + const std::string payload = GenerateTestData(file_size); + + ZipWriterFileOptions options; + CHECK_OK(options.SetCompressionMethod(CompressionMethod::kStored)); + + for (auto _ : state) { + absl::StatusOr writer_or = + ZipWriter::FromBuffer("", /*append=*/false); + CHECK_OK(writer_or.status()); + ZipWriter writer = *std::move(writer_or); + for (size_t i = 0; i < file_count; ++i) { + std::string name = absl::StrCat("seek_file_", i, ".bin"); + CHECK_OK(writer.StartFile(name, options)); + CHECK_OK(writer.WriteData(payload)); + } + auto result = writer.Finish(); + benchmark::DoNotOptimize(result); + } + + state.SetBytesProcessed(state.iterations() * file_count * file_size); +} +BENCHMARK(BM_SeekableWriter_Write) + ->Args({1, 1024}) + ->Args({10, 1024}) + ->Args({1, 65536}) + ->Args({10, 65536}); + +// Benchmark streaming write with Deflate compression. +void BM_StreamWriter_Deflated(benchmark::State& state) { + const size_t file_size = static_cast(state.range(0)); + const std::string payload = GenerateTestData(file_size); + + ZipWriterFileOptions options; + CHECK_OK(options.SetCompressionMethod(CompressionMethod::kDeflated)); + + for (auto _ : state) { + absl::StatusOr writer_or = ZipStreamWriter::FromBuffer(); + CHECK_OK(writer_or.status()); + ZipStreamWriter writer = *std::move(writer_or); + CHECK_OK(writer.StartFile("deflated.bin", options)); + CHECK_OK(writer.WriteData(payload)); + auto result = writer.Finish(); + benchmark::DoNotOptimize(result); + } + + state.SetBytesProcessed(state.iterations() * file_size); +} +BENCHMARK(BM_StreamWriter_Deflated)->Args({1024})->Args({65536}); + +// Benchmark streaming read throughput. +void BM_StreamReader_SequentialRead(benchmark::State& state) { + const size_t file_count = static_cast(state.range(0)); + const size_t file_size = static_cast(state.range(1)); + const std::string zip_data = + BuildTestZip(file_count, file_size, CompressionMethod::kStored); + + for (auto _ : state) { + absl::StatusOr reader_or = + ZipStreamReader::FromBuffer(zip_data); + CHECK_OK(reader_or.status()); + ZipStreamReader reader = *std::move(reader_or); + for (size_t i = 0; i < file_count; ++i) { + absl::StatusOr> file_opt_or = + reader.ReadNextFile(); + CHECK_OK(file_opt_or.status()); + std::optional file_opt = *std::move(file_opt_or); + if (!file_opt.has_value()) break; + absl::StatusOr content_or = file_opt->GetFileData(); + CHECK_OK(content_or.status()); + RustVecU8Wrapper content = *std::move(content_or); + benchmark::DoNotOptimize(content); + } + } + + state.SetBytesProcessed(state.iterations() * file_count * file_size); +} +BENCHMARK(BM_StreamReader_SequentialRead) + ->Args({1, 1024}) + ->Args({10, 1024}) + ->Args({1, 65536}) + ->Args({10, 65536}); + +// Benchmark seekable archive read throughput. +void BM_ArchiveReader_GetFileByIndex(benchmark::State& state) { + const size_t file_count = static_cast(state.range(0)); + const size_t file_size = static_cast(state.range(1)); + const std::string zip_data = + BuildTestZip(file_count, file_size, CompressionMethod::kStored); + + for (auto _ : state) { + absl::StatusOr archive_or = ZipArchive::FromBuffer(zip_data); + CHECK_OK(archive_or.status()); + ZipArchive archive = *std::move(archive_or); + for (size_t i = 0; i < file_count; ++i) { + absl::StatusOr file_or = archive.GetFileByIndex(i); + CHECK_OK(file_or.status()); + ZipFile file = *std::move(file_or); + absl::StatusOr content_or = file.GetFileData(); + CHECK_OK(content_or.status()); + RustVecU8Wrapper content = *std::move(content_or); + benchmark::DoNotOptimize(content); + } + } + + state.SetBytesProcessed(state.iterations() * file_count * file_size); +} +BENCHMARK(BM_ArchiveReader_GetFileByIndex) + ->Args({1, 1024}) + ->Args({10, 1024}) + ->Args({1, 65536}) + ->Args({10, 65536}); + +// Benchmark streaming chunked reading. +void BM_StreamReader_ChunkedRead(benchmark::State& state) { + const size_t chunk_size = static_cast(state.range(0)); + constexpr size_t kFileSize = 65536; + const std::string zip_data = + BuildTestZip(1, kFileSize, CompressionMethod::kStored); + + for (auto _ : state) { + absl::StatusOr reader_or = + ZipStreamReader::FromBuffer(zip_data); + CHECK_OK(reader_or.status()); + ZipStreamReader reader = *std::move(reader_or); + absl::StatusOr> file_opt_or = reader.ReadNextFile(); + CHECK_OK(file_opt_or.status()); + std::optional file_opt = *std::move(file_opt_or); + if (file_opt.has_value()) { + size_t total_read = 0; + while (total_read < kFileSize) { + absl::StatusOr chunk_or = + file_opt->ReadBytes(chunk_size); + CHECK_OK(chunk_or.status()); + RustVecU8Wrapper chunk = *std::move(chunk_or); + if (chunk.empty()) break; + total_read += chunk.size(); + benchmark::DoNotOptimize(chunk); + } + } + } + + state.SetBytesProcessed(state.iterations() * kFileSize); +} +BENCHMARK(BM_StreamReader_ChunkedRead) + ->Args({1024}) + ->Args({4096}) + ->Args({16384}); + +} // namespace +} // namespace security::zip