diff --git a/zip/converters.cc b/zip/converters.cc index faac231a..2051fff1 100644 --- a/zip/converters.cc +++ b/zip/converters.cc @@ -1,5 +1,6 @@ #include "converters.h" +#include #include #include @@ -51,6 +52,14 @@ absl::Status FromRustResultUnit( return absl::OkStatus(); } +absl::StatusOr FromRustResultU64( + rs_std::Result result_u64) { + if (!result_u64.has_value()) { + return ZipErrorToStatus(std::move(result_u64).err()); + } + return std::move(result_u64).value(); +} + absl::StatusOr FromRustBufferedZipArchive( rs_std::Result result_buffered_zip_archive) { @@ -87,6 +96,15 @@ absl::StatusOr FromRustFsZipFile( return std::move(result_fs_zip_file).value(); } +absl::StatusOr FromRustZipEntryMetadata( + rs_std::Result + result_zip_entry_metadata) { + if (!result_zip_entry_metadata.has_value()) { + return ZipErrorToStatus(std::move(result_zip_entry_metadata).err()); + } + return std::move(result_zip_entry_metadata).value(); +} + absl::StatusOr FromRustBufferedZipWriter( rs_std::Result result_buffered_zip_writer) { @@ -105,4 +123,32 @@ absl::StatusOr FromRustFsZipWriter( return std::move(result_fs_zip_writer).value(); } +absl::StatusOr +FromRustBufferedZipStreamReader( + rs_std::Result + result_buffered_zip_stream_reader) { + if (!result_buffered_zip_stream_reader.has_value()) { + return ZipErrorToStatus(std::move(result_buffered_zip_stream_reader).err()); + } + return std::move(result_buffered_zip_stream_reader).value(); +} + +absl::StatusOr FromRustFsZipStreamReader( + rs_std::Result + result_fs_zip_stream_reader) { + if (!result_fs_zip_stream_reader.has_value()) { + return ZipErrorToStatus(std::move(result_fs_zip_stream_reader).err()); + } + return std::move(result_fs_zip_stream_reader).value(); +} + +absl::StatusOr FromRustFsZipStreamWriter( + rs_std::Result + result_fs_zip_stream_writer) { + if (!result_fs_zip_stream_writer.has_value()) { + return ZipErrorToStatus(std::move(result_fs_zip_stream_writer).err()); + } + return std::move(result_fs_zip_stream_writer).value(); +} + } // namespace security::zip diff --git a/zip/converters.h b/zip/converters.h index a75bb805..82281bd7 100644 --- a/zip/converters.h +++ b/zip/converters.h @@ -1,6 +1,8 @@ #ifndef SECURITY_ZIP_CONVERTERS_H_ #define SECURITY_ZIP_CONVERTERS_H_ +#include +#include #include #include "crubit_helpers/string_conversions.h" @@ -49,11 +51,30 @@ class RustVecU8Wrapper { // // Consuming inside the full-expression is safe: // std::string owned(f.GetFileData()->AsStringView()); // OK - absl::string_view AsStringView() const& ABSL_ATTRIBUTE_LIFETIME_BOUND { + [[nodiscard]] absl::string_view AsStringView() + const& ABSL_ATTRIBUTE_LIFETIME_BOUND { return security::crubit_helpers::StringViewFromVecU8(vec_u8_); } absl::string_view AsStringView() const&& = delete; + // Same lifetime rules as AsStringView(); see the warning above. The `const&&` + // overloads are deleted so that the pointer/view cannot be taken from a + // temporary. + [[nodiscard]] const char* data() const& ABSL_ATTRIBUTE_LIFETIME_BOUND { + return AsStringView().data(); + } + const char* data() const&& = delete; + + explicit operator absl::string_view() const& ABSL_ATTRIBUTE_LIFETIME_BOUND { + return AsStringView(); + } + explicit operator absl::string_view() const&& = delete; + + // size() and empty() return by value, so they are safe on temporaries. + [[nodiscard]] size_t size() const { return AsStringView().size(); } + + [[nodiscard]] bool empty() const { return AsStringView().empty(); } + private: rust::VecU8 vec_u8_; }; @@ -62,6 +83,10 @@ absl::StatusOr FromRustResultVecU8( rs_std::Result result_vec_u8); absl::Status FromRustResultUnit( rs_std::Result result_unit); +// Converts a byte count, as returned by the reads that write into a buffer the +// caller owns rather than returning one. +absl::StatusOr FromRustResultU64( + rs_std::Result result_u64); absl::StatusOr FromRustBufferedZipArchive( rs_std::Result @@ -76,6 +101,9 @@ absl::StatusOr FromRustBufferedZipFile( absl::StatusOr FromRustFsZipFile( rs_std::Result result_fs_zip_file); +absl::StatusOr FromRustZipEntryMetadata( + rs_std::Result + result_zip_entry_metadata); absl::StatusOr FromRustBufferedZipWriter( rs_std::Result @@ -84,6 +112,18 @@ absl::StatusOr FromRustFsZipWriter( rs_std::Result result_fs_zip_writer); +absl::StatusOr +FromRustBufferedZipStreamReader( + rs_std::Result + result_buffered_zip_stream_reader); +absl::StatusOr FromRustFsZipStreamReader( + rs_std::Result + result_fs_zip_stream_reader); + +absl::StatusOr FromRustFsZipStreamWriter( + rs_std::Result + result_fs_zip_stream_writer); + } // namespace security::zip #endif // SECURITY_ZIP_CONVERTERS_H_ diff --git a/zip/file.cc b/zip/file.cc index 92604a77..754a1393 100644 --- a/zip/file.cc +++ b/zip/file.cc @@ -11,6 +11,8 @@ #include "absl/status/status.h" #include "absl/status/status_macros.h" #include "absl/status/statusor.h" +#include "absl/types/span.h" +#include "support/rs_std/slice_ref.h" namespace security::zip { namespace { @@ -36,8 +38,64 @@ absl::StatusOr ToSecurityZipCompressionMethod( return absl::InvalidArgumentError("Unknown compression method"); } +// Rust writes into the caller's buffer through this; the slice borrows it and +// must not outlive the call it is passed to. +rs_std::SliceRef AsMutableSlice(absl::Span buf) { + return rs_std::SliceRef( + absl::Span(reinterpret_cast(buf.data()), buf.size())); +} + +// Shared by the backends' ReadInto and Skip, whose bodies are otherwise +// identical. `Zip` is a Rust entry type (rust::{Buffered,Fs}ZipFile). +template +absl::StatusOr ReadIntoImpl(Zip& zip, absl::Span buf) { + ABSL_ASSIGN_OR_RETURN(const uint64_t read, + FromRustResultU64(zip.read_into(AsMutableSlice(buf)))); + return static_cast(read); +} +template +absl::StatusOr SkipImpl(Zip& zip, uint64_t max_bytes) { + return FromRustResultU64(zip.skip(max_bytes)); +} + } // namespace +bool ZipEntryMetadata::IsFile() const { return metadata_.is_file(); } +bool ZipEntryMetadata::IsDir() const { return metadata_.is_dir(); } +bool ZipEntryMetadata::IsEncrypted() const { return metadata_.is_encrypted(); } +std::string ZipEntryMetadata::GetFileName() const { + rust::VecU8 name_vec = metadata_.get_file_name(); + return std::string(security::crubit_helpers::StringViewFromVecU8(name_vec)); +} +absl::StatusOr ZipEntryMetadata::GetCompressionMethod() + const { + return ToSecurityZipCompressionMethod(metadata_.get_compression_method()); +} +std::string ZipEntryMetadata::GetComment() const { + rust::VecU8 comment_vec = metadata_.get_comment(); + return std::string( + security::crubit_helpers::StringViewFromVecU8(comment_vec)); +} +uint64_t ZipEntryMetadata::GetUncompressedSize() const { + return metadata_.get_uncompressed_size(); +} +uint64_t ZipEntryMetadata::GetCompressedSize() const { + return metadata_.get_compressed_size(); +} +uint32_t ZipEntryMetadata::GetCrc32() const { return metadata_.get_crc32(); } +uint16_t ZipEntryMetadata::GetLastModifiedDate() const { + return metadata_.get_last_modified_date(); +} +uint16_t ZipEntryMetadata::GetLastModifiedTime() const { + return metadata_.get_last_modified_time(); +} +uint32_t ZipEntryMetadata::GetUnixMode() const { + return metadata_.get_unix_mode(); +} +uint64_t ZipEntryMetadata::GetHeaderStart() const { + return metadata_.get_header_start(); +} + absl::Status BufferedZipFile::CheckNone() const { if (IsNone()) { return absl::InternalError("ZipFile object used after being moved-out"); @@ -98,6 +156,14 @@ absl::StatusOr BufferedZipFile::GetExtraData() const { ABSL_RETURN_IF_ERROR(CheckNone()); return RustVecU8Wrapper(zip_.get_extra_data()); } +absl::StatusOr BufferedZipFile::GetHeaderStart() const { + ABSL_RETURN_IF_ERROR(CheckNone()); + return zip_.get_header_start(); +} +absl::StatusOr BufferedZipFile::GetDataStart() const { + ABSL_RETURN_IF_ERROR(CheckNone()); + return zip_.get_data_start(); +} absl::StatusOr BufferedZipFile::GetFileData() { ABSL_RETURN_IF_ERROR(CheckNone()); return FromRustResultVecU8(zip_.get_file_data()); @@ -108,6 +174,18 @@ absl::StatusOr BufferedZipFile::GetFileData(size_t max_size) { } return FromRustResultVecU8(zip_.get_file_data_with_max_size(max_size)); } +absl::StatusOr BufferedZipFile::ReadBytes(size_t max_bytes) { + ABSL_RETURN_IF_ERROR(CheckNone()); + return FromRustResultVecU8(zip_.read_bytes(max_bytes)); +} +absl::StatusOr BufferedZipFile::ReadInto(absl::Span buf) { + ABSL_RETURN_IF_ERROR(CheckNone()); + return ReadIntoImpl(zip_, buf); +} +absl::StatusOr BufferedZipFile::Skip(uint64_t max_bytes) { + ABSL_RETURN_IF_ERROR(CheckNone()); + return SkipImpl(zip_, max_bytes); +} absl::Status FsZipFile::CheckNone() const { if (IsNone()) { @@ -168,6 +246,14 @@ absl::StatusOr FsZipFile::GetExtraData() const { ABSL_RETURN_IF_ERROR(CheckNone()); return RustVecU8Wrapper(zip_.get_extra_data()); } +absl::StatusOr FsZipFile::GetHeaderStart() const { + ABSL_RETURN_IF_ERROR(CheckNone()); + return zip_.get_header_start(); +} +absl::StatusOr FsZipFile::GetDataStart() const { + ABSL_RETURN_IF_ERROR(CheckNone()); + return zip_.get_data_start(); +} absl::StatusOr FsZipFile::GetFileData() { ABSL_RETURN_IF_ERROR(CheckNone()); return FromRustResultVecU8(zip_.get_file_data()); @@ -178,6 +264,18 @@ absl::StatusOr FsZipFile::GetFileData(size_t max_size) { } return FromRustResultVecU8(zip_.get_file_data_with_max_size(max_size)); } +absl::StatusOr FsZipFile::ReadBytes(size_t max_bytes) { + ABSL_RETURN_IF_ERROR(CheckNone()); + return FromRustResultVecU8(zip_.read_bytes(max_bytes)); +} +absl::StatusOr FsZipFile::ReadInto(absl::Span buf) { + ABSL_RETURN_IF_ERROR(CheckNone()); + return ReadIntoImpl(zip_, buf); +} +absl::StatusOr FsZipFile::Skip(uint64_t max_bytes) { + ABSL_RETURN_IF_ERROR(CheckNone()); + return SkipImpl(zip_, max_bytes); +} absl::StatusOr ZipFile::IsFile() const { return std::visit([](const auto& zip) { return zip.IsFile(); }, zip_); @@ -223,6 +321,12 @@ absl::StatusOr ZipFile::GetUnixMode() const { absl::StatusOr ZipFile::GetExtraData() const { return std::visit([](const auto& zip) { return zip.GetExtraData(); }, zip_); } +absl::StatusOr ZipFile::GetHeaderStart() const { + return std::visit([](const auto& zip) { return zip.GetHeaderStart(); }, zip_); +} +absl::StatusOr ZipFile::GetDataStart() const { + return std::visit([](const auto& zip) { return zip.GetDataStart(); }, zip_); +} absl::StatusOr ZipFile::GetFileData() { return std::visit([](auto& zip) { return zip.GetFileData(); }, zip_); } @@ -230,5 +334,16 @@ absl::StatusOr ZipFile::GetFileData(size_t max_size) { return std::visit([max_size](auto& zip) { return zip.GetFileData(max_size); }, zip_); } +absl::StatusOr ZipFile::ReadBytes(size_t max_bytes) { + return std::visit([max_bytes](auto& zip) { return zip.ReadBytes(max_bytes); }, + zip_); +} +absl::StatusOr ZipFile::ReadInto(absl::Span buf) { + return std::visit([buf](auto& zip) { return zip.ReadInto(buf); }, zip_); +} +absl::StatusOr ZipFile::Skip(uint64_t max_bytes) { + return std::visit([max_bytes](auto& zip) { return zip.Skip(max_bytes); }, + zip_); +} } // namespace security::zip diff --git a/zip/file.h b/zip/file.h index 4fa278c2..696e6085 100644 --- a/zip/file.h +++ b/zip/file.h @@ -11,11 +11,14 @@ #include "crubit/rust.h" #include "absl/status/status.h" #include "absl/status/statusor.h" +#include "absl/types/span.h" namespace security::zip { class BufferedZipWriter; class FsZipWriter; +class BufferedZipStreamWriter; +class FsZipStreamWriter; enum class CompressionMethod : int32_t { kStored, @@ -26,13 +29,86 @@ enum class CompressionMethod : int32_t { kXz, }; +// What an archive's central directory records about one entry. +// +// Obtaining this reads nothing: the central directory is parsed when the +// archive is opened. Contrast a ZipFile, which additionally reads the entry's +// local file header to find where its data begins, so that listing an archive +// entry by entry costs one read apiece. Prefer this whenever the entry's +// contents are not wanted; over a network the difference is one round trip +// against one per entry. +// +// This is a snapshot and does not borrow the archive, so it stays valid after +// the archive is destroyed and several may be held at once. +// +// Correspondingly there is no GetDataStart(): where an entry's data begins is +// the one thing the central directory does not record. Open the entry for it. +// Nor is there GetExtraData(): the zip crate exposes an entry's extra fields +// only once the entry is opened, so open it (ZipFile::GetExtraData()) for them. +// +// A moved-from ZipEntryMetadata describes no entry: its name and comment are +// empty, every numeric accessor returns 0, IsFile()/IsDir()/IsEncrypted() +// return false and GetCompressionMethod() returns an error. +class ZipEntryMetadata final { + public: + explicit ZipEntryMetadata(rust::ZipEntryMetadata metadata) + : metadata_(std::move(metadata)) {} + + // Returns whether the entry is a file, that is, neither a directory nor a + // symbolic link. + [[nodiscard]] bool IsFile() const; + // Returns whether the entry is a directory. + [[nodiscard]] bool IsDir() const; + // Returns whether the entry's data is encrypted. This library does not + // decrypt, so such an entry can be listed but not read. + [[nodiscard]] bool IsEncrypted() const; + // WARNING: Path Traversal ("Zip Slip") Hazard: The returned file name is + // extracted directly from the archive header without sanitization. If using + // this path for filesystem operations, callers MUST sanitize and validate + // that the resolved path does not traverse outside the target destination + // directory (e.g., rejecting paths with ".." components or leading directory + // separators). + [[nodiscard]] std::string GetFileName() const; + absl::StatusOr GetCompressionMethod() const; + [[nodiscard]] std::string GetComment() const; + // WARNING: Untrusted Header Value: Returns the uncompressed size declared in + // the ZIP header without verifying it against the actual decompressed + // payload. Crafty or corrupted archives (including zip bombs) may spoof this + // value; do not trust it for unbounded memory pre-allocation. + [[nodiscard]] uint64_t GetUncompressedSize() const; + [[nodiscard]] uint64_t GetCompressedSize() const; + // WARNING: Untrusted Header Value: Returns the raw CRC-32 checksum declared + // in the ZIP header. The checksum is only verified when the entry payload is + // actually read and decompressed. + [[nodiscard]] uint32_t GetCrc32() const; + // Returns the last-modified date encoded as an MS-DOS FAT date bitfield + // (bits 15-9: year offset from 1980, bits 8-5: month [1-12], bits 4-0: day + // [1-31]), or 0 if the entry records none. + [[nodiscard]] uint16_t GetLastModifiedDate() const; + // Returns the last-modified time encoded as an MS-DOS FAT time bitfield + // (bits 15-11: hour [0-23], bits 10-5: minute [0-59], bits 4-0: second / 2 + // [0-29]), or 0 if the entry records none. + [[nodiscard]] uint16_t GetLastModifiedTime() const; + // Returns the full POSIX `st_mode` bitmask (including `S_IFMT` file-type bits + // such as `0100644` for regular files), or 0 if Unix mode metadata is absent + // (e.g. non-Unix archives). + [[nodiscard]] uint32_t GetUnixMode() const; + // Returns the offset of this entry's local file header within the archive. + // Unlike ZipFile::GetHeaderStart() this is always meaningful, since a + // ZipEntryMetadata only ever comes from an archive's central directory. + [[nodiscard]] uint64_t GetHeaderStart() const; + + private: + rust::ZipEntryMetadata metadata_; +}; + class BufferedZipFile final { public: explicit BufferedZipFile(rust::BufferedZipFile zip) : zip_(std::move(zip)) {} absl::StatusOr IsFile() const; absl::StatusOr IsDir() const; - bool IsNone() const; + [[nodiscard]] bool IsNone() const; // WARNING: Path Traversal ("Zip Slip") Hazard: The returned file name is // extracted directly from the archive header without sanitization. If using // this path for filesystem operations, callers MUST sanitize and validate @@ -68,13 +144,63 @@ class BufferedZipFile final { // omits ExtendedTimestamp (0x5455) and Ntfs (0x000a) extra fields when // re-serializing `extra_data()`). absl::StatusOr GetExtraData() const; + // Returns the offset of this entry's local file header within the archive. + // Only meaningful for entries opened from a *ZipArchive: entries produced by + // a *ZipStreamReader do not know their offset and always report 0, which is + // indistinguishable from the genuine offset of an archive's first entry. + absl::StatusOr GetHeaderStart() const; + // Returns the offset of this entry's data within the archive, or 0 if it is + // unknown. Together with GetCompressedSize() this delimits the entry's raw + // bytes, so a caller that only needs a kStored entry can read that byte range + // directly rather than through this object. The offset is only known once the + // local file header has been parsed, which happens when an archive opens the + // entry; entries produced by a *ZipStreamReader always report 0. + absl::StatusOr GetDataStart() const; // WARNING: Decompression Bomb Hazard: This overload decompresses the entire // entry into an unbounded heap buffer. For untrusted zip archives, prefer // `GetFileData(size_t max_size)` below to cap memory allocation and defend - // against zip bomb memory exhaustion (OOM) attacks. + // against zip bomb memory exhaustion (OOM) attacks, or read the entry + // incrementally with `ReadBytes()`. absl::StatusOr GetFileData(); + // Reads up to `max_bytes` decompressed bytes from the entry, advancing the + // read position. This is the chunked, memory-bounded alternative to + // `GetFileData()`: at most `max_bytes` are buffered per call. The result may + // be shorter than `max_bytes`; a short but non-empty result does NOT mean + // the entry is exhausted. Only an empty result signals end of entry. + // Returns `absl::StatusCode::kFailedPrecondition` on a moved-from entry and + // `absl::StatusCode::kInternal` on decompression/CRC/I/O errors. + absl::StatusOr ReadBytes(size_t max_bytes); + // Reads into `buf` (decompressing unless the entry was opened raw) and + // returns how many bytes were written, which is less than `buf.size()` only + // at end of entry. + // + // Unlike ReadBytes(), nothing is allocated and the bytes are not copied + // again on the way out, so this is what a caller reading an entry of any + // size should use. + // + // zip-rs verifies the CRC only on the read that reaches the end of the + // entry, so reading exactly the uncompressed size never checks it; read + // until a short count (or 0) to have the entry validated. + // + // Returns `absl::StatusCode::kFailedPrecondition` on a moved-from entry and + // `absl::StatusCode::kInternal` on decompression/CRC/I/O errors. + absl::StatusOr ReadInto(absl::Span buf); + // Discards up to `max_bytes` and returns how many were discarded, which is + // less than `max_bytes` only at end of entry. + // + // A zip entry is a forward-only stream, so this is how a caller seeks + // within one. Nothing is allocated, whatever the distance. + // + // As with ReadInto(), the CRC is checked only by the read or skip that + // reaches the end of the entry, so skipping exactly the uncompressed size + // never checks it. + // + // Returns `absl::StatusCode::kFailedPrecondition` on a moved-from entry and + // `absl::StatusCode::kInternal` on decompression/CRC/I/O errors. + absl::StatusOr Skip(uint64_t max_bytes); + // Decompresses at most `max_size` bytes from the entry, returning // `absl::StatusCode::kOutOfRange` if the decompressed size exceeds // `max_size`. @@ -85,6 +211,8 @@ class BufferedZipFile final { rust::BufferedZipFile zip_; friend class BufferedZipWriter; friend class FsZipWriter; + friend class BufferedZipStreamWriter; + friend class FsZipStreamWriter; }; class FsZipFile final { @@ -92,7 +220,7 @@ class FsZipFile final { explicit FsZipFile(rust::FsZipFile zip) : zip_(std::move(zip)) {} absl::StatusOr IsFile() const; absl::StatusOr IsDir() const; - bool IsNone() const; + [[nodiscard]] bool IsNone() const; // WARNING: Path Traversal ("Zip Slip") Hazard: The returned file name is // extracted directly from the archive header without sanitization. If using @@ -129,13 +257,63 @@ class FsZipFile final { // omits ExtendedTimestamp (0x5455) and Ntfs (0x000a) extra fields when // re-serializing `extra_data()`). absl::StatusOr GetExtraData() const; + // Returns the offset of this entry's local file header within the archive. + // Only meaningful for entries opened from a *ZipArchive: entries produced by + // a *ZipStreamReader do not know their offset and always report 0, which is + // indistinguishable from the genuine offset of an archive's first entry. + absl::StatusOr GetHeaderStart() const; + // Returns the offset of this entry's data within the archive, or 0 if it is + // unknown. Together with GetCompressedSize() this delimits the entry's raw + // bytes, so a caller that only needs a kStored entry can read that byte range + // directly rather than through this object. The offset is only known once the + // local file header has been parsed, which happens when an archive opens the + // entry; entries produced by a *ZipStreamReader always report 0. + absl::StatusOr GetDataStart() const; // WARNING: Decompression Bomb Hazard: This overload decompresses the entire // entry into an unbounded heap buffer. For untrusted zip archives, prefer // `GetFileData(size_t max_size)` below to cap memory allocation and defend - // against zip bomb memory exhaustion (OOM) attacks. + // against zip bomb memory exhaustion (OOM) attacks, or read the entry + // incrementally with `ReadBytes()`. absl::StatusOr GetFileData(); + // Reads up to `max_bytes` decompressed bytes from the entry, advancing the + // read position. This is the chunked, memory-bounded alternative to + // `GetFileData()`: at most `max_bytes` are buffered per call. The result may + // be shorter than `max_bytes`; a short but non-empty result does NOT mean + // the entry is exhausted. Only an empty result signals end of entry. + // Returns `absl::StatusCode::kFailedPrecondition` on a moved-from entry and + // `absl::StatusCode::kInternal` on decompression/CRC/I/O errors. + absl::StatusOr ReadBytes(size_t max_bytes); + // Reads into `buf` (decompressing unless the entry was opened raw) and + // returns how many bytes were written, which is less than `buf.size()` only + // at end of entry. + // + // Unlike ReadBytes(), nothing is allocated and the bytes are not copied + // again on the way out, so this is what a caller reading an entry of any + // size should use. + // + // zip-rs verifies the CRC only on the read that reaches the end of the + // entry, so reading exactly the uncompressed size never checks it; read + // until a short count (or 0) to have the entry validated. + // + // Returns `absl::StatusCode::kFailedPrecondition` on a moved-from entry and + // `absl::StatusCode::kInternal` on decompression/CRC/I/O errors. + absl::StatusOr ReadInto(absl::Span buf); + // Discards up to `max_bytes` and returns how many were discarded, which is + // less than `max_bytes` only at end of entry. + // + // A zip entry is a forward-only stream, so this is how a caller seeks + // within one. Nothing is allocated, whatever the distance. + // + // As with ReadInto(), the CRC is checked only by the read or skip that + // reaches the end of the entry, so skipping exactly the uncompressed size + // never checks it. + // + // Returns `absl::StatusCode::kFailedPrecondition` on a moved-from entry and + // `absl::StatusCode::kInternal` on decompression/CRC/I/O errors. + absl::StatusOr Skip(uint64_t max_bytes); + // Decompresses at most `max_size` bytes from the entry, returning // `absl::StatusCode::kOutOfRange` if the decompressed size exceeds // `max_size`. @@ -146,6 +324,8 @@ class FsZipFile final { rust::FsZipFile zip_; friend class BufferedZipWriter; friend class FsZipWriter; + friend class BufferedZipStreamWriter; + friend class FsZipStreamWriter; }; class ZipFile final { @@ -160,7 +340,7 @@ class ZipFile final { absl::StatusOr IsFile() const; absl::StatusOr IsDir() const; - bool IsNone() const; + [[nodiscard]] bool IsNone() const; // WARNING: Path Traversal ("Zip Slip") Hazard: The returned file name is // extracted directly from the archive header without sanitization. If using @@ -197,13 +377,63 @@ class ZipFile final { // omits ExtendedTimestamp (0x5455) and Ntfs (0x000a) extra fields when // re-serializing `extra_data()`). absl::StatusOr GetExtraData() const; + // Returns the offset of this entry's local file header within the archive. + // Only meaningful for entries opened from a *ZipArchive: entries produced by + // a *ZipStreamReader do not know their offset and always report 0, which is + // indistinguishable from the genuine offset of an archive's first entry. + absl::StatusOr GetHeaderStart() const; + // Returns the offset of this entry's data within the archive, or 0 if it is + // unknown. Together with GetCompressedSize() this delimits the entry's raw + // bytes, so a caller that only needs a kStored entry can read that byte range + // directly rather than through this object. The offset is only known once the + // local file header has been parsed, which happens when an archive opens the + // entry; entries produced by a *ZipStreamReader always report 0. + absl::StatusOr GetDataStart() const; // WARNING: Decompression Bomb Hazard: This overload decompresses the entire // entry into an unbounded heap buffer. For untrusted zip archives, prefer // `GetFileData(size_t max_size)` below to cap memory allocation and defend - // against zip bomb memory exhaustion (OOM) attacks. + // against zip bomb memory exhaustion (OOM) attacks, or read the entry + // incrementally with `ReadBytes()`. absl::StatusOr GetFileData(); + // Reads up to `max_bytes` decompressed bytes from the entry, advancing the + // read position. This is the chunked, memory-bounded alternative to + // `GetFileData()`: at most `max_bytes` are buffered per call. The result may + // be shorter than `max_bytes`; a short but non-empty result does NOT mean + // the entry is exhausted. Only an empty result signals end of entry. + // Returns `absl::StatusCode::kFailedPrecondition` on a moved-from entry and + // `absl::StatusCode::kInternal` on decompression/CRC/I/O errors. + absl::StatusOr ReadBytes(size_t max_bytes); + // Reads into `buf` (decompressing unless the entry was opened raw) and + // returns how many bytes were written, which is less than `buf.size()` only + // at end of entry. + // + // Unlike ReadBytes(), nothing is allocated and the bytes are not copied + // again on the way out, so this is what a caller reading an entry of any + // size should use. + // + // zip-rs verifies the CRC only on the read that reaches the end of the + // entry, so reading exactly the uncompressed size never checks it; read + // until a short count (or 0) to have the entry validated. + // + // Returns `absl::StatusCode::kFailedPrecondition` on a moved-from entry and + // `absl::StatusCode::kInternal` on decompression/CRC/I/O errors. + absl::StatusOr ReadInto(absl::Span buf); + // Discards up to `max_bytes` and returns how many were discarded, which is + // less than `max_bytes` only at end of entry. + // + // A zip entry is a forward-only stream, so this is how a caller seeks + // within one. Nothing is allocated, whatever the distance. + // + // As with ReadInto(), the CRC is checked only by the read or skip that + // reaches the end of the entry, so skipping exactly the uncompressed size + // never checks it. + // + // Returns `absl::StatusCode::kFailedPrecondition` on a moved-from entry and + // `absl::StatusCode::kInternal` on decompression/CRC/I/O errors. + absl::StatusOr Skip(uint64_t max_bytes); + // Decompresses at most `max_size` bytes from the entry, returning // `absl::StatusCode::kOutOfRange` if the decompressed size exceeds // `max_size`. @@ -215,6 +445,8 @@ class ZipFile final { explicit ZipFile(BackendType b) : zip_(std::move(b)) {} friend class BufferedZipWriter; friend class FsZipWriter; + friend class BufferedZipStreamWriter; + friend class FsZipStreamWriter; }; } // namespace security::zip diff --git a/zip/read.cc b/zip/read.cc index f130f3c8..e4608487 100644 --- a/zip/read.cc +++ b/zip/read.cc @@ -1,6 +1,7 @@ #include "read.h" #include +#include #include #include #include @@ -9,6 +10,7 @@ #include "converters.h" #include "file.h" #include "crubit/rust.h" +#include "absl/functional/overload.h" #include "absl/status/status.h" #include "absl/status/status_macros.h" #include "absl/status/statusor.h" @@ -28,23 +30,27 @@ absl::StatusOr BufferedZipArchive::NewFromData( return BufferedZipArchive(std::move(archive)); } -absl::StatusOr BufferedZipArchive::GetLength() const { +absl::Status BufferedZipArchive::CheckNone() const { if (archive_.is_none()) { - return absl::InternalError("ZipArchive object used after being moved-out"); + return absl::FailedPreconditionError("Zip archive is not open"); } + return absl::OkStatus(); +} + +absl::StatusOr BufferedZipArchive::GetLength() const { + ABSL_RETURN_IF_ERROR(CheckNone()); return archive_.get_length(); } absl::StatusOr BufferedZipArchive::GetComment() const { - if (archive_.is_none()) { - return absl::InternalError("ZipArchive object used after being moved-out"); - } + ABSL_RETURN_IF_ERROR(CheckNone()); rust::VecU8 comment_vec = archive_.get_comment(); return std::string( security::crubit_helpers::StringViewFromVecU8(comment_vec)); } absl::StatusOr BufferedZipArchive::GetFileByIndex(uintptr_t index) { + ABSL_RETURN_IF_ERROR(CheckNone()); ABSL_ASSIGN_OR_RETURN( rust::BufferedZipFile file, FromRustBufferedZipFile(archive_.get_file_by_index(index))); @@ -52,12 +58,24 @@ absl::StatusOr BufferedZipArchive::GetFileByIndex(uintptr_t index) { } absl::StatusOr BufferedZipArchive::GetFileByIndexRaw(uintptr_t index) { + ABSL_RETURN_IF_ERROR(CheckNone()); ABSL_ASSIGN_OR_RETURN( rust::BufferedZipFile file, FromRustBufferedZipFile(archive_.get_file_by_index_raw(index))); return ZipFile::FromBuffer(std::move(file)); } +absl::StatusOr BufferedZipArchive::GetEntryMetadata( + uintptr_t index) const { + ABSL_RETURN_IF_ERROR(CheckNone()); + ABSL_ASSIGN_OR_RETURN( + rust::ZipEntryMetadata metadata, + FromRustZipEntryMetadata(archive_.get_entry_metadata(index))); + return ZipEntryMetadata(std::move(metadata)); +} + +bool BufferedZipArchive::IsNone() const { return archive_.is_none(); } + absl::StatusOr FsZipArchive::NewFromPath(absl::string_view path) { ABSL_ASSIGN_OR_RETURN( rust::FsZipArchive archive, @@ -67,35 +85,60 @@ absl::StatusOr FsZipArchive::NewFromPath(absl::string_view path) { return FsZipArchive(std::move(archive)); } -absl::StatusOr FsZipArchive::GetLength() const { +absl::Status FsZipArchive::CheckNone() const { if (archive_.is_none()) { - return absl::InternalError("ZipArchive object used after being moved-out"); + return absl::FailedPreconditionError("Zip archive is not open"); } - return archive_.get_length(); + return absl::OkStatus(); } -absl::StatusOr FsZipArchive::GetComment() const { +absl::StatusOr FsZipArchive::TryClone() const { if (archive_.is_none()) { - return absl::InternalError("ZipArchive object used after being moved-out"); + return absl::FailedPreconditionError("Zip archive is not open"); } + ABSL_ASSIGN_OR_RETURN(rust::FsZipArchive archive, + FromRustFsZipArchive(archive_.try_clone())); + return FsZipArchive(std::move(archive)); +} + +absl::StatusOr FsZipArchive::GetLength() const { + ABSL_RETURN_IF_ERROR(CheckNone()); + return archive_.get_length(); +} + +absl::StatusOr FsZipArchive::GetComment() const { + ABSL_RETURN_IF_ERROR(CheckNone()); rust::VecU8 comment_vec = archive_.get_comment(); return std::string( security::crubit_helpers::StringViewFromVecU8(comment_vec)); } absl::StatusOr FsZipArchive::GetFileByIndex(uintptr_t index) { + ABSL_RETURN_IF_ERROR(CheckNone()); ABSL_ASSIGN_OR_RETURN(rust::FsZipFile file, FromRustFsZipFile(archive_.get_file_by_index(index))); return ZipFile::FromFile(std::move(file)); } absl::StatusOr FsZipArchive::GetFileByIndexRaw(uintptr_t index) { + ABSL_RETURN_IF_ERROR(CheckNone()); ABSL_ASSIGN_OR_RETURN( rust::FsZipFile file, FromRustFsZipFile(archive_.get_file_by_index_raw(index))); return ZipFile::FromFile(std::move(file)); } +absl::StatusOr FsZipArchive::GetEntryMetadata( + uintptr_t index) const { + ABSL_RETURN_IF_ERROR(CheckNone()); + ABSL_ASSIGN_OR_RETURN( + rust::ZipEntryMetadata metadata, + FromRustZipEntryMetadata(archive_.get_entry_metadata(index))); + return ZipEntryMetadata(std::move(metadata)); +} + +bool FsZipArchive::IsNone() const { return archive_.is_none(); } + absl::StatusOr ZipArchive::FromFile(absl::string_view path) { ABSL_ASSIGN_OR_RETURN(FsZipArchive archive, FsZipArchive::NewFromPath(path)); return ZipArchive(std::move(archive)); @@ -107,6 +150,20 @@ absl::StatusOr ZipArchive::FromBuffer(absl::string_view data) { return ZipArchive(std::move(archive)); } +absl::StatusOr ZipArchive::TryClone() const { + return std::visit( + absl::Overload( + [](const BufferedZipArchive&) -> absl::StatusOr { + return absl::UnimplementedError( + "TryClone is not supported for archives opened from a buffer"); + }, + [](const auto& archive) -> absl::StatusOr { + ABSL_ASSIGN_OR_RETURN(auto clone, archive.TryClone()); + return ZipArchive(std::move(clone)); + }), + archive_); +} + absl::StatusOr ZipArchive::GetLength() const { return std::visit([](const auto& arg) { return arg.GetLength(); }, archive_); } @@ -125,4 +182,173 @@ absl::StatusOr ZipArchive::GetFileByIndexRaw(uintptr_t index) { archive_); } +absl::StatusOr ZipArchive::GetEntryMetadata( + uintptr_t index) const { + return std::visit( + [index](const auto& arg) { return arg.GetEntryMetadata(index); }, + archive_); +} + +bool ZipArchive::IsNone() const { + return std::visit([](const auto& arg) { return arg.IsNone(); }, archive_); +} + +absl::StatusOr BufferedZipStreamReader::NewFromData( + absl::string_view data) { + rust::VecU8 input_data = + rust::VecU8::copy_from_slice(absl::Span( + reinterpret_cast(data.data()), data.size())); + ABSL_ASSIGN_OR_RETURN( + rust::BufferedZipStreamReader reader, + FromRustBufferedZipStreamReader( + rust::BufferedZipStreamReader::new_from_data(input_data))); + return BufferedZipStreamReader(std::move(reader)); +} + +absl::Status BufferedZipStreamReader::CheckNone() const { + if (reader_.is_none()) { + return absl::FailedPreconditionError("Zip stream reader is not open"); + } + return absl::OkStatus(); +} + +absl::StatusOr> BufferedZipStreamReader::ReadNextFile() { + ABSL_RETURN_IF_ERROR(CheckNone()); + ABSL_ASSIGN_OR_RETURN(rust::BufferedZipFile file, + FromRustBufferedZipFile(reader_.read_next_file())); + if (file.is_none()) { + return std::nullopt; + } + return ZipFile::FromBuffer(std::move(file)); +} + +absl::StatusOr> +BufferedZipStreamReader::ReadNextFileWithCompressedSize( + uint64_t compressed_size) { + ABSL_RETURN_IF_ERROR(CheckNone()); + ABSL_ASSIGN_OR_RETURN( + rust::BufferedZipFile file, + FromRustBufferedZipFile( + reader_.read_next_file_with_compressed_size(compressed_size))); + if (file.is_none()) { + return std::nullopt; + } + return ZipFile::FromBuffer(std::move(file)); +} + +bool BufferedZipStreamReader::IsFinished() const { + return reader_.is_finished(); +} + +bool BufferedZipStreamReader::IsNone() const { return reader_.is_none(); } + +absl::StatusOr FsZipStreamReader::NewFromPath( + absl::string_view path) { + ABSL_ASSIGN_OR_RETURN( + rust::FsZipStreamReader reader, + FromRustFsZipStreamReader(rust::FsZipStreamReader::new_from_path( + absl::Span( + reinterpret_cast(path.data()), path.size())))); + return FsZipStreamReader(std::move(reader)); +} + +absl::Status FsZipStreamReader::CheckNone() const { + if (reader_.is_none()) { + return absl::FailedPreconditionError("Zip stream reader is not open"); + } + return absl::OkStatus(); +} + +absl::StatusOr> FsZipStreamReader::ReadNextFile() { + ABSL_RETURN_IF_ERROR(CheckNone()); + ABSL_ASSIGN_OR_RETURN(rust::FsZipFile file, + FromRustFsZipFile(reader_.read_next_file())); + if (file.is_none()) { + return std::nullopt; + } + return ZipFile::FromFile(std::move(file)); +} + +absl::StatusOr> +FsZipStreamReader::ReadNextFileWithCompressedSize(uint64_t compressed_size) { + ABSL_RETURN_IF_ERROR(CheckNone()); + ABSL_ASSIGN_OR_RETURN( + rust::FsZipFile file, + FromRustFsZipFile( + reader_.read_next_file_with_compressed_size(compressed_size))); + if (file.is_none()) { + return std::nullopt; + } + return ZipFile::FromFile(std::move(file)); +} + +bool FsZipStreamReader::IsFinished() const { return reader_.is_finished(); } + +bool FsZipStreamReader::IsNone() const { return reader_.is_none(); } + +absl::StatusOr ZipStreamReader::FromFile( + absl::string_view path) { + ABSL_ASSIGN_OR_RETURN(FsZipStreamReader reader, + FsZipStreamReader::NewFromPath(path)); + return ZipStreamReader(std::move(reader)); +} + +absl::StatusOr ZipStreamReader::FromBuffer( + absl::string_view data) { + ABSL_ASSIGN_OR_RETURN(BufferedZipStreamReader reader, + BufferedZipStreamReader::NewFromData(data)); + return ZipStreamReader(std::move(reader)); +} + +absl::StatusOr> ZipStreamReader::ReadNextFile() { + return std::visit([](auto& arg) { return arg.ReadNextFile(); }, reader_); +} + +absl::StatusOr> +ZipStreamReader::ReadNextFileWithCompressedSize(uint64_t compressed_size) { + return std::visit( + [compressed_size](auto& arg) { + return arg.ReadNextFileWithCompressedSize(compressed_size); + }, + reader_); +} + +bool ZipStreamReader::IsFinished() const { + return std::visit([](const auto& arg) { return arg.IsFinished(); }, reader_); +} + +bool ZipStreamReader::IsNone() const { + return std::visit([](const auto& arg) { return arg.IsNone(); }, reader_); +} + +absl::StatusOr> ReadZipFileFromStream( + BufferedZipStreamReader& reader) { + return reader.ReadNextFile(); +} + +absl::StatusOr> ReadZipFileFromStream( + FsZipStreamReader& reader) { + return reader.ReadNextFile(); +} + +absl::StatusOr> ReadZipFileFromStream( + ZipStreamReader& reader) { + return reader.ReadNextFile(); +} + +absl::StatusOr> ReadZipFileFromStreamWithCompressedSize( + BufferedZipStreamReader& reader, uint64_t compressed_size) { + return reader.ReadNextFileWithCompressedSize(compressed_size); +} + +absl::StatusOr> ReadZipFileFromStreamWithCompressedSize( + FsZipStreamReader& reader, uint64_t compressed_size) { + return reader.ReadNextFileWithCompressedSize(compressed_size); +} + +absl::StatusOr> ReadZipFileFromStreamWithCompressedSize( + ZipStreamReader& reader, uint64_t compressed_size) { + return reader.ReadNextFileWithCompressedSize(compressed_size); +} + } // namespace security::zip diff --git a/zip/read.h b/zip/read.h index 40da10ed..35a69cb3 100644 --- a/zip/read.h +++ b/zip/read.h @@ -2,12 +2,15 @@ #define SECURITY_ZIP_READ_H_ #include +#include #include #include #include #include "file.h" #include "crubit/rust.h" +#include "absl/base/attributes.h" +#include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" @@ -20,14 +23,23 @@ class BufferedZipArchive final { absl::StatusOr GetComment() const; // Returns a ZipFile by index. The result file returns decompressed data when // read. - absl::StatusOr GetFileByIndex(uintptr_t index); + absl::StatusOr GetFileByIndex(uintptr_t index) + ABSL_ATTRIBUTE_LIFETIME_BOUND; // Returns a ZipFile by index. The result file returns compressed data as-is // without decompressing when read. // Warning: Writers in zip-rs do not support writing compressed data as-is. // This data will be compressed again when written through a `*ZipWriter`. - absl::StatusOr GetFileByIndexRaw(uintptr_t index); + absl::StatusOr GetFileByIndexRaw(uintptr_t index) + ABSL_ATTRIBUTE_LIFETIME_BOUND; + // Returns what the central directory records about the entry at `index`, + // reading nothing. Prefer this to GetFileByIndex when the entry's contents + // are not wanted: that one also reads the entry's local file header, so + // listing an archive with it costs one read per entry. + absl::StatusOr GetEntryMetadata(uintptr_t index) const; + [[nodiscard]] bool IsNone() const; private: + absl::Status CheckNone() const; rust::BufferedZipArchive archive_; explicit BufferedZipArchive(rust::BufferedZipArchive archive) : archive_(std::move(archive)) {} @@ -36,18 +48,47 @@ class BufferedZipArchive final { class FsZipArchive final { public: static absl::StatusOr NewFromPath(absl::string_view path); + // Returns an independent handle onto the same archive. + // + // The new handle shares the central directory that NewFromPath already + // parsed, so this costs one open(2) and is O(1) in the number of entries, + // whereas NewFromPath is O(entries). Use it when several entries must be read + // at once: GetFileByIndex keeps the archive borrowed for as long as the + // returned ZipFile lives, so N concurrently open entries need N handles, and + // reopening by path for each one is quadratic on large archives. + // + // Each handle owns its own file descriptor and therefore its own file + // position, so handles may be read independently. + // + // Returns `absl::StatusCode::kFailedPrecondition` on a moved-from archive + // and when the path can no longer be opened. The caller must ensure the path + // still resolves to the same, unmodified file it was opened from: not + // renamed, replaced or rewritten in between (for a relative path, the + // working directory must not have changed either). The shared central + // directory would otherwise describe byte ranges that no longer hold the + // entries it names. + absl::StatusOr TryClone() const; absl::StatusOr GetLength() const; absl::StatusOr GetComment() const; // Returns a ZipFile by index. The result file returns decompressed data when // read. - absl::StatusOr GetFileByIndex(uintptr_t index); + absl::StatusOr GetFileByIndex(uintptr_t index) + ABSL_ATTRIBUTE_LIFETIME_BOUND; // Returns a ZipFile by index. The result file returns compressed data as-is // without decompressing when read. // Warning: Writers in zip-rs do not support writing compressed data as-is. // This data will be compressed again when written through a `*ZipWriter`. - absl::StatusOr GetFileByIndexRaw(uintptr_t index); + absl::StatusOr GetFileByIndexRaw(uintptr_t index) + ABSL_ATTRIBUTE_LIFETIME_BOUND; + // Returns what the central directory records about the entry at `index`, + // reading nothing. Prefer this to GetFileByIndex when the entry's contents + // are not wanted: that one also reads the entry's local file header, so + // listing an archive with it costs one read per entry. + absl::StatusOr GetEntryMetadata(uintptr_t index) const; + [[nodiscard]] bool IsNone() const; private: + absl::Status CheckNone() const; rust::FsZipArchive archive_; explicit FsZipArchive(rust::FsZipArchive archive) : archive_(std::move(archive)) {} @@ -57,16 +98,36 @@ class ZipArchive final { public: static absl::StatusOr FromFile(absl::string_view path); static absl::StatusOr FromBuffer(absl::string_view data); + // Returns an independent handle onto the same archive, sharing the central + // directory this one already parsed. See FsZipArchive::TryClone for the cost + // model and the caller's obligations. + // + // Returns `absl::StatusCode::kFailedPrecondition` on a moved-from file-backed + // archive and when the path can no longer be opened. + // + // Not supported for archives opened with FromBuffer, which return + // `absl::StatusCode::kUnimplemented` (even when moved from), because their + // reader cannot currently be shared and cloning one would copy the whole + // archive. + absl::StatusOr TryClone() const; absl::StatusOr GetLength() const; absl::StatusOr GetComment() const; // Returns a ZipFile by index. The result file returns decompressed data when // read. - absl::StatusOr GetFileByIndex(uintptr_t index); + absl::StatusOr GetFileByIndex(uintptr_t index) + ABSL_ATTRIBUTE_LIFETIME_BOUND; // Returns a ZipFile by index. The result file returns compressed data as-is // without decompressing when read. // Warning: Writers in zip-rs do not support writing compressed data as-is. // This data will be compressed again when written through a `*ZipWriter`. - absl::StatusOr GetFileByIndexRaw(uintptr_t index); + absl::StatusOr GetFileByIndexRaw(uintptr_t index) + ABSL_ATTRIBUTE_LIFETIME_BOUND; + // Returns what the central directory records about the entry at `index`, + // reading nothing. Prefer this to GetFileByIndex when the entry's contents + // are not wanted: that one also reads the entry's local file header, so + // listing an archive with it costs one read per entry. + absl::StatusOr GetEntryMetadata(uintptr_t index) const; + [[nodiscard]] bool IsNone() const; private: using BackendType = std::variant; @@ -75,6 +136,117 @@ class ZipArchive final { explicit ZipArchive(BackendType archive) : archive_(std::move(archive)) {} }; +// Streaming readers that read sequential zip entries from a stream without +// seeking. The contract below applies to all three classes. +// +// Construction: `NewFromData`/`FromBuffer` copy `data` and never fail; +// nothing is validated until the first `ReadNextFile*` call (unlike +// `*ZipArchive`, which parses the central directory up front). +// `NewFromPath`/`FromFile` return kInvalidArgument if `path` cannot be opened. +// +// ReadNextFile(): Reads the next local file header and returns the entry, or +// std::nullopt once the central directory (end of entries) is reached. +// Error codes: +// - kFailedPrecondition: the reader is moved-from, or the entry was written +// with general purpose bit 3 (sizes only in the trailing data descriptor), +// so its compressed size is not available in the local header. Use +// ReadNextFileWithCompressedSize() for such entries. +// - kInvalidArgument: corrupt local file header. +// - kInternal: I/O error. +// +// ReadNextFileWithCompressedSize(compressed_size): Same as ReadNextFile() but +// uses `compressed_size` for entries whose local header carries no size (bit-3 +// / data-descriptor entries), e.g. taken from the central directory of a +// previous seekable pass. A wrong hint truncates or over-reads the entry and +// surfaces as a CRC or decoding error (kInternal) when the data is read, after +// which the stream position is undefined. +// +// IMPORTANT: The returned ZipFile borrows the underlying stream (a Rust `&mut` +// that Crubit cannot enforce in C++) and drains any unread compressed bytes +// when destroyed so the stream is positioned at the next local file header. +// Only one entry may be alive per reader at a time: callers MUST destroy the +// returned ZipFile before calling ReadNextFile* again. +// +// IsFinished(): True once ReadNextFile* has returned std::nullopt; subsequent +// calls keep returning std::nullopt. False for a moved-from reader and after a +// read error. +// IsNone(): True if moved-from; ReadNextFile* then return kFailedPrecondition. +class BufferedZipStreamReader final { + public: + static absl::StatusOr NewFromData( + absl::string_view data); + // See the contract at the top of this section. + absl::StatusOr> ReadNextFile() + ABSL_ATTRIBUTE_LIFETIME_BOUND; + absl::StatusOr> ReadNextFileWithCompressedSize( + uint64_t compressed_size) ABSL_ATTRIBUTE_LIFETIME_BOUND; + [[nodiscard]] bool IsFinished() const; + [[nodiscard]] bool IsNone() const; + + private: + absl::Status CheckNone() const; + rust::BufferedZipStreamReader reader_; + explicit BufferedZipStreamReader(rust::BufferedZipStreamReader reader) + : reader_(std::move(reader)) {} +}; + +class FsZipStreamReader final { + public: + static absl::StatusOr NewFromPath(absl::string_view path); + // See the contract at the top of this section. + absl::StatusOr> ReadNextFile() + ABSL_ATTRIBUTE_LIFETIME_BOUND; + absl::StatusOr> ReadNextFileWithCompressedSize( + uint64_t compressed_size) ABSL_ATTRIBUTE_LIFETIME_BOUND; + [[nodiscard]] bool IsFinished() const; + [[nodiscard]] bool IsNone() const; + + private: + absl::Status CheckNone() const; + rust::FsZipStreamReader reader_; + explicit FsZipStreamReader(rust::FsZipStreamReader reader) + : reader_(std::move(reader)) {} +}; + +// Facade over BufferedZipStreamReader / FsZipStreamReader; every method +// forwards to the selected backend. +class ZipStreamReader final { + public: + static absl::StatusOr FromFile(absl::string_view path); + static absl::StatusOr FromBuffer(absl::string_view data); + // See the contract at the top of this section. + absl::StatusOr> ReadNextFile() + ABSL_ATTRIBUTE_LIFETIME_BOUND; + absl::StatusOr> ReadNextFileWithCompressedSize( + uint64_t compressed_size) ABSL_ATTRIBUTE_LIFETIME_BOUND; + [[nodiscard]] bool IsFinished() const; + [[nodiscard]] bool IsNone() const; + + private: + using BackendType = std::variant; + BackendType reader_; + + explicit ZipStreamReader(BackendType reader) : reader_(std::move(reader)) {} +}; + +// Free function helpers mirroring zip::read::read_zipfile_from_stream. +absl::StatusOr> ReadZipFileFromStream( + BufferedZipStreamReader& reader ABSL_ATTRIBUTE_LIFETIME_BOUND); +absl::StatusOr> ReadZipFileFromStream( + FsZipStreamReader& reader ABSL_ATTRIBUTE_LIFETIME_BOUND); +absl::StatusOr> ReadZipFileFromStream( + ZipStreamReader& reader ABSL_ATTRIBUTE_LIFETIME_BOUND); + +absl::StatusOr> ReadZipFileFromStreamWithCompressedSize( + BufferedZipStreamReader& reader ABSL_ATTRIBUTE_LIFETIME_BOUND, + uint64_t compressed_size); +absl::StatusOr> ReadZipFileFromStreamWithCompressedSize( + FsZipStreamReader& reader ABSL_ATTRIBUTE_LIFETIME_BOUND, + uint64_t compressed_size); +absl::StatusOr> ReadZipFileFromStreamWithCompressedSize( + ZipStreamReader& reader ABSL_ATTRIBUTE_LIFETIME_BOUND, + uint64_t compressed_size); + } // namespace security::zip #endif // SECURITY_ZIP_READ_H_ diff --git a/zip/rust/entry_metadata.rs b/zip/rust/entry_metadata.rs new file mode 100644 index 00000000..247042c4 --- /dev/null +++ b/zip/rust/entry_metadata.rs @@ -0,0 +1,172 @@ +//! A zip entry's central directory record, readable without any I/O. +//! +//! Crubit does not support generic type parameters, so the archive types that +//! hand these out are written out one by one; this type itself owns its data +//! and so is shared by all of them. + +use crate::write::CompressionMethod; +use crate::VecU8; +use zip::read::ZipArchiveMetadata; + +/// What a zip archive's central directory records about one entry. +/// +/// Every field here is read when the archive is opened, so obtaining this +/// costs no I/O. Note the contrast with `*ZipFile`, which additionally seeks +/// to the entry's local file header in order to find where its data begins: +/// listing an archive through `get_file_by_index` therefore costs one read per +/// entry, which is ruinous when the archive is behind a network. Prefer this +/// whenever the entry's contents are not wanted. +/// +/// Correspondingly, this deliberately has no `get_data_start()`: that offset +/// is the one piece of per-entry information that is not in the central +/// directory. Open the entry if you need it. +/// +/// The default (and so the moved-from) value describes no entry: empty name +/// and comment, zero for every number, `false` for every flag and +/// `CompressionMethod::Unsupported`. +#[derive(Debug)] +pub struct ZipEntryMetadata { + name: Vec, + comment: Vec, + uncompressed_size: u64, + compressed_size: u64, + header_start: u64, + crc32: u32, + unix_mode: u32, + last_modified_date: u16, + last_modified_time: u16, + compression_method: CompressionMethod, + is_dir: bool, + is_file: bool, + is_encrypted: bool, +} + +impl Default for ZipEntryMetadata { + /// The default value exists only so that Crubit can generate a movable + /// type; it describes no entry. + fn default() -> Self { + Self { + name: Vec::new(), + comment: Vec::new(), + uncompressed_size: 0, + compressed_size: 0, + header_start: 0, + crc32: 0, + unix_mode: 0, + last_modified_date: 0, + last_modified_time: 0, + compression_method: CompressionMethod::Unsupported, + is_dir: false, + is_file: false, + is_encrypted: false, + } + } +} + +impl ZipEntryMetadata { + /// Copies out what `metadata` records about the entry at `index`, or + /// `None` if there is no such entry. + /// + /// This function won't be generated by Crubit and is only called from + /// rust. (b/259749095) + pub(crate) fn from_central_directory( + metadata: &ZipArchiveMetadata, + index: usize, + ) -> Option { + let entry = metadata.entry(index).ok()?; + let last_modified = entry.last_modified(); + Some(Self { + // Decoded the same way as `*ZipFile::get_file_name`, so that listing + // and opening an entry agree on its name (UTF-8, else CP437). + name: match entry.name() { + Ok(name) => name.as_bytes().to_vec(), + Err(_) => entry.name_raw().to_vec(), + }, + comment: entry.comment().as_bytes().to_vec(), + uncompressed_size: entry.size(), + compressed_size: entry.compressed_size(), + header_start: entry.header_start(), + crc32: entry.crc32(), + unix_mode: entry.unix_mode().unwrap_or(0), + last_modified_date: last_modified.map(|dt| dt.datepart()).unwrap_or(0), + last_modified_time: last_modified.map(|dt| dt.timepart()).unwrap_or(0), + compression_method: CompressionMethod::from_zip_crate(entry.compression()), + is_dir: entry.is_dir(), + is_file: entry.is_file(), + is_encrypted: entry.encrypted(), + }) + } + + /// Returns the name of the entry. + pub fn get_file_name(&self) -> VecU8 { + VecU8::copy_from_slice(&self.name) + } + + /// Returns whether the entry is a file, that is, neither a directory nor a + /// symbolic link. + pub fn is_file(&self) -> bool { + self.is_file + } + + /// Returns whether the entry is a directory. + pub fn is_dir(&self) -> bool { + self.is_dir + } + + /// Returns whether the entry's data is encrypted. This library does not + /// decrypt, so such an entry can be listed but not read. + pub fn is_encrypted(&self) -> bool { + self.is_encrypted + } + + /// Returns the compression method of the entry. + pub fn get_compression_method(&self) -> CompressionMethod { + self.compression_method + } + + /// Returns the comment on the entry. + pub fn get_comment(&self) -> VecU8 { + VecU8::copy_from_slice(&self.comment) + } + + /// Returns the uncompressed size of the entry. + pub fn get_uncompressed_size(&self) -> u64 { + self.uncompressed_size + } + + /// Returns the compressed size of the entry. + pub fn get_compressed_size(&self) -> u64 { + self.compressed_size + } + + /// Returns the CRC32 of the entry. + pub fn get_crc32(&self) -> u32 { + self.crc32 + } + + /// Returns the last modified date of the entry in DOS format, or 0 if it + /// does not record one. + pub fn get_last_modified_date(&self) -> u16 { + self.last_modified_date + } + + /// Returns the last modified time of the entry in DOS format, or 0 if it + /// does not record one. + pub fn get_last_modified_time(&self) -> u16 { + self.last_modified_time + } + + /// Returns the Unix mode of the entry, or 0 if it does not record one. + pub fn get_unix_mode(&self) -> u32 { + self.unix_mode + } + + /// Returns the offset of the entry's local file header in the archive. + /// + /// Unlike `*ZipFile::get_header_start()` this is always meaningful, since + /// a `ZipEntryMetadata` only ever comes from an archive's central + /// directory. + pub fn get_header_start(&self) -> u64 { + self.header_start + } +} diff --git a/zip/rust/error.rs b/zip/rust/error.rs index 79599674..9838fd48 100644 --- a/zip/rust/error.rs +++ b/zip/rust/error.rs @@ -37,3 +37,19 @@ impl ZipError { Self::new(message, ZipErrorCode::Internal) } } + +impl ZipError { + /// Converts a `zip` crate error. Deliberately not a `From` impl: a public + /// `From` would make Crubit generate C++ bindings for + /// the `zip` crate itself (and include them from zip_wrapper.h), which + /// clients neither need nor want. + pub(crate) fn from_zip_error(err: zip::result::ZipError) -> Self { + match err { + zip::result::ZipError::Io(e) => ZipError::internal(e.to_string()), + zip::result::ZipError::InvalidArchive(e) => ZipError::invalid_argument(e.as_ref()), + zip::result::ZipError::UnsupportedArchive(e) => ZipError::failed_precondition(e), + zip::result::ZipError::FileNotFound => ZipError::out_of_range("File not found"), + _ => ZipError::internal(err.to_string()), + } + } +} diff --git a/zip/rust/file.rs b/zip/rust/file.rs index 5d8cf466..f6a3c75f 100644 --- a/zip/rust/file.rs +++ b/zip/rust/file.rs @@ -7,7 +7,6 @@ use std::fs::File; use std::io::{Cursor, Error, ErrorKind, Read}; use std::vec::Vec; use zip::read::ZipFile as WrappedZipFile; -use zip::CompressionMethod as ZipCrateCompressionMethod; #[derive(Default)] pub struct BufferedZipFile<'a> { @@ -112,6 +111,32 @@ impl<'a> BufferedZipFile<'a> { get_extra_data_impl(&self.file) } + /// Returns the offset of this entry's local file header within the archive. + /// + /// Only meaningful for entries opened from a `*ZipArchive`: entries + /// obtained from a `*ZipStreamReader` do not know their offset and always + /// report 0, which is indistinguishable from the genuine offset of an + /// archive's first entry. + /// + /// The result is only valid if `is_none()` returns false. + pub fn get_header_start(&self) -> u64 { + get_header_start_impl(&self.file) + } + + /// Returns the offset of this entry's data within the archive, or 0 if it + /// is unknown. Together with `get_compressed_size()` this delimits the + /// entry's raw bytes, so a caller that only needs a `Stored` entry can read + /// that byte range directly instead of going through this archive. + /// + /// The offset is only known once the local file header has been parsed, + /// which archive-based readers do when the entry is opened. It is never + /// known for entries obtained from a `*ZipStreamReader`, which report 0. + /// + /// The result is only valid if `is_none()` returns false. + pub fn get_data_start(&self) -> u64 { + get_data_start_impl(&self.file) + } + /// Returns the data of the file. /// The result is only valid if `is_none()` returns false. pub fn get_file_data(&mut self) -> Result { @@ -124,6 +149,47 @@ impl<'a> BufferedZipFile<'a> { pub fn get_file_data_with_max_size(&mut self, max_size: usize) -> Result { get_file_data_with_max_size_impl(&mut self.file, max_size) } + + /// Reads up to `max_bytes` decompressed data from the file. + /// Returns an empty VecU8 on EOF. + /// Returns an error if `is_none()` returns true. + pub fn read_bytes(&mut self, max_bytes: usize) -> Result { + read_bytes_impl(&mut self.file, max_bytes) + } + + /// Reads into `buf` (decompressing unless the entry was opened raw) and + /// returns how many bytes were written, which is less than `buf.len()` + /// only at end of entry. + /// + /// Unlike `read_bytes`, nothing is allocated and the bytes are not copied + /// again on the way out, so this is what a caller reading an entry of any + /// size should use. + /// + /// zip-rs verifies the CRC only on the read that reaches the end of the + /// entry, so reading exactly the uncompressed size never checks it; read + /// until a short count (or 0) to have the entry validated. + /// + /// Returns an error if `is_none()` returns true, or + /// `ZipErrorCode::Internal` on decompression, CRC or I/O errors. + pub fn read_into(&mut self, buf: &mut [u8]) -> Result { + read_into_impl(&mut self.file, buf) + } + + /// Discards up to `max_bytes` and returns how many were discarded, which + /// is less than `max_bytes` only at end of entry. + /// + /// A zip entry is a forward-only stream, so this is how a caller seeks + /// within one. Nothing is allocated, whatever the distance. + /// + /// As with `read_into`, the CRC is checked only by the read or skip that + /// reaches the end of the entry, so skipping exactly the uncompressed size + /// never checks it. + /// + /// Returns an error if `is_none()` returns true, or + /// `ZipErrorCode::Internal` on decompression, CRC or I/O errors. + pub fn skip(&mut self, max_bytes: u64) -> Result { + skip_impl(&mut self.file, max_bytes) + } } impl<'a> Read for BufferedZipFile<'a> { @@ -238,6 +304,32 @@ impl<'a> FsZipFile<'a> { get_extra_data_impl(&self.file) } + /// Returns the offset of this entry's local file header within the archive. + /// + /// Only meaningful for entries opened from a `*ZipArchive`: entries + /// obtained from a `*ZipStreamReader` do not know their offset and always + /// report 0, which is indistinguishable from the genuine offset of an + /// archive's first entry. + /// + /// The result is only valid if `is_none()` returns false. + pub fn get_header_start(&self) -> u64 { + get_header_start_impl(&self.file) + } + + /// Returns the offset of this entry's data within the archive, or 0 if it + /// is unknown. Together with `get_compressed_size()` this delimits the + /// entry's raw bytes, so a caller that only needs a `Stored` entry can read + /// that byte range directly instead of going through this archive. + /// + /// The offset is only known once the local file header has been parsed, + /// which archive-based readers do when the entry is opened. It is never + /// known for entries obtained from a `*ZipStreamReader`, which report 0. + /// + /// The result is only valid if `is_none()` returns false. + pub fn get_data_start(&self) -> u64 { + get_data_start_impl(&self.file) + } + /// Returns the data of the file. /// The result is only valid if `is_none()` returns false. pub fn get_file_data(&mut self) -> Result { @@ -250,6 +342,47 @@ impl<'a> FsZipFile<'a> { pub fn get_file_data_with_max_size(&mut self, max_size: usize) -> Result { get_file_data_with_max_size_impl(&mut self.file, max_size) } + + /// Reads up to `max_bytes` decompressed data from the file. + /// Returns an empty VecU8 on EOF. + /// Returns an error if `is_none()` returns true. + pub fn read_bytes(&mut self, max_bytes: usize) -> Result { + read_bytes_impl(&mut self.file, max_bytes) + } + + /// Reads into `buf` (decompressing unless the entry was opened raw) and + /// returns how many bytes were written, which is less than `buf.len()` + /// only at end of entry. + /// + /// Unlike `read_bytes`, nothing is allocated and the bytes are not copied + /// again on the way out, so this is what a caller reading an entry of any + /// size should use. + /// + /// zip-rs verifies the CRC only on the read that reaches the end of the + /// entry, so reading exactly the uncompressed size never checks it; read + /// until a short count (or 0) to have the entry validated. + /// + /// Returns an error if `is_none()` returns true, or + /// `ZipErrorCode::Internal` on decompression, CRC or I/O errors. + pub fn read_into(&mut self, buf: &mut [u8]) -> Result { + read_into_impl(&mut self.file, buf) + } + + /// Discards up to `max_bytes` and returns how many were discarded, which + /// is less than `max_bytes` only at end of entry. + /// + /// A zip entry is a forward-only stream, so this is how a caller seeks + /// within one. Nothing is allocated, whatever the distance. + /// + /// As with `read_into`, the CRC is checked only by the read or skip that + /// reaches the end of the entry, so skipping exactly the uncompressed size + /// never checks it. + /// + /// Returns an error if `is_none()` returns true, or + /// `ZipErrorCode::Internal` on decompression, CRC or I/O errors. + pub fn skip(&mut self, max_bytes: u64) -> Result { + skip_impl(&mut self.file, max_bytes) + } } impl<'a> Read for FsZipFile<'a> { @@ -317,19 +450,28 @@ fn get_extra_data_impl<'a, R: Read>(file: &Option>>) - file.as_ref().and_then(|f| f.extra_data()).map(VecU8::from).unwrap_or_default() } +fn get_header_start_impl<'a, R: Read>(file: &Option>>) -> u64 { + match file.as_ref() { + Some(file) => file.header_start(), + None => 0, + } +} + +fn get_data_start_impl<'a, R: Read>(file: &Option>>) -> u64 { + match file.as_ref() { + // `data_start` is `None` until the local file header has been parsed, + // and a real entry can never start at offset 0 because the header + // itself occupies at least 30 bytes, so 0 is unambiguously "unknown". + Some(file) => file.data_start().unwrap_or(0), + None => 0, + } +} + fn get_compression_method_impl<'a, R: Read>( file: &Option>>, ) -> CompressionMethod { match file.as_ref() { - Some(file) => match file.compression() { - ZipCrateCompressionMethod::Stored => CompressionMethod::Stored, - ZipCrateCompressionMethod::Deflated => CompressionMethod::Deflated, - ZipCrateCompressionMethod::Bzip2 => CompressionMethod::Bzip2, - ZipCrateCompressionMethod::Zstd => CompressionMethod::Zstd, - ZipCrateCompressionMethod::Lzma => CompressionMethod::Lzma, - ZipCrateCompressionMethod::Xz => CompressionMethod::Xz, - _ => CompressionMethod::Unsupported, - }, + Some(file) => CompressionMethod::from_zip_crate(file.compression()), None => CompressionMethod::Unsupported, } } @@ -345,7 +487,7 @@ fn get_file_data_impl<'a, R: Read>( Err(e) => Err(ZipError::internal(e.to_string())), } } - None => Ok(VecU8::default()), + None => Err(ZipError::internal("ZipFile is not available")), } } @@ -371,6 +513,60 @@ fn get_file_data_with_max_size_impl<'a, R: Read>( Err(e) => Err(ZipError::internal(e.to_string())), } } - None => Ok(VecU8::default()), + None => Err(ZipError::internal("ZipFile is not available")), } } + +fn read_bytes_impl<'a, R: Read>( + file: &mut Option>>, + max_bytes: usize, +) -> Result { + match file.as_mut() { + Some(file) => { + // Grow the buffer only as data arrives instead of eagerly + // allocating `max_bytes` (which is caller-chosen and may be huge). + let mut buffer = Vec::new(); + match (&mut **file).take(max_bytes as u64).read_to_end(&mut buffer) { + Ok(_) => Ok(buffer.into()), + Err(e) => Err(ZipError::internal(e.to_string())), + } + } + None => Err(ZipError::internal("ZipFile is not available")), + } +} + +fn read_into_impl<'a, R: Read>( + file: &mut Option>>, + buf: &mut [u8], +) -> Result { + let Some(file) = file.as_mut() else { + return Err(ZipError::internal("ZipFile is not available")); + }; + // A decompressor returns what the current block happens to yield, which is + // routinely less than was asked for, so fill the buffer here rather than + // making the caller loop -- across a language boundary, in C++'s case. + let mut filled = 0usize; + while filled < buf.len() { + match file.read(&mut buf[filled..]) { + Ok(0) => break, + Ok(bytes_read) => filled += bytes_read, + // Read contracts allow this and it means nothing was consumed. + Err(e) if e.kind() == ErrorKind::Interrupted => {} + Err(e) => return Err(ZipError::internal(e.to_string())), + } + } + Ok(filled as u64) +} + +fn skip_impl<'a, R: Read>( + file: &mut Option>>, + max_bytes: u64, +) -> Result { + let Some(file) = file.as_mut() else { + return Err(ZipError::internal("ZipFile is not available")); + }; + // `io::copy` drives this with a reused stack buffer, so a long skip costs + // no allocation however far it goes. + std::io::copy(&mut (&mut **file).take(max_bytes), &mut std::io::sink()) + .map_err(|e| ZipError::internal(e.to_string())) +} diff --git a/zip/rust/lib.rs b/zip/rust/lib.rs index ce9153f2..207e28d6 100644 --- a/zip/rust/lib.rs +++ b/zip/rust/lib.rs @@ -4,14 +4,24 @@ //! //! WARNING: This crate should never be used from Rust, instead use zip directly. +mod entry_metadata; +pub use entry_metadata::ZipEntryMetadata; + mod file; pub use file::{BufferedZipFile, FsZipFile}; mod read; -pub use read::{BufferedZipArchive, FsZipArchive}; +pub use read::{ + read_buffered_zipfile_from_stream, read_buffered_zipfile_from_stream_with_compressed_size, + read_fs_zipfile_from_stream, read_fs_zipfile_from_stream_with_compressed_size, + BufferedZipArchive, BufferedZipStreamReader, FsZipArchive, FsZipStreamReader, +}; mod write; -pub use write::{BufferedZipWriter, CompressionMethod, FsZipWriter, ZipWriterFileOptions}; +pub use write::{ + new_buffered_zip_stream_writer, new_fs_zip_stream_writer, BufferedZipStreamWriter, + BufferedZipWriter, CompressionMethod, FsZipStreamWriter, FsZipWriter, ZipWriterFileOptions, +}; mod vec_u8; pub use vec_u8::VecU8; diff --git a/zip/rust/read.rs b/zip/rust/read.rs index f38b872c..cb5625e3 100644 --- a/zip/rust/read.rs +++ b/zip/rust/read.rs @@ -1,10 +1,11 @@ // Crubit does not support generic type parameters, so we need to implement // BufferedZipArchive and FsZipArchive manually. -use crate::{BufferedZipFile, FsZipFile, VecU8, ZipError}; +use crate::{BufferedZipFile, FsZipFile, VecU8, ZipEntryMetadata, ZipError}; use std::fmt::{Debug, Formatter}; use std::fs::File; use std::io::{Cursor, Read, Seek}; +use zip::read::ZipFile as WrappedZipFile; use zip::ZipArchive as WrappedZipArchive; #[derive(Default)] @@ -75,7 +76,7 @@ impl BufferedZipArchive { /// Returns a zip file by its index. /// - /// Returns an empty zip file if archive is not open. + /// Returns an error if the archive is not open (`is_none()`). /// Returns an error if `index` is out of bounds. pub fn get_file_by_index(&mut self, index: usize) -> Result, ZipError> { match self.reader.as_mut() { @@ -83,7 +84,7 @@ impl BufferedZipArchive { Ok(file) => Ok(BufferedZipFile::new(file)), Err(e) => Err(ZipError::out_of_range(e.to_string())), }, - None => Ok(BufferedZipFile::default()), + None => Err(ZipError::failed_precondition("Zip archive is not open")), } } @@ -92,7 +93,7 @@ impl BufferedZipArchive { /// Warning: Writers in zip-rs do not support writing compressed data as-is. /// This data will be compressed again when writing. /// - /// Returns an empty zip file if archive is not open. + /// Returns an error if the archive is not open (`is_none()`). /// Returns an error if `index` is out of bounds. pub fn get_file_by_index_raw(&mut self, index: usize) -> Result, ZipError> { match self.reader.as_mut() { @@ -100,14 +101,31 @@ impl BufferedZipArchive { Ok(file) => Ok(BufferedZipFile::new(file)), Err(e) => Err(ZipError::out_of_range(e.to_string())), }, - None => Ok(BufferedZipFile::default()), + None => Err(ZipError::failed_precondition("Zip archive is not open")), } } + + /// Returns what the central directory records about the entry at `index`, + /// without reading anything. + /// + /// Prefer this to `get_file_by_index` when the entry's contents are not + /// wanted: that one also reads the entry's local file header, so listing + /// an archive with it costs one read per entry. + /// + /// Returns an error if the archive is not open or `index` is out of + /// bounds. + pub fn get_entry_metadata(&self, index: usize) -> Result { + get_entry_metadata_impl(&self.reader, index) + } } #[derive(Default)] pub struct FsZipArchive { reader: Option>>, + /// Path the archive was opened from, retained so that `try_clone` can open + /// an independent file descriptor onto the same archive. Empty until + /// `open` succeeds. + path: String, } impl Debug for FsZipArchive { @@ -154,6 +172,7 @@ impl FsZipArchive { Ok(file) => match WrappedZipArchive::new(file) { Ok(reader) => { self.reader = Some(Box::new(reader)); + self.path = path_str.to_string(); Ok(()) } Err(e) => Err(e.to_string()), @@ -162,6 +181,51 @@ impl FsZipArchive { } } + /// Returns an independent handle onto the same archive. + /// + /// The new handle shares the central directory that `new_from_path` already + /// parsed, so this costs one `open(2)` and is O(1) in the number of entries, + /// whereas `new_from_path` is O(entries). Use it when several entries must + /// be read at once: `get_file_by_index` borrows the archive mutably, so a + /// caller that wants N concurrently open entries needs N handles, and + /// reopening by path for each one is quadratic on large archives. + /// + /// Each handle owns its own file descriptor and therefore its own file + /// position, so handles may be read independently. + /// + /// Returns `ZipErrorCode::FailedPrecondition` if the archive is not open + /// (default-constructed or moved-from) or if the path can no longer be + /// opened. The caller must ensure the path still resolves to the same, + /// unmodified file it was opened from: not renamed, replaced or rewritten + /// in between (for a relative path, the working directory must not have + /// changed either). The shared central directory would otherwise describe + /// byte ranges that no longer hold the entries it names. + pub fn try_clone(&self) -> Result { + let Some(reader) = self.reader.as_ref() else { + return Err(ZipError::failed_precondition("Zip archive is not open")); + }; + let file = match File::open(&self.path) { + Ok(file) => file, + Err(e) => { + return Err(ZipError::failed_precondition(format!( + "Failed to reopen zip archive: {}", + e + ))) + } + }; + // SAFETY: `unsafe_new_with_metadata` requires `reader.metadata()` to + // describe the bytes `file` will yield. `file` is a freshly opened + // descriptor for the same path `reader` parsed its central directory + // from, and this function's contract (see the doc comment) makes the + // caller responsible for that path still resolving to the same, + // unmodified file. Should it not anyway, every subsequent read goes + // through zip-rs's bounds, signature and CRC checks, so a mismatch + // surfaces as an error rather than as memory unsafety. + let archive = + unsafe { WrappedZipArchive::unsafe_new_with_metadata(file, reader.metadata()) }; + Ok(FsZipArchive { reader: Some(Box::new(archive)), path: self.path.clone() }) + } + /// Returns whether the zip archive is none (due to being /// default-constructed or moved-from). pub fn is_none(&self) -> bool { @@ -180,7 +244,7 @@ impl FsZipArchive { /// Returns a zip file by its index. /// - /// Returns an empty zip file if archive is not open. + /// Returns an error if the archive is not open (`is_none()`). /// Returns an error if `index` is out of bounds. pub fn get_file_by_index(&mut self, index: usize) -> Result, ZipError> { match self.reader.as_mut() { @@ -188,7 +252,7 @@ impl FsZipArchive { Ok(file) => Ok(FsZipFile::new(file)), Err(e) => Err(ZipError::out_of_range(e.to_string())), }, - None => Ok(FsZipFile::default()), + None => Err(ZipError::failed_precondition("Zip archive is not open")), } } @@ -197,7 +261,7 @@ impl FsZipArchive { /// Warning: Writers in zip-rs do not support writing compressed data as-is. /// This data will be compressed again when writing. /// - /// Returns an empty zip file if archive is not open. + /// Returns an error if the archive is not open (`is_none()`). /// Returns an error if `index` is out of bounds. pub fn get_file_by_index_raw(&mut self, index: usize) -> Result, ZipError> { match self.reader.as_mut() { @@ -205,9 +269,22 @@ impl FsZipArchive { Ok(file) => Ok(FsZipFile::new(file)), Err(e) => Err(ZipError::out_of_range(e.to_string())), }, - None => Ok(FsZipFile::default()), + None => Err(ZipError::failed_precondition("Zip archive is not open")), } } + + /// Returns what the central directory records about the entry at `index`, + /// without reading anything. + /// + /// Prefer this to `get_file_by_index` when the entry's contents are not + /// wanted: that one also reads the entry's local file header, so listing + /// an archive with it costs one read per entry. + /// + /// Returns an error if the archive is not open or `index` is out of + /// bounds. + pub fn get_entry_metadata(&self, index: usize) -> Result { + get_entry_metadata_impl(&self.reader, index) + } } fn get_length_impl(reader: &Option>>) -> usize { @@ -220,3 +297,437 @@ fn get_length_impl(reader: &Option>>) - fn get_comment_impl(reader: &Option>>) -> VecU8 { reader.as_ref().map(|r| VecU8::copy_from_slice(r.comment())).unwrap_or_default() } + +fn get_entry_metadata_impl( + reader: &Option>>, + index: usize, +) -> Result { + let Some(r) = reader.as_ref() else { + return Err(ZipError::failed_precondition("Zip archive is not open")); + }; + // The central directory was parsed when the archive was opened, so this + // reads nothing. + match ZipEntryMetadata::from_central_directory(&r.metadata(), index) { + Some(metadata) => Ok(metadata), + None => Err(ZipError::out_of_range(format!( + "Index {index} is out of bounds for an archive of {} entries", + r.len() + ))), + } +} + +/// Shared body of `read_next_file` / `read_next_file_with_compressed_size` for +/// both stream readers. +/// +/// Returns `Ok(None)` if the stream is already finished or once the start of +/// the central directory is reached (which also sets `finished`). Returns a +/// `FailedPrecondition` error if `reader` is `None`. When `compressed_size` is +/// `Some`, it is used for entries whose local header carries no size (general +/// purpose bit 3 / data descriptor). +fn read_next_file_impl<'a, R: Read>( + reader: &'a mut Option>, + finished: &mut bool, + compressed_size: Option, +) -> Result>, ZipError> { + if *finished { + return Ok(None); + } + let reader = match reader.as_deref_mut() { + Some(reader) => reader, + None => return Err(ZipError::failed_precondition("Zip stream reader is not open")), + }; + let result = match compressed_size { + Some(size) => zip::read::read_zipfile_from_stream_with_compressed_size(reader, size), + None => zip::read::read_zipfile_from_stream(reader), + }; + match result { + Ok(Some(file)) => Ok(Some(file)), + Ok(None) => { + *finished = true; + Ok(None) + } + Err(e) => Err(ZipError::from_zip_error(e)), + } +} + +/// A streaming zip reader that reads from an in-memory buffer without seeking. +#[derive(Default)] +pub struct BufferedZipStreamReader { + reader: Option>>>, + finished: bool, +} + +impl Debug for BufferedZipStreamReader { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("BufferedZipStreamReader") + .field("reader", &if self.reader.is_some() { "Some(Cursor>)" } else { "None" }) + .field("finished", &self.finished) + .finish() + } +} + +impl BufferedZipStreamReader { + /// Initializes an empty `BufferedZipStreamReader`. + pub fn new() -> Self { + Self::default() + } + + /// Creates a new `BufferedZipStreamReader` from data. + pub fn new_from_data(data: VecU8) -> Result { + Ok(Self { reader: Some(Box::new(Cursor::new(data.into_vec()))), finished: false }) + } + + /// Returns whether the stream reader is none (due to being default-constructed + /// or moved-from). + pub fn is_none(&self) -> bool { + self.reader.is_none() + } + + /// Returns whether the end of the archive stream has been reached. + pub fn is_finished(&self) -> bool { + self.finished + } + + /// Reads the next zip file from the stream. + /// + /// When there are no more files in the archive (start of central directory + /// is reached), returns a `BufferedZipFile` where `is_none()` returns true. + pub fn read_next_file(&mut self) -> Result, ZipError> { + Ok(read_next_file_impl(&mut self.reader, &mut self.finished, None)? + .map(BufferedZipFile::new) + .unwrap_or_default()) + } + + /// Reads the next zip file from the stream with an assumed compressed size. + /// + /// When there are no more files in the archive, returns a `BufferedZipFile` + /// where `is_none()` returns true. + pub fn read_next_file_with_compressed_size( + &mut self, + compressed_size: u64, + ) -> Result, ZipError> { + Ok(read_next_file_impl(&mut self.reader, &mut self.finished, Some(compressed_size))? + .map(BufferedZipFile::new) + .unwrap_or_default()) + } +} + +/// A streaming zip reader that reads sequentially from a file without seeking. +#[derive(Default)] +pub struct FsZipStreamReader { + reader: Option>, + finished: bool, +} + +impl Debug for FsZipStreamReader { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("FsZipStreamReader") + .field("reader", &if self.reader.is_some() { "Some(File)" } else { "None" }) + .field("finished", &self.finished) + .finish() + } +} + +impl FsZipStreamReader { + /// Initializes an empty `FsZipStreamReader`. + pub fn new() -> Self { + Self::default() + } + + /// Creates a new `FsZipStreamReader` from a path. + pub fn new_from_path(path: &[u8]) -> Result { + let path_str = std::str::from_utf8(path) + .map_err(|e| ZipError::invalid_argument(format!("Failed to open zip stream: {}", e)))?; + let file = File::open(path_str) + .map_err(|e| ZipError::invalid_argument(format!("Failed to open zip stream: {}", e)))?; + Ok(Self { reader: Some(Box::new(file)), finished: false }) + } + + /// Returns whether the stream reader is none (due to being default-constructed + /// or moved-from). + pub fn is_none(&self) -> bool { + self.reader.is_none() + } + + /// Returns whether the end of the archive stream has been reached. + pub fn is_finished(&self) -> bool { + self.finished + } + + /// Reads the next zip file from the stream. + /// + /// When there are no more files in the archive (start of central directory + /// is reached), returns a `FsZipFile` where `is_none()` returns true. + pub fn read_next_file(&mut self) -> Result, ZipError> { + Ok(read_next_file_impl(&mut self.reader, &mut self.finished, None)? + .map(FsZipFile::new) + .unwrap_or_default()) + } + + /// Reads the next zip file from the stream with an assumed compressed size. + /// + /// When there are no more files in the archive, returns a `FsZipFile` + /// where `is_none()` returns true. + pub fn read_next_file_with_compressed_size( + &mut self, + compressed_size: u64, + ) -> Result, ZipError> { + Ok(read_next_file_impl(&mut self.reader, &mut self.finished, Some(compressed_size))? + .map(FsZipFile::new) + .unwrap_or_default()) + } +} + +/// Reads the next zip file from a buffered stream reader. +pub fn read_buffered_zipfile_from_stream( + reader: &mut BufferedZipStreamReader, +) -> Result, ZipError> { + reader.read_next_file() +} + +/// Reads the next zip file from a buffered stream reader with an assumed compressed size. +pub fn read_buffered_zipfile_from_stream_with_compressed_size( + reader: &mut BufferedZipStreamReader, + compressed_size: u64, +) -> Result, ZipError> { + reader.read_next_file_with_compressed_size(compressed_size) +} + +/// Reads the next zip file from a filesystem stream reader. +pub fn read_fs_zipfile_from_stream( + reader: &mut FsZipStreamReader, +) -> Result, ZipError> { + reader.read_next_file() +} + +/// Reads the next zip file from a filesystem stream reader with an assumed compressed size. +pub fn read_fs_zipfile_from_stream_with_compressed_size( + reader: &mut FsZipStreamReader, + compressed_size: u64, +) -> Result, ZipError> { + reader.read_next_file_with_compressed_size(compressed_size) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{BufferedZipWriter, CompressionMethod, ZipErrorCode, ZipWriterFileOptions}; + use googletest::prelude::*; + use std::io::Write; + + fn create_test_zip() -> VecU8 { + let mut writer = BufferedZipWriter::new_from_data(VecU8::default(), false).unwrap(); + let options = ZipWriterFileOptions::new().compression_method(CompressionMethod::Stored); + writer.start_file(b"hello.txt", options).unwrap(); + writer.write_data(VecU8::from(b"Hello, world!".to_vec())).unwrap(); + writer.start_file(b"second.txt", options).unwrap(); + writer.write_data(VecU8::from(b"Second file content".to_vec())).unwrap(); + writer.finish().unwrap() + } + + /// Writes `create_test_zip()` to a uniquely named file under the test's + /// temporary directory and returns the path. + fn create_test_zip_file(name: &str) -> String { + let dir = std::env::var("TEST_TMPDIR").unwrap_or_else(|_| "/tmp".to_string()); + let path = format!("{}/{}.zip", dir, name); + let mut file = File::create(&path).unwrap(); + file.write_all(create_test_zip().as_slice()).unwrap(); + file.flush().unwrap(); + path + } + + #[gtest] + fn test_fs_archive_try_clone_reads_the_same_entries() { + let path = create_test_zip_file("try_clone_same_entries"); + let archive = FsZipArchive::new_from_path(path.as_bytes()).unwrap(); + let mut clone = archive.try_clone().unwrap(); + + expect_false!(clone.is_none()); + expect_eq!(clone.get_length(), archive.get_length()); + let mut file = clone.get_file_by_index(0).unwrap(); + expect_eq!(file.get_file_name().as_slice(), b"hello.txt"); + expect_eq!(file.get_file_data().unwrap().as_slice(), b"Hello, world!"); + } + + #[gtest] + fn test_fs_archive_clones_have_independent_positions() { + let path = create_test_zip_file("try_clone_independent"); + let mut archive = FsZipArchive::new_from_path(path.as_bytes()).unwrap(); + let mut clone = archive.try_clone().unwrap(); + + // Each handle owns its own descriptor, so reading one entry to + // completion must not disturb the other handle's entry. This is the + // whole point of `try_clone`: `get_file_by_index` borrows the archive + // mutably, so two entries can only be open at once via two handles. + // Open both before reading either, then read them in the opposite + // order, so that a shared file position would be detected. + let mut first = archive.get_file_by_index(0).unwrap(); + let mut second = clone.get_file_by_index(1).unwrap(); + expect_eq!(second.get_file_data().unwrap().as_slice(), b"Second file content"); + expect_eq!(first.get_file_data().unwrap().as_slice(), b"Hello, world!"); + } + + #[gtest] + fn test_fs_archive_try_clone_fails_when_not_open() { + let archive = FsZipArchive::new(); + expect_eq!(archive.try_clone().unwrap_err().code, ZipErrorCode::FailedPrecondition); + } + + #[gtest] + fn test_entry_offsets_delimit_the_stored_bytes() { + let zip_data = create_test_zip(); + let raw = zip_data.as_slice().to_vec(); + let mut archive = BufferedZipArchive::new_from_data(zip_data).unwrap(); + let file = archive.get_file_by_index(0).unwrap(); + + let header_start = file.get_header_start(); + let data_start = file.get_data_start(); + let compressed_size = file.get_compressed_size(); + + // The first entry's local header sits at the very start of the archive, + // and its data follows the 30-byte fixed header plus the file name. + expect_eq!(header_start, 0); + expect_eq!(data_start, 30 + "hello.txt".len() as u64); + // The entry is Stored, so the raw bytes in that range are the contents. + let begin = data_start as usize; + let end = begin + compressed_size as usize; + expect_eq!(&raw[begin..end], b"Hello, world!"); + } + + #[gtest] + fn test_stream_entries_report_unknown_data_start() { + let zip_data = create_test_zip(); + let mut reader = BufferedZipStreamReader::new_from_data(zip_data).unwrap(); + let file = reader.read_next_file().unwrap(); + + // Stream readers never learn where the entry lives in the archive. + // `data_start` reports that as 0, which no real entry can have; + // `header_start` is simply 0 and not meaningful for stream entries (the + // first entry of an archive genuinely starts at 0 too). + expect_eq!(file.get_data_start(), 0); + expect_eq!(file.get_header_start(), 0); + } + + #[gtest] + fn test_get_entry_metadata_lists_the_central_directory() { + let zip_data = create_test_zip(); + let archive = BufferedZipArchive::new_from_data(zip_data).unwrap(); + + let first = archive.get_entry_metadata(0).unwrap(); + expect_eq!(first.get_file_name().as_slice(), b"hello.txt"); + expect_eq!(first.get_uncompressed_size(), "Hello, world!".len() as u64); + expect_eq!(first.get_compressed_size(), "Hello, world!".len() as u64); + expect_eq!(first.get_compression_method(), CompressionMethod::Stored); + expect_eq!(first.get_header_start(), 0); + expect_true!(first.is_file()); + expect_false!(first.is_dir()); + expect_false!(first.is_encrypted()); + + let second = archive.get_entry_metadata(1).unwrap(); + expect_eq!(second.get_file_name().as_slice(), b"second.txt"); + expect_eq!(second.get_uncompressed_size(), "Second file content".len() as u64); + // The second entry's local header follows the first entry: its 30-byte + // fixed header, its name and its stored contents. + expect_eq!( + second.get_header_start(), + 30 + "hello.txt".len() as u64 + "Hello, world!".len() as u64 + ); + + let code = archive.get_entry_metadata(2).err().map(|e| e.code); + expect_eq!(code, Some(ZipErrorCode::OutOfRange)); + } + + #[gtest] + fn test_get_entry_metadata_on_an_unopened_archive_fails_precondition() { + let archive = BufferedZipArchive::new(); + expect_true!(archive.is_none()); + let code = archive.get_entry_metadata(0).err().map(|e| e.code); + expect_eq!(code, Some(ZipErrorCode::FailedPrecondition)); + } + + #[gtest] + fn test_buffered_stream_reader_read_all_files() { + let zip_data = create_test_zip(); + let mut reader = BufferedZipStreamReader::new_from_data(zip_data).unwrap(); + expect_false!(reader.is_none()); + expect_false!(reader.is_finished()); + + { + let mut file1 = reader.read_next_file().unwrap(); + expect_false!(file1.is_none()); + expect_eq!(file1.get_file_name().as_slice(), b"hello.txt"); + expect_true!(file1.is_file()); + expect_false!(file1.is_dir()); + expect_eq!(file1.get_file_data().unwrap().as_slice(), b"Hello, world!"); + } + + { + let mut file2 = reader.read_next_file().unwrap(); + expect_false!(file2.is_none()); + expect_eq!(file2.get_file_name().as_slice(), b"second.txt"); + expect_eq!(file2.get_file_data().unwrap().as_slice(), b"Second file content"); + } + + // Third read should return None (end of files) + { + let file3 = reader.read_next_file().unwrap(); + expect_true!(file3.is_none()); + } + expect_true!(reader.is_finished()); + + // Repeated reads after finish should be idempotent + { + let file4 = reader.read_next_file().unwrap(); + expect_true!(file4.is_none()); + } + } + + #[gtest] + fn test_buffered_stream_reader_chunked_read() { + let zip_data = create_test_zip(); + let mut reader = BufferedZipStreamReader::new_from_data(zip_data).unwrap(); + let mut file = reader.read_next_file().unwrap(); + expect_false!(file.is_none()); + + // Read in chunks of 5 bytes + let chunk1 = file.read_bytes(5).unwrap(); + expect_eq!(chunk1.as_slice(), b"Hello"); + let chunk2 = file.read_bytes(5).unwrap(); + expect_eq!(chunk2.as_slice(), b", wor"); + let chunk3 = file.read_bytes(5).unwrap(); + expect_eq!(chunk3.as_slice(), b"ld!"); + let chunk4 = file.read_bytes(5).unwrap(); + expect_true!(chunk4.is_empty()); + } + + #[gtest] + fn test_buffered_stream_reader_free_functions() { + let zip_data = create_test_zip(); + let mut reader = BufferedZipStreamReader::new_from_data(zip_data).unwrap(); + let mut file = read_buffered_zipfile_from_stream(&mut reader).unwrap(); + expect_false!(file.is_none()); + expect_eq!(file.get_file_name().as_slice(), b"hello.txt"); + expect_eq!(file.get_file_data().unwrap().as_slice(), b"Hello, world!"); + } + + #[gtest] + fn test_buffered_stream_reader_with_compressed_size() { + let zip_data = create_test_zip(); + let mut reader = BufferedZipStreamReader::new_from_data(zip_data).unwrap(); + // File 1 is uncompressed "Hello, world!" which is 13 bytes stored + let mut file = reader.read_next_file_with_compressed_size(13).unwrap(); + expect_false!(file.is_none()); + expect_eq!(file.get_file_name().as_slice(), b"hello.txt"); + expect_eq!(file.get_file_data().unwrap().as_slice(), b"Hello, world!"); + } + + #[gtest] + fn test_buffered_stream_reader_invalid_data() { + let mut reader = + BufferedZipStreamReader::new_from_data(VecU8::from(b"not a zip".to_vec())).unwrap(); + let res = reader.read_next_file(); + // A bad local file header signature maps to InvalidArchive. + let code = res.err().map(|e| e.code); + expect_eq!(code, Some(ZipErrorCode::InvalidArgument)); + } +} diff --git a/zip/rust/write.rs b/zip/rust/write.rs index 90e77b9f..1016ae57 100644 --- a/zip/rust/write.rs +++ b/zip/rust/write.rs @@ -6,8 +6,8 @@ use std::fmt::{Debug, Formatter}; use std::fs::{File, OpenOptions}; use std::io::{copy, Cursor, Read, Seek, Write}; use zip::{ - write::FileOptions, CompressionMethod as ZipCrateCompressionMethod, - ZipWriter as WrappedZipWriter, + write::{FileOptions, StreamWriter}, + CompressionMethod as ZipCrateCompressionMethod, ZipWriter as WrappedZipWriter, }; // Expose some of the options for writing files to the zip archive. @@ -50,6 +50,23 @@ impl CompressionMethod { } } + /// Maps a method read from an archive onto the mirror, `Unsupported` for + /// anything the mirror does not name. + /// + /// This function won't be generated by Crubit and is only called from + /// rust. (b/259749095) + pub(crate) fn from_zip_crate(method: ZipCrateCompressionMethod) -> CompressionMethod { + match method { + ZipCrateCompressionMethod::Stored => CompressionMethod::Stored, + ZipCrateCompressionMethod::Deflated => CompressionMethod::Deflated, + ZipCrateCompressionMethod::Bzip2 => CompressionMethod::Bzip2, + ZipCrateCompressionMethod::Zstd => CompressionMethod::Zstd, + ZipCrateCompressionMethod::Lzma => CompressionMethod::Lzma, + ZipCrateCompressionMethod::Xz => CompressionMethod::Xz, + _ => CompressionMethod::Unsupported, + } + } + fn into_zip(self) -> ZipCrateCompressionMethod { match self { CompressionMethod::Deflated => ZipCrateCompressionMethod::Deflated, @@ -211,14 +228,7 @@ impl BufferedZipWriter { /// Finishes writing the zip archive and returns the buffered data. pub fn finish(&mut self) -> Result { - if let Some(writer) = self.writer.take() { - match writer.finish() { - Ok(cursor) => Ok(cursor.into_inner().into()), - Err(e) => Err(ZipError::internal(e.to_string())), - } - } else { - Err(ZipError::failed_precondition("writer is not open")) - } + finish_impl(&mut self.writer).map(|cursor| cursor.into_inner().into()) } /// Creates a new file in the zip archive and start writing to it. @@ -264,6 +274,23 @@ impl BufferedZipWriter { pub fn write_file_content(&mut self, path: &[u8]) -> Result<(), ZipError> { write_file_content_impl(&mut self.writer, path) } + + /// Sets the archive comment written by `finish`. `comment` must be at most + /// `u16::MAX` (65535) bytes; a longer comment returns InvalidArgument and + /// leaves the previously set comment untouched. + pub fn set_comment(&mut self, comment: &[u8]) -> Result<(), ZipError> { + set_comment_impl(&mut self.writer, comment) + } + + /// Flushes the underlying sink; does not finalize the archive. + pub fn flush(&mut self) -> Result<(), ZipError> { + flush_impl(&mut self.writer) + } + + /// Returns whether seeking is possible (always true for BufferedZipWriter). + pub fn is_seek_possible(&self) -> bool { + true + } } #[derive(Default)] @@ -335,14 +362,7 @@ impl FsZipWriter { /// Finishes writing the zip archive to file. pub fn finish(&mut self) -> Result<(), ZipError> { - if let Some(writer) = self.writer.take() { - match writer.finish() { - Ok(_) => Ok(()), - Err(e) => Err(ZipError::internal(e.to_string())), - } - } else { - Err(ZipError::failed_precondition("writer is not open")) - } + finish_impl(&mut self.writer).map(|_| ()) } /// Creates a new file in the zip archive and start writing to it. @@ -388,6 +408,23 @@ impl FsZipWriter { pub fn write_file_content(&mut self, path: &[u8]) -> Result<(), ZipError> { write_file_content_impl(&mut self.writer, path) } + + /// Sets the archive comment written by `finish`. `comment` must be at most + /// `u16::MAX` (65535) bytes; a longer comment returns InvalidArgument and + /// leaves the previously set comment untouched. + pub fn set_comment(&mut self, comment: &[u8]) -> Result<(), ZipError> { + set_comment_impl(&mut self.writer, comment) + } + + /// Flushes the underlying sink; does not finalize the archive. + pub fn flush(&mut self) -> Result<(), ZipError> { + flush_impl(&mut self.writer) + } + + /// Returns whether seeking is possible (always true for FsZipWriter). + pub fn is_seek_possible(&self) -> bool { + true + } } fn start_file_impl( @@ -476,3 +513,384 @@ fn write_file_content_impl( Err(ZipError::failed_precondition("writer is not open")) } } + +fn set_comment_impl( + writer: &mut Option>>, + comment: &[u8], +) -> Result<(), ZipError> { + if let Some(writer) = writer.as_deref_mut() { + // Checked up front: zip-rs's `set_raw_comment` stores the first + // `u16::MAX` bytes and only then returns an error, which would leave a + // silently truncated comment in the finished archive. + if comment.len() > u16::MAX as usize { + return Err(ZipError::invalid_argument(format!( + "archive comment is {} bytes; at most {} allowed", + comment.len(), + u16::MAX + ))); + } + match writer.set_raw_comment(Box::from(comment)) { + Ok(_) => Ok(()), + Err(e) => Err(ZipError::invalid_argument(e.to_string())), + } + } else { + Err(ZipError::failed_precondition("writer is not open")) + } +} + +fn flush_impl( + writer: &mut Option>>, +) -> Result<(), ZipError> { + if let Some(writer) = writer.as_deref_mut() { + match writer.flush() { + Ok(_) => Ok(()), + Err(e) => Err(ZipError::internal(e.to_string())), + } + } else { + Err(ZipError::failed_precondition("writer is not open")) + } +} + +/// Writes the central directory and consumes the inner writer, returning the +/// underlying sink. +fn finish_impl( + writer: &mut Option>>, +) -> Result { + match writer.take() { + Some(writer) => writer.finish().map_err(|e| ZipError::internal(e.to_string())), + None => Err(ZipError::failed_precondition("writer is not open")), + } +} + +type InnerBufferedStreamWriter = WrappedZipWriter>>>; +type InnerFsStreamWriter = WrappedZipWriter>; + +#[derive(Default)] +/// A zip writer that streams to an in-memory buffer without seeking. +pub struct BufferedZipStreamWriter { + writer: Option>, +} + +impl Debug for BufferedZipStreamWriter { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("BufferedZipStreamWriter") + .field( + "writer", + &if self.writer.is_some() { + "Some(WrappedZipWriter>>>>)" + } else { + "None" + }, + ) + .finish() + } +} + +impl BufferedZipStreamWriter { + /// Creates a none `BufferedZipStreamWriter` (Crubit requires `Default`); + /// every method on it returns FailedPrecondition until it is replaced by + /// `new_stream()`. + pub fn new() -> Self { + Self::default() + } + + /// Creates a new streaming `BufferedZipStreamWriter`. + pub fn new_stream() -> Self { + let cursor = Cursor::new(Vec::new()); + Self { writer: Some(Box::new(WrappedZipWriter::new_stream(cursor))) } + } + + /// Returns whether the zip writer is none (due to being + /// default-constructed or moved-from). + pub fn is_none(&self) -> bool { + self.writer.is_none() + } + + /// Returns whether seeking is possible (always false for stream writers). + pub fn is_seek_possible(&self) -> bool { + false + } + + /// Finishes writing the zip archive and returns the buffered data. + pub fn finish(&mut self) -> Result { + finish_impl(&mut self.writer) + .map(|stream_writer| stream_writer.into_inner().into_inner().into()) + } + + /// Creates a new file in the zip archive and starts writing to it. + pub fn start_file( + &mut self, + name: &[u8], + options: ZipWriterFileOptions, + ) -> Result<(), ZipError> { + start_file_impl(&mut self.writer, name, options) + } + + /// Adds a directory to the zip archive. + pub fn add_directory( + &mut self, + name: &[u8], + options: ZipWriterFileOptions, + ) -> Result<(), ZipError> { + add_directory_impl(&mut self.writer, name, options) + } + + /// Writes data to the current file in the zip archive. + pub fn write_data(&mut self, data: VecU8) -> Result<(), ZipError> { + write_data_impl(&mut self.writer, data) + } + + /// Writes file content from a `BufferedZipFile` to the current file in the zip archive. + pub fn write_buffered_zip_file_content( + &mut self, + file: &mut BufferedZipFile, + ) -> Result<(), ZipError> { + do_copy_impl(&mut self.writer, file) + } + + /// Writes file content from a `FsZipFile` to the current file in the zip archive. + pub fn write_fs_zip_file_content(&mut self, file: &mut FsZipFile) -> Result<(), ZipError> { + do_copy_impl(&mut self.writer, file) + } + + /// Writes file content from a path to the current file in the zip archive. + pub fn write_file_content(&mut self, path: &[u8]) -> Result<(), ZipError> { + write_file_content_impl(&mut self.writer, path) + } + + /// Archive comments are not supported by streaming writers: zip 9.x + /// finalizes a commented archive by seeking back over the comment, which a + /// non-seekable writer cannot do (`zip::write::StreamWriter` only permits + /// no-op seeks). Always returns a FailedPrecondition error. + pub fn set_comment(&mut self, _comment: &[u8]) -> Result<(), ZipError> { + Err(ZipError::failed_precondition( + "archive comments are not supported by streaming zip writers", + )) + } + + /// Flushes the underlying sink; does not finalize the archive. + pub fn flush(&mut self) -> Result<(), ZipError> { + flush_impl(&mut self.writer) + } +} + +#[derive(Default)] +/// A zip writer that streams to a file on the filesystem without seeking. +pub struct FsZipStreamWriter { + writer: Option>, +} + +impl Debug for FsZipStreamWriter { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("FsZipStreamWriter") + .field( + "writer", + &if self.writer.is_some() { + "Some(WrappedZipWriter>)" + } else { + "None" + }, + ) + .finish() + } +} + +impl FsZipStreamWriter { + /// Creates a none `FsZipStreamWriter` (Crubit requires `Default`); every + /// method on it returns FailedPrecondition until it is replaced by + /// `new_from_path()`. + pub fn new() -> Self { + Self::default() + } + + /// Creates a new streaming `FsZipStreamWriter` writing to `path`. The file + /// is created or truncated; there is no append mode. + pub fn new_from_path(path: &[u8]) -> Result { + let mut writer = Self::default(); + if let Err(e) = writer.open(path) { + return Err(ZipError::invalid_argument(format!("Failed to open zip archive: {}", e))); + } + Ok(writer) + } + + fn open(&mut self, path: &[u8]) -> Result<(), String> { + let path_lossy = String::from_utf8_lossy(path); + let path_str = path_lossy.as_ref(); + match OpenOptions::new().write(true).create(true).truncate(true).open(path_str) { + Ok(file) => { + self.writer = Some(Box::new(WrappedZipWriter::new_stream(file))); + Ok(()) + } + Err(e) => Err(e.to_string()), + } + } + + /// Returns whether the zip writer is none (due to being + /// default-constructed or moved-from). + pub fn is_none(&self) -> bool { + self.writer.is_none() + } + + /// Returns whether seeking is possible (always false for stream writers). + pub fn is_seek_possible(&self) -> bool { + false + } + + /// Finishes writing the zip archive to file. + pub fn finish(&mut self) -> Result<(), ZipError> { + finish_impl(&mut self.writer).map(|_| ()) + } + + /// Creates a new file in the zip archive and starts writing to it. + pub fn start_file( + &mut self, + name: &[u8], + options: ZipWriterFileOptions, + ) -> Result<(), ZipError> { + start_file_impl(&mut self.writer, name, options) + } + + /// Adds a directory to the zip archive. + pub fn add_directory( + &mut self, + name: &[u8], + options: ZipWriterFileOptions, + ) -> Result<(), ZipError> { + add_directory_impl(&mut self.writer, name, options) + } + + /// Writes data to the current file in the zip archive. + pub fn write_data(&mut self, data: VecU8) -> Result<(), ZipError> { + write_data_impl(&mut self.writer, data) + } + + /// Writes file content from a `BufferedZipFile` to the current file in the zip archive. + pub fn write_buffered_zip_file_content( + &mut self, + file: &mut BufferedZipFile, + ) -> Result<(), ZipError> { + do_copy_impl(&mut self.writer, file) + } + + /// Writes file content from a `FsZipFile` to the current file in the zip archive. + pub fn write_fs_zip_file_content(&mut self, file: &mut FsZipFile) -> Result<(), ZipError> { + do_copy_impl(&mut self.writer, file) + } + + /// Writes file content from a path to the current file in the zip archive. + pub fn write_file_content(&mut self, path: &[u8]) -> Result<(), ZipError> { + write_file_content_impl(&mut self.writer, path) + } + + /// Archive comments are not supported by streaming writers: zip 9.x + /// finalizes a commented archive by seeking back over the comment, which a + /// non-seekable writer cannot do (`zip::write::StreamWriter` only permits + /// no-op seeks). Always returns a FailedPrecondition error. + pub fn set_comment(&mut self, _comment: &[u8]) -> Result<(), ZipError> { + Err(ZipError::failed_precondition( + "archive comments are not supported by streaming zip writers", + )) + } + + /// Flushes the underlying sink; does not finalize the archive. + pub fn flush(&mut self) -> Result<(), ZipError> { + flush_impl(&mut self.writer) + } +} + +/// Creates a new in-memory streaming zip writer. +pub fn new_buffered_zip_stream_writer() -> BufferedZipStreamWriter { + BufferedZipStreamWriter::new_stream() +} + +/// Creates a new filesystem streaming zip writer. +pub fn new_fs_zip_stream_writer(path: &[u8]) -> Result { + FsZipStreamWriter::new_from_path(path) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{BufferedZipArchive, ZipErrorCode}; + use googletest::prelude::*; + + #[gtest] + fn test_buffered_zip_stream_writer() { + let mut writer = BufferedZipStreamWriter::new_stream(); + expect_false!(writer.is_none()); + expect_false!(writer.is_seek_possible()); + + let options = ZipWriterFileOptions::new().compression_method(CompressionMethod::Deflated); + writer.start_file(b"test.txt", options).unwrap(); + writer.write_data(VecU8::from(b"stream writer data".to_vec())).unwrap(); + writer.add_directory(b"mydir", options).unwrap(); + // Comments need a seek on finish, which a stream writer cannot do. + let err = writer.set_comment(b"archive comment").unwrap_err(); + expect_eq!(err.code, ZipErrorCode::FailedPrecondition); + writer.flush().unwrap(); + + let zip_bytes = writer.finish().unwrap(); + expect_false!(zip_bytes.is_empty()); + + // Verify that ZipArchive can read this streaming-written zip archive + let mut archive = BufferedZipArchive::new_from_data(zip_bytes).unwrap(); + expect_eq!(archive.get_length(), 2); + expect_true!(archive.get_comment().is_empty()); + + { + let mut file1 = archive.get_file_by_index(0).unwrap(); + expect_eq!(file1.get_file_name().as_slice(), b"test.txt"); + expect_eq!(file1.get_file_data().unwrap().as_slice(), b"stream writer data"); + } + + { + let file2 = archive.get_file_by_index(1).unwrap(); + expect_true!(file2.is_dir()); + expect_eq!(file2.get_file_name().as_slice(), b"mydir/"); + } + } + + #[gtest] + fn test_free_functions_stream_writer() { + let mut writer = new_buffered_zip_stream_writer(); + expect_false!(writer.is_seek_possible()); + let options = ZipWriterFileOptions::new(); + writer.start_file(b"a.txt", options).unwrap(); + writer.write_data(VecU8::from(b"content".to_vec())).unwrap(); + let zip_bytes = writer.finish().unwrap(); + + let mut archive = BufferedZipArchive::new_from_data(zip_bytes).unwrap(); + expect_eq!(archive.get_length(), 1); + let mut file = archive.get_file_by_index(0).unwrap(); + expect_eq!(file.get_file_data().unwrap().as_slice(), b"content"); + } + + #[gtest] + fn test_regular_writer_is_seek_possible_and_set_comment() { + let mut writer = BufferedZipWriter::new_from_data(VecU8::default(), false).unwrap(); + expect_true!(writer.is_seek_possible()); + let options = ZipWriterFileOptions::new(); + writer.start_file(b"b.txt", options).unwrap(); + writer.write_data(VecU8::from(b"data".to_vec())).unwrap(); + writer.set_comment(b"comment on regular writer").unwrap(); + writer.flush().unwrap(); + let zip_bytes = writer.finish().unwrap(); + + let archive = BufferedZipArchive::new_from_data(zip_bytes).unwrap(); + expect_eq!(archive.get_comment().as_slice(), b"comment on regular writer"); + } + + #[gtest] + fn test_regular_writer_rejects_overlong_comment_without_truncating() { + let mut writer = BufferedZipWriter::new_from_data(VecU8::default(), false).unwrap(); + writer.set_comment(b"kept").unwrap(); + // One byte over the u16 limit: rejected up front, previous comment kept. + let too_long = vec![b'x'; u16::MAX as usize + 1]; + let err = writer.set_comment(&too_long).unwrap_err(); + expect_eq!(err.code, ZipErrorCode::InvalidArgument); + let zip_bytes = writer.finish().unwrap(); + + let archive = BufferedZipArchive::new_from_data(zip_bytes).unwrap(); + expect_eq!(archive.get_comment().as_slice(), b"kept"); + } +} diff --git a/zip/write.cc b/zip/write.cc index d5b49432..cbf4cc11 100644 --- a/zip/write.cc +++ b/zip/write.cc @@ -22,6 +22,51 @@ absl::StatusOr ToZipWrapperCompressionMethod( return rust::CompressionMethod::from_i32(static_cast(method)); } +absl::Span ToByteSpan(absl::string_view s) { + return absl::Span(reinterpret_cast(s.data()), + s.size()); +} + +// The four Rust writer types (Buffered/Fs x seekable/stream) expose the same +// method set, so the bodies below are shared. `WriteZipFileContent` is not +// shared because it needs the friend access to `*ZipFile::zip_` that only the +// writer classes have. +template +absl::Status StartFileImpl(RustWriter& writer, absl::string_view file_name, + const rust::ZipWriterFileOptions& options) { + return FromRustResultUnit(writer.start_file(ToByteSpan(file_name), options)); +} + +template +absl::Status AddDirectoryImpl( + RustWriter& writer, absl::string_view file_name, + const rust::ZipWriterFileOptions& options) { + return FromRustResultUnit( + writer.add_directory(ToByteSpan(file_name), options)); +} + +template +absl::Status WriteDataImpl(RustWriter& writer, absl::string_view data) { + rust::VecU8 input_data = + rust::VecU8::copy_from_slice(ToByteSpan(data)); + return FromRustResultUnit(writer.write_data(input_data)); +} + +template +absl::Status WriteFileContentImpl(RustWriter& writer, absl::string_view path) { + return FromRustResultUnit(writer.write_file_content(ToByteSpan(path))); +} + +template +absl::Status SetCommentImpl(RustWriter& writer, absl::string_view comment) { + return FromRustResultUnit(writer.set_comment(ToByteSpan(comment))); +} + +template +absl::Status FlushImpl(RustWriter& writer) { + return FromRustResultUnit(writer.flush()); +} + } // namespace absl::Status ZipWriterFileOptions::SetCompressionMethod( @@ -36,8 +81,7 @@ absl::Status ZipWriterFileOptions::SetCompressionMethod( absl::StatusOr BufferedZipWriter::NewFromData( absl::string_view data, bool append) { rust::VecU8 input_data = - rust::VecU8::copy_from_slice(absl::Span( - reinterpret_cast(data.data()), data.size())); + rust::VecU8::copy_from_slice(ToByteSpan(data)); ABSL_ASSIGN_OR_RETURN( rust::BufferedZipWriter writer, FromRustBufferedZipWriter( @@ -51,25 +95,16 @@ absl::StatusOr BufferedZipWriter::Finish() { absl::Status BufferedZipWriter::StartFile(absl::string_view file_name, const ZipWriterFileOptions& options) { - return FromRustResultUnit(writer_.start_file( - absl::Span( - reinterpret_cast(file_name.data()), file_name.size()), - options.options_)); + return StartFileImpl(writer_, file_name, options.options_); } absl::Status BufferedZipWriter::AddDirectory( absl::string_view file_name, const ZipWriterFileOptions& options) { - return FromRustResultUnit(writer_.add_directory( - absl::Span( - reinterpret_cast(file_name.data()), file_name.size()), - options.options_)); + return AddDirectoryImpl(writer_, file_name, options.options_); } absl::Status BufferedZipWriter::WriteData(absl::string_view data) { - rust::VecU8 input_data = - rust::VecU8::copy_from_slice(absl::Span( - reinterpret_cast(data.data()), data.size())); - return FromRustResultUnit(writer_.write_data(input_data)); + return WriteDataImpl(writer_, data); } absl::Status BufferedZipWriter::WriteZipFileContent(ZipFile& file) { @@ -86,19 +121,27 @@ absl::Status BufferedZipWriter::WriteZipFileContent(ZipFile& file) { } absl::Status BufferedZipWriter::WriteFileContent(absl::string_view path) { - return FromRustResultUnit( - writer_.write_file_content(absl::Span( - reinterpret_cast(path.data()), path.size()))); + return WriteFileContentImpl(writer_, path); +} + +absl::Status BufferedZipWriter::SetComment(absl::string_view comment) { + return SetCommentImpl(writer_, comment); } +absl::Status BufferedZipWriter::Flush() { return FlushImpl(writer_); } + +bool BufferedZipWriter::IsSeekPossible() const { + return writer_.is_seek_possible(); +} + +bool BufferedZipWriter::IsNone() const { return writer_.is_none(); } + absl::StatusOr FsZipWriter::NewFromPath(absl::string_view path, bool append) { ABSL_ASSIGN_OR_RETURN( rust::FsZipWriter writer, - FromRustFsZipWriter(rust::FsZipWriter::new_from_path( - absl::Span( - reinterpret_cast(path.data()), path.size()), - append))); + FromRustFsZipWriter( + rust::FsZipWriter::new_from_path(ToByteSpan(path), append))); return FsZipWriter(std::move(writer)); } @@ -108,25 +151,16 @@ absl::Status FsZipWriter::Finish() { absl::Status FsZipWriter::StartFile(absl::string_view file_name, const ZipWriterFileOptions& options) { - return FromRustResultUnit(writer_.start_file( - absl::Span( - reinterpret_cast(file_name.data()), file_name.size()), - options.options_)); + return StartFileImpl(writer_, file_name, options.options_); } absl::Status FsZipWriter::AddDirectory(absl::string_view file_name, const ZipWriterFileOptions& options) { - return FromRustResultUnit(writer_.add_directory( - absl::Span( - reinterpret_cast(file_name.data()), file_name.size()), - options.options_)); + return AddDirectoryImpl(writer_, file_name, options.options_); } absl::Status FsZipWriter::WriteData(absl::string_view data) { - rust::VecU8 input_data = - rust::VecU8::copy_from_slice(absl::Span( - reinterpret_cast(data.data()), data.size())); - return FromRustResultUnit(writer_.write_data(input_data)); + return WriteDataImpl(writer_, data); } absl::Status FsZipWriter::WriteZipFileContent(ZipFile& file) { @@ -143,11 +177,19 @@ absl::Status FsZipWriter::WriteZipFileContent(ZipFile& file) { } absl::Status FsZipWriter::WriteFileContent(absl::string_view path) { - return FromRustResultUnit( - writer_.write_file_content(absl::Span( - reinterpret_cast(path.data()), path.size()))); + return WriteFileContentImpl(writer_, path); +} + +absl::Status FsZipWriter::SetComment(absl::string_view comment) { + return SetCommentImpl(writer_, comment); } +absl::Status FsZipWriter::Flush() { return FlushImpl(writer_); } + +bool FsZipWriter::IsSeekPossible() const { return writer_.is_seek_possible(); } + +bool FsZipWriter::IsNone() const { return writer_.is_none(); } + absl::StatusOr ZipWriter::FromFile(absl::string_view path, bool append) { ABSL_ASSIGN_OR_RETURN(FsZipWriter writer, @@ -207,4 +249,214 @@ absl::Status ZipWriter::WriteFileContent(absl::string_view path) { writer_); } +absl::Status ZipWriter::SetComment(absl::string_view comment) { + return std::visit([&](auto& writer) { return writer.SetComment(comment); }, + writer_); +} + +absl::Status ZipWriter::Flush() { + return std::visit([&](auto& writer) { return writer.Flush(); }, writer_); +} + +bool ZipWriter::IsSeekPossible() const { + return std::visit([&](const auto& writer) { return writer.IsSeekPossible(); }, + writer_); +} + +bool ZipWriter::IsNone() const { + return std::visit([&](const auto& writer) { return writer.IsNone(); }, + writer_); +} + +absl::StatusOr BufferedZipStreamWriter::New() { + return BufferedZipStreamWriter( + rust::BufferedZipStreamWriter::new_stream()); +} + +absl::StatusOr BufferedZipStreamWriter::Finish() { + return FromRustResultVecU8(writer_.finish()); +} + +absl::Status BufferedZipStreamWriter::StartFile( + absl::string_view file_name, const ZipWriterFileOptions& options) { + return StartFileImpl(writer_, file_name, options.options_); +} + +absl::Status BufferedZipStreamWriter::AddDirectory( + absl::string_view file_name, const ZipWriterFileOptions& options) { + return AddDirectoryImpl(writer_, file_name, options.options_); +} + +absl::Status BufferedZipStreamWriter::WriteData(absl::string_view data) { + return WriteDataImpl(writer_, data); +} + +absl::Status BufferedZipStreamWriter::WriteZipFileContent(ZipFile& file) { + return std::visit( + absl::Overload{[&](BufferedZipFile& file) { + return FromRustResultUnit( + writer_.write_buffered_zip_file_content(file.zip_)); + }, + [&](FsZipFile& file) { + return FromRustResultUnit( + writer_.write_fs_zip_file_content(file.zip_)); + }}, + file.zip_); +} + +absl::Status BufferedZipStreamWriter::WriteFileContent(absl::string_view path) { + return WriteFileContentImpl(writer_, path); +} + +absl::Status BufferedZipStreamWriter::SetComment(absl::string_view comment) { + return SetCommentImpl(writer_, comment); +} + +absl::Status BufferedZipStreamWriter::Flush() { return FlushImpl(writer_); } + +bool BufferedZipStreamWriter::IsSeekPossible() const { + return writer_.is_seek_possible(); +} + +bool BufferedZipStreamWriter::IsNone() const { return writer_.is_none(); } + +absl::StatusOr FsZipStreamWriter::NewFromPath( + absl::string_view path) { + ABSL_ASSIGN_OR_RETURN( + rust::FsZipStreamWriter writer, + FromRustFsZipStreamWriter( + rust::FsZipStreamWriter::new_from_path(ToByteSpan(path)))); + return FsZipStreamWriter(std::move(writer)); +} + +absl::Status FsZipStreamWriter::Finish() { + return FromRustResultUnit(writer_.finish()); +} + +absl::Status FsZipStreamWriter::StartFile(absl::string_view file_name, + const ZipWriterFileOptions& options) { + return StartFileImpl(writer_, file_name, options.options_); +} + +absl::Status FsZipStreamWriter::AddDirectory( + absl::string_view file_name, const ZipWriterFileOptions& options) { + return AddDirectoryImpl(writer_, file_name, options.options_); +} + +absl::Status FsZipStreamWriter::WriteData(absl::string_view data) { + return WriteDataImpl(writer_, data); +} + +absl::Status FsZipStreamWriter::WriteZipFileContent(ZipFile& file) { + return std::visit( + absl::Overload{[&](BufferedZipFile& file) { + return FromRustResultUnit( + writer_.write_buffered_zip_file_content(file.zip_)); + }, + [&](FsZipFile& file) { + return FromRustResultUnit( + writer_.write_fs_zip_file_content(file.zip_)); + }}, + file.zip_); +} + +absl::Status FsZipStreamWriter::WriteFileContent(absl::string_view path) { + return WriteFileContentImpl(writer_, path); +} + +absl::Status FsZipStreamWriter::SetComment(absl::string_view comment) { + return SetCommentImpl(writer_, comment); +} + +absl::Status FsZipStreamWriter::Flush() { return FlushImpl(writer_); } + +bool FsZipStreamWriter::IsSeekPossible() const { + return writer_.is_seek_possible(); +} + +bool FsZipStreamWriter::IsNone() const { return writer_.is_none(); } + +absl::StatusOr ZipStreamWriter::FromFile( + absl::string_view path) { + ABSL_ASSIGN_OR_RETURN(FsZipStreamWriter writer, + FsZipStreamWriter::NewFromPath(path)); + return ZipStreamWriter(std::move(writer)); +} + +absl::StatusOr ZipStreamWriter::FromBuffer() { + ABSL_ASSIGN_OR_RETURN(BufferedZipStreamWriter writer, + BufferedZipStreamWriter::New()); + return ZipStreamWriter(std::move(writer)); +} + +absl::StatusOr ZipStreamWriter::Finish() { + return std::visit( + absl::Overload{ + [](BufferedZipStreamWriter& writer) + -> absl::StatusOr { return writer.Finish(); }, + [](FsZipStreamWriter& writer) -> absl::StatusOr { + ABSL_RETURN_IF_ERROR(writer.Finish()); + // FsZipStreamWriter writes to a file on the filesystem and does + // not return an owned buffer, so return an empty wrapper. + return RustVecU8Wrapper(); + }}, + writer_); +} + +absl::Status ZipStreamWriter::StartFile(absl::string_view file_name, + const ZipWriterFileOptions& options) { + return std::visit( + [&](auto& writer) { return writer.StartFile(file_name, options); }, + writer_); +} + +absl::Status ZipStreamWriter::AddDirectory( + absl::string_view file_name, const ZipWriterFileOptions& options) { + return std::visit( + [&](auto& writer) { return writer.AddDirectory(file_name, options); }, + writer_); +} + +absl::Status ZipStreamWriter::WriteData(absl::string_view data) { + return std::visit([&](auto& writer) { return writer.WriteData(data); }, + writer_); +} + +absl::Status ZipStreamWriter::WriteZipFileContent(ZipFile& file) { + return std::visit( + [&](auto& writer) { return writer.WriteZipFileContent(file); }, writer_); +} + +absl::Status ZipStreamWriter::WriteFileContent(absl::string_view path) { + return std::visit([&](auto& writer) { return writer.WriteFileContent(path); }, + writer_); +} + +absl::Status ZipStreamWriter::SetComment(absl::string_view comment) { + return std::visit([&](auto& writer) { return writer.SetComment(comment); }, + writer_); +} + +absl::Status ZipStreamWriter::Flush() { + return std::visit([&](auto& writer) { return writer.Flush(); }, writer_); +} + +bool ZipStreamWriter::IsSeekPossible() const { + return std::visit([&](const auto& writer) { return writer.IsSeekPossible(); }, + writer_); +} + +bool ZipStreamWriter::IsNone() const { + return std::visit([&](const auto& writer) { return writer.IsNone(); }, + writer_); +} + +absl::StatusOr NewBufferedZipStreamWriter() { + return BufferedZipStreamWriter::New(); +} + +absl::StatusOr NewFsZipStreamWriter(absl::string_view path) { + return FsZipStreamWriter::NewFromPath(path); +} + } // namespace security::zip diff --git a/zip/write.h b/zip/write.h index 44775b5a..143b21b0 100644 --- a/zip/write.h +++ b/zip/write.h @@ -16,6 +16,8 @@ namespace security::zip { class BufferedZipWriter; class FsZipWriter; +class BufferedZipStreamWriter; +class FsZipStreamWriter; class ZipWriterFileOptions { public: @@ -40,8 +42,17 @@ class ZipWriterFileOptions { rust::ZipWriterFileOptions options_; friend class BufferedZipWriter; friend class FsZipWriter; + friend class BufferedZipStreamWriter; + friend class FsZipStreamWriter; }; +// Seekable zip writers. SetComment(comment) stores the archive comment that +// Finish() writes; `comment` must be at most 65535 bytes (the zip format's +// u16 limit), otherwise kInvalidArgument is returned and the previously set +// comment is left untouched. Flush() flushes the underlying sink; Finish() +// writes the central directory. IsSeekPossible() is always true. All methods +// on a moved-from or finished writer return kFailedPrecondition; IsNone() +// reports that state. class BufferedZipWriter final { public: static absl::StatusOr NewFromData(absl::string_view data, @@ -54,6 +65,10 @@ class BufferedZipWriter final { absl::Status WriteData(absl::string_view data); absl::Status WriteZipFileContent(ZipFile& file); absl::Status WriteFileContent(absl::string_view path); + absl::Status SetComment(absl::string_view comment); + absl::Status Flush(); + [[nodiscard]] bool IsSeekPossible() const; + [[nodiscard]] bool IsNone() const; private: explicit BufferedZipWriter(rust::BufferedZipWriter writer) @@ -73,6 +88,10 @@ class FsZipWriter final { absl::Status WriteData(absl::string_view data); absl::Status WriteZipFileContent(ZipFile& file); absl::Status WriteFileContent(absl::string_view path); + absl::Status SetComment(absl::string_view comment); + absl::Status Flush(); + [[nodiscard]] bool IsSeekPossible() const; + [[nodiscard]] bool IsNone() const; private: explicit FsZipWriter(rust::FsZipWriter writer) @@ -94,6 +113,10 @@ class ZipWriter final { absl::Status WriteData(absl::string_view data); absl::Status WriteZipFileContent(ZipFile& file); absl::Status WriteFileContent(absl::string_view path); + absl::Status SetComment(absl::string_view comment); + absl::Status Flush(); + [[nodiscard]] bool IsSeekPossible() const; + [[nodiscard]] bool IsNone() const; private: using BackendType = std::variant; @@ -102,6 +125,129 @@ class ZipWriter final { BackendType writer_; }; +// Streaming zip writers write zip archives sequentially without seeking. The +// contract below applies to all three classes. +// +// Construction: `New()`/`FromBuffer()`/`NewBufferedZipStreamWriter()` create +// an in-memory writer and never fail (StatusOr is kept for symmetry with the +// other factories). `NewFromPath`/`FromFile`/`NewFsZipStreamWriter` create or +// truncate the file at `path` (there is no append mode) and return +// kInvalidArgument if it cannot be opened. +// +// Writing: StartFile()/AddDirectory() begin a new entry; WriteData(), +// WriteZipFileContent() and WriteFileContent() append to the entry started by +// the last StartFile() and return kInternal if no entry has been started. +// Because the sink is not seekable, every entry is written with general +// purpose bit 3 set and its sizes/CRC in a trailing data descriptor. Such an +// archive is read back by `ZipArchive` (which uses the central directory), but +// NOT by this library's `ZipStreamReader::ReadNextFile()`, which returns +// kFailedPrecondition on data-descriptor entries; use +// `ReadNextFileWithCompressedSize()` with a size hint instead. An entry that +// may exceed 4 GiB must be started with `SetIsLargeFile(true)`: zip-rs cannot +// upgrade an in-flight entry to Zip64 without seeking back. +// +// SetComment(): Unsupported. Always returns kFailedPrecondition, because +// zip-rs finalizes a commented archive by seeking back over the comment. The +// finished archive always has an empty comment. +// +// Flush(): Flushes the underlying sink; it does not finalize anything. +// +// Finish(): Writes the central directory and consumes the writer. The buffer +// backend returns the archive bytes; the file backend returns OK (and +// `ZipStreamWriter::Finish()` returns an empty `RustVecU8Wrapper` for it). +// +// IsSeekPossible(): Always false. IsNone(): True if moved-from or finished; +// every other method then returns kFailedPrecondition. +// +// Error codes: kFailedPrecondition (moved-from/finished writer, SetComment), +// kInvalidArgument (bad path, bad compression level), kInternal (I/O or +// zip-rs error). +class BufferedZipStreamWriter final { + public: + // See the contract at the top of this section. + static absl::StatusOr New(); + absl::StatusOr Finish(); + absl::Status StartFile(absl::string_view file_name, + const ZipWriterFileOptions& options); + absl::Status AddDirectory(absl::string_view file_name, + const ZipWriterFileOptions& options); + absl::Status WriteData(absl::string_view data); + absl::Status WriteZipFileContent(ZipFile& file); + absl::Status WriteFileContent(absl::string_view path); + absl::Status SetComment(absl::string_view comment); + absl::Status Flush(); + [[nodiscard]] bool IsSeekPossible() const; + [[nodiscard]] bool IsNone() const; + + private: + explicit BufferedZipStreamWriter(rust::BufferedZipStreamWriter writer) + : writer_(std::move(writer)) {} + rust::BufferedZipStreamWriter writer_; +}; + +class FsZipStreamWriter final { + public: + // See the contract at the top of this section. + static absl::StatusOr NewFromPath(absl::string_view path); + absl::Status Finish(); + absl::Status StartFile(absl::string_view file_name, + const ZipWriterFileOptions& options); + absl::Status AddDirectory(absl::string_view file_name, + const ZipWriterFileOptions& options); + absl::Status WriteData(absl::string_view data); + absl::Status WriteZipFileContent(ZipFile& file); + absl::Status WriteFileContent(absl::string_view path); + absl::Status SetComment(absl::string_view comment); + absl::Status Flush(); + [[nodiscard]] bool IsSeekPossible() const; + [[nodiscard]] bool IsNone() const; + + private: + explicit FsZipStreamWriter(rust::FsZipStreamWriter writer) + : writer_(std::move(writer)) {} + rust::FsZipStreamWriter writer_; +}; + +// Facade over BufferedZipStreamWriter / FsZipStreamWriter; every method +// forwards to the selected backend. See the contract at the top of this +// section. +class ZipStreamWriter final { + public: + // Creates or truncates the file at `path`; kInvalidArgument if it cannot be + // opened. + static absl::StatusOr FromFile(absl::string_view path); + // Writes to an in-memory buffer that Finish() returns. Takes no input + // buffer (unlike `ZipWriter::FromBuffer`), since a stream writer cannot + // append to an existing archive. + static absl::StatusOr FromBuffer(); + // Returns the archive bytes for the buffer backend and an empty + // `RustVecU8Wrapper` for the file backend. + absl::StatusOr Finish(); + absl::Status StartFile(absl::string_view file_name, + const ZipWriterFileOptions& options); + absl::Status AddDirectory(absl::string_view file_name, + const ZipWriterFileOptions& options); + absl::Status WriteData(absl::string_view data); + absl::Status WriteZipFileContent(ZipFile& file); + absl::Status WriteFileContent(absl::string_view path); + absl::Status SetComment(absl::string_view comment); + absl::Status Flush(); + [[nodiscard]] bool IsSeekPossible() const; + [[nodiscard]] bool IsNone() const; + + private: + using BackendType = std::variant; + + explicit ZipStreamWriter(BackendType writer) : writer_(std::move(writer)) {} + BackendType writer_; +}; + +// Free function helpers mirroring rust::new_buffered_zip_stream_writer +// and rust::new_fs_zip_stream_writer; they are one-line aliases of +// BufferedZipStreamWriter::New() and FsZipStreamWriter::NewFromPath(). +absl::StatusOr NewBufferedZipStreamWriter(); +absl::StatusOr NewFsZipStreamWriter(absl::string_view path); + } // namespace security::zip #endif // SECURITY_ZIP_WRITE_H_ diff --git a/zip/zip_benchmark.cc b/zip/zip_benchmark.cc new file mode 100644 index 00000000..f2683091 --- /dev/null +++ b/zip/zip_benchmark.cc @@ -0,0 +1,234 @@ +#include +#include +#include +#include + +#include "converters.h" +#include "file.h" +#include "read.h" +#include "write.h" +#include "testing/base/public/benchmark.h" +#include "absl/log/check.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" + +namespace security::zip { +namespace { + +std::string GenerateTestData(size_t size) { + std::string data; + data.reserve(size); + for (size_t i = 0; i < size; ++i) { + data.push_back(static_cast((i * 73 + 19) % 256)); + } + return data; +} + +std::string BuildTestZip(size_t file_count, size_t file_size, + CompressionMethod method) { + absl::StatusOr writer_or = + ZipWriter::FromBuffer("", /*append=*/false); + CHECK_OK(writer_or.status()); + ZipWriter writer = *std::move(writer_or); + ZipWriterFileOptions options; + CHECK_OK(options.SetCompressionMethod(method)); + std::string payload = GenerateTestData(file_size); + for (size_t i = 0; i < file_count; ++i) { + std::string name = absl::StrCat("entry_", i, ".bin"); + CHECK_OK(writer.StartFile(name, options)); + CHECK_OK(writer.WriteData(payload)); + } + absl::StatusOr res_or = writer.Finish(); + CHECK_OK(res_or.status()); + RustVecU8Wrapper res = *std::move(res_or); + return std::string(res.data(), res.size()); +} + +// Benchmark streaming write throughput. +void BM_StreamWriter_Write(benchmark::State& state) { + const size_t file_count = static_cast(state.range(0)); + const size_t file_size = static_cast(state.range(1)); + const std::string payload = GenerateTestData(file_size); + + ZipWriterFileOptions options; + CHECK_OK(options.SetCompressionMethod(CompressionMethod::kStored)); + + for (auto _ : state) { + absl::StatusOr writer_or = ZipStreamWriter::FromBuffer(); + CHECK_OK(writer_or.status()); + ZipStreamWriter writer = *std::move(writer_or); + for (size_t i = 0; i < file_count; ++i) { + std::string name = absl::StrCat("stream_file_", i, ".bin"); + CHECK_OK(writer.StartFile(name, options)); + CHECK_OK(writer.WriteData(payload)); + } + absl::StatusOr result = writer.Finish(); + CHECK_OK(result.status()); + benchmark::DoNotOptimize(*result); + } + + state.SetBytesProcessed(state.iterations() * file_count * file_size); +} +BENCHMARK(BM_StreamWriter_Write) + ->Args({1, 1024}) + ->Args({10, 1024}) + ->Args({1, 65536}) + ->Args({10, 65536}); + +// Benchmark seekable write throughput. +void BM_SeekableWriter_Write(benchmark::State& state) { + const size_t file_count = static_cast(state.range(0)); + const size_t file_size = static_cast(state.range(1)); + const std::string payload = GenerateTestData(file_size); + + ZipWriterFileOptions options; + CHECK_OK(options.SetCompressionMethod(CompressionMethod::kStored)); + + for (auto _ : state) { + absl::StatusOr writer_or = + ZipWriter::FromBuffer("", /*append=*/false); + CHECK_OK(writer_or.status()); + ZipWriter writer = *std::move(writer_or); + for (size_t i = 0; i < file_count; ++i) { + std::string name = absl::StrCat("seek_file_", i, ".bin"); + CHECK_OK(writer.StartFile(name, options)); + CHECK_OK(writer.WriteData(payload)); + } + absl::StatusOr result = writer.Finish(); + CHECK_OK(result.status()); + benchmark::DoNotOptimize(*result); + } + + state.SetBytesProcessed(state.iterations() * file_count * file_size); +} +BENCHMARK(BM_SeekableWriter_Write) + ->Args({1, 1024}) + ->Args({10, 1024}) + ->Args({1, 65536}) + ->Args({10, 65536}); + +// Benchmark streaming write with Deflate compression. +void BM_StreamWriter_Deflated(benchmark::State& state) { + const size_t file_size = static_cast(state.range(0)); + const std::string payload = GenerateTestData(file_size); + + ZipWriterFileOptions options; + CHECK_OK(options.SetCompressionMethod(CompressionMethod::kDeflated)); + + for (auto _ : state) { + absl::StatusOr writer_or = ZipStreamWriter::FromBuffer(); + CHECK_OK(writer_or.status()); + ZipStreamWriter writer = *std::move(writer_or); + CHECK_OK(writer.StartFile("deflated.bin", options)); + CHECK_OK(writer.WriteData(payload)); + absl::StatusOr result = writer.Finish(); + CHECK_OK(result.status()); + benchmark::DoNotOptimize(*result); + } + + state.SetBytesProcessed(state.iterations() * file_size); +} +BENCHMARK(BM_StreamWriter_Deflated)->Args({1024})->Args({65536}); + +// Benchmark streaming read throughput. +void BM_StreamReader_SequentialRead(benchmark::State& state) { + const size_t file_count = static_cast(state.range(0)); + const size_t file_size = static_cast(state.range(1)); + const std::string zip_data = + BuildTestZip(file_count, file_size, CompressionMethod::kStored); + + for (auto _ : state) { + absl::StatusOr reader_or = + ZipStreamReader::FromBuffer(zip_data); + CHECK_OK(reader_or.status()); + ZipStreamReader reader = *std::move(reader_or); + for (size_t i = 0; i < file_count; ++i) { + absl::StatusOr> file_opt_or = + reader.ReadNextFile(); + CHECK_OK(file_opt_or.status()); + std::optional file_opt = *std::move(file_opt_or); + if (!file_opt.has_value()) break; + absl::StatusOr content_or = file_opt->GetFileData(); + CHECK_OK(content_or.status()); + RustVecU8Wrapper content = *std::move(content_or); + benchmark::DoNotOptimize(content); + } + } + + state.SetBytesProcessed(state.iterations() * file_count * file_size); +} +BENCHMARK(BM_StreamReader_SequentialRead) + ->Args({1, 1024}) + ->Args({10, 1024}) + ->Args({1, 65536}) + ->Args({10, 65536}); + +// Benchmark seekable archive read throughput. +void BM_ArchiveReader_GetFileByIndex(benchmark::State& state) { + const size_t file_count = static_cast(state.range(0)); + const size_t file_size = static_cast(state.range(1)); + const std::string zip_data = + BuildTestZip(file_count, file_size, CompressionMethod::kStored); + + for (auto _ : state) { + absl::StatusOr archive_or = ZipArchive::FromBuffer(zip_data); + CHECK_OK(archive_or.status()); + ZipArchive archive = *std::move(archive_or); + for (size_t i = 0; i < file_count; ++i) { + absl::StatusOr file_or = archive.GetFileByIndex(i); + CHECK_OK(file_or.status()); + ZipFile file = *std::move(file_or); + absl::StatusOr content_or = file.GetFileData(); + CHECK_OK(content_or.status()); + RustVecU8Wrapper content = *std::move(content_or); + benchmark::DoNotOptimize(content); + } + } + + state.SetBytesProcessed(state.iterations() * file_count * file_size); +} +BENCHMARK(BM_ArchiveReader_GetFileByIndex) + ->Args({1, 1024}) + ->Args({10, 1024}) + ->Args({1, 65536}) + ->Args({10, 65536}); + +// Benchmark streaming chunked reading. +void BM_StreamReader_ChunkedRead(benchmark::State& state) { + const size_t chunk_size = static_cast(state.range(0)); + constexpr size_t kFileSize = 65536; + const std::string zip_data = + BuildTestZip(1, kFileSize, CompressionMethod::kStored); + + for (auto _ : state) { + absl::StatusOr reader_or = + ZipStreamReader::FromBuffer(zip_data); + CHECK_OK(reader_or.status()); + ZipStreamReader reader = *std::move(reader_or); + absl::StatusOr> file_opt_or = reader.ReadNextFile(); + CHECK_OK(file_opt_or.status()); + std::optional file_opt = *std::move(file_opt_or); + if (file_opt.has_value()) { + size_t total_read = 0; + while (total_read < kFileSize) { + absl::StatusOr chunk_or = + file_opt->ReadBytes(chunk_size); + CHECK_OK(chunk_or.status()); + RustVecU8Wrapper chunk = *std::move(chunk_or); + if (chunk.empty()) break; + total_read += chunk.size(); + benchmark::DoNotOptimize(chunk); + } + } + } + + state.SetBytesProcessed(state.iterations() * kFileSize); +} +BENCHMARK(BM_StreamReader_ChunkedRead) + ->Args({1024}) + ->Args({4096}) + ->Args({16384}); + +} // namespace +} // namespace security::zip