Read xlsx, pptx, ods and odp

Package.swift

@@ -44,7 +44,7 @@ let portableTargets: [Target] = [
         .define("STBI_WINDOWS_UTF8", .when(platforms: [.windows])),
         .define("STBIW_WINDOWS_UTF8", .when(platforms: [.windows])), .unsafeFlags(["-w"]),
     ]),
-    // Zip reading, for docx and odt.
+    // Zip reading, for docx and odt (spreadsheets and presentations too: OfficeText).
     .target(name: "CMiniz", path: "Vendor/miniz", cSettings: [.unsafeFlags(["-w"])]),
     // PDF text and page rendering (Chromium's PDFium, prebuilt).
     .systemLibrary(name: "CPDFium", path: "Sources/CPDFium"),

README.md

@@ -109,6 +109,8 @@ Only the folders you choose:
   else is in (up to 5,000 pages a file)
 - documents: txt, md, rtf, docx, odt, html, and on a Mac doc, long ones the same way. Text that's mostly numbers,
   like a data export, is matched by its exact text only
+- spreadsheets and presentations: xlsx, ods, pptx and odp, read for their text: each sheet's name and rows, each
+  slide's text and speaker notes
 - audio of any length: the model listens to 30 seconds at a time, so DigUp takes overlapping 30-second windows and a
   match points to its moment
 - video of any length: a frame every few seconds, one per shot, plus the soundtrack the same way as audio

Sources/DigUpKit/Extractors.swift

@@ -47,12 +47,14 @@ enum PDFExtractor {
 }
 
 enum DocExtractor {
-    /// Plain text from txt/md directly, HTML by stripping tags, and rtf/docx/doc/odt through `richText` (the Cocoa
-    /// text system on macOS, the files' own XML elsewhere).
+    /// Plain text from txt/md directly, HTML by stripping tags, rtf/docx/doc/odt through `richText` (the Cocoa text
+    /// system on macOS, the files' own XML elsewhere), and spreadsheets and presentations from their XML everywhere
+    /// (`OfficeText`).
     static func text(of url: URL) throws -> String {
         let ext = url.pathExtension.lowercased()
         switch ext {
         case "docx", "doc", "odt", "rtf": return try richText(of: url, type: ext)
+        case "xlsx", "pptx", "ods", "odp": return try OfficeText.text(of: url, type: ext)
         case "html", "htm": return plainText(fromHTML: try readText(url))
         default: return try readText(url)
         }

Sources/DigUpKit/Platform/Apple/Zip+Apple.swift

@@ -0,0 +1,82 @@
+// Zip files on macOS, for the spreadsheets and presentations the Cocoa text system doesn't read (`OfficeText`): the
+// central directory read here, deflated entries inflated by the Compression framework. Platform/Portable has the same
+// name over miniz.
+#if canImport(AppKit)
+import Compression
+import Foundation
+
+final class ZipArchive {
+    private struct Entry {
+        let method: Int
+        let compressedSize: Int
+        let size: Int
+        let headerOffset: Int
+    }
+
+    private let file: Data
+    private var entries: [String: Entry] = [:]
+
+    init(_ url: URL) throws {
+        file = try Data(contentsOf: url, options: .alwaysMapped)
+        // The end of the central directory: the last 22 bytes, unless a comment (up to 64 KB) follows it.
+        var end = file.count - 22
+        while end >= max(0, file.count - 22 - 0xFFFF), (try? number(end, 4)) != 0x0605_4B50 { end -= 1 }
+        guard end >= 0, end >= max(0, file.count - 22 - 0xFFFF) else { throw ExtractError.unreadable("not a zip file") }
+        let count = try number(end + 10, 2)
+        var offset = try number(end + 16, 4)
+        guard offset != 0xFFFF_FFFF else { throw ExtractError.unreadable("zip64 files aren't read") }
+        for _ in 0..<count {
+            guard try number(offset, 4) == 0x0201_4B50 else { throw ExtractError.unreadable("damaged zip file") }
+            let nameLength = try number(offset + 28, 2)
+            let skip = try number(offset + 30, 2) + number(offset + 32, 2)   // extra field, comment
+            guard offset + 46 + nameLength <= file.count else { throw ExtractError.unreadable("damaged zip file") }
+            let name = String(decoding: file[(offset + 46)..<(offset + 46 + nameLength)], as: UTF8.self)
+            if try number(offset + 8, 2) & 1 == 0 {   // encrypted entries are left out
+                entries[name] = Entry(method: try number(offset + 10, 2), compressedSize: try number(offset + 20, 4),
+                                      size: try number(offset + 24, 4), headerOffset: try number(offset + 42, 4))
+            }
+            offset += 46 + nameLength + skip
+        }
+    }
+
+    /// The entry's bytes, or nil when the zip has no such entry.
+    func data(_ name: String) throws -> [UInt8]? {
+        guard let entry = entries[name] else { return nil }
+        guard entry.size <= Self.maxEntrySize else {
+            throw ExtractError.unreadable("\(name) is \(entry.size >> 20) MB, more than is read")
+        }
+        let header = entry.headerOffset
+        guard try number(header, 4) == 0x0403_4B50 else { throw ExtractError.unreadable("damaged zip file") }
+        let start = header + 30 + (try number(header + 26, 2)) + (try number(header + 28, 2))
+        guard start + entry.compressedSize <= file.count else { throw ExtractError.unreadable("damaged zip file") }
+        let stored = file[start..<(start + entry.compressedSize)]
+        switch entry.method {
+        case 0:
+            return Array(stored)
+        case 8:
+            guard entry.size > 0 else { return [] }
+            var out = [UInt8](repeating: 0, count: entry.size)
+            // COMPRESSION_ZLIB is raw deflate, as zip stores it.
+            let written = stored.withUnsafeBytes { source in
+                out.withUnsafeMutableBytes { target in
+                    compression_decode_buffer(target.bindMemory(to: UInt8.self).baseAddress!, entry.size,
+                                              source.bindMemory(to: UInt8.self).baseAddress!, entry.compressedSize,
+                                              nil, COMPRESSION_ZLIB)
+                }
+            }
+            guard written == entry.size else { throw ExtractError.unreadable("can't inflate \(name)") }
+            return out
+        default:
+            throw ExtractError.unreadable("unknown compression in \(name)")
+        }
+    }
+
+    /// A little-endian number of `size` bytes at `offset`.
+    private func number(_ offset: Int, _ size: Int) throws -> Int {
+        guard offset >= 0, offset + size <= file.count else { throw ExtractError.unreadable("damaged zip file") }
+        var value = 0
+        for index in (offset..<(offset + size)).reversed() { value = value << 8 | Int(file[file.startIndex + index]) }
+        return value
+    }
+}
+#endif

Sources/DigUpKit/Platform/Portable/Docs+Portable.swift

@@ -35,43 +35,60 @@ extension DocExtractor {
     }
 }
 
-/// One entry of a zip file, through miniz (reading through `FileReader`, so any file name opens).
+/// One entry of a zip file, as text.
 enum Zip {
-    /// The most an entry may declare. The indexer keeps at most 5,000 chunks of a document (about 9 M characters), so
-    /// 64 MB of XML is already far more than it can use.
-    static let maxEntryBytes: UInt64 = 64 << 20
+    /// The most an entry may declare (`ZipArchive.maxEntrySize`). The indexer keeps at most 5,000 chunks of a document
+    /// (about 9 M characters), so 64 MB of XML is already far more than it can use.
+    static let maxEntryBytes = UInt64(ZipArchive.maxEntrySize)
 
     static func entry(_ name: String, in url: URL) throws -> String {
-        let reader = try FileReader(url)
-        var zip = mz_zip_archive()
-        zip.m_pIO_opaque = reader.opaque
-        zip.m_pRead = { opaque, offset, buffer, count in
+        String(decoding: try ZipArchive(url).required(name), as: UTF8.self)
+    }
+}
+
+/// A zip file's entries, through miniz (reading through `FileReader`, so any file name opens). Platform/Apple has the
+/// same name.
+final class ZipArchive {
+    private let reader: FileReader
+    private let zip = UnsafeMutablePointer<mz_zip_archive>.allocate(capacity: 1)
+    private var open = false
+
+    init(_ url: URL) throws {
+        reader = try FileReader(url)
+        zip.initialize(to: mz_zip_archive())
+        zip.pointee.m_pIO_opaque = reader.opaque
+        zip.pointee.m_pRead = { opaque, offset, buffer, count in
             guard let buffer else { return 0 }
             return FileReader.from(opaque).read(at: offset, into: buffer, count: count)
         }
-        return try withExtendedLifetime(reader) {   // miniz reads through it until the end
-            guard mz_zip_reader_init(&zip, reader.size, 0) != 0 else {
-                throw ExtractError.unreadable("not a zip file")
-            }
-            defer { mz_zip_reader_end(&zip) }
-            // The entry's declared size first, before anything is allocated for it: a zip of a few KB can declare
-            // gigabytes, and extracting allocates the declared size whole.
-            let index = mz_zip_reader_locate_file(&zip, name, nil, 0)
-            guard index >= 0 else { throw ExtractError.unreadable("no \(name) inside") }
-            var stat = mz_zip_archive_file_stat()
-            guard mz_zip_reader_file_stat(&zip, UInt32(index), &stat) != 0 else {
-                throw ExtractError.unreadable("can't read \(name)")
-            }
-            guard stat.m_uncomp_size <= maxEntryBytes else {
-                throw ExtractError.unreadable("\(name) is \(stat.m_uncomp_size >> 20) MB, more than is read")
-            }
-            var size = 0
-            guard let bytes = mz_zip_reader_extract_to_heap(&zip, UInt32(index), &size, 0) else {
-                throw ExtractError.unreadable("can't read \(name)")
-            }
-            defer { mz_free(bytes) }
-            return String(decoding: UnsafeRawBufferPointer(start: bytes, count: size), as: UTF8.self)
+        guard mz_zip_reader_init(zip, reader.size, 0) != 0 else { throw ExtractError.unreadable("not a zip file") }
+        open = true
+    }
+
+    deinit {
+        if open { mz_zip_reader_end(zip) }   // miniz reads through `reader` until here
+        zip.deinitialize(count: 1)
+        zip.deallocate()
+    }
+
+    /// The entry's bytes, or nil when the zip has no such entry. Its declared size is checked first, before anything is
+    /// allocated for it: a zip of a few KB can declare gigabytes, and extracting allocates the declared size whole.
+    func data(_ name: String) throws -> [UInt8]? {
+        let index = mz_zip_reader_locate_file(zip, name, nil, 0)
+        guard index >= 0 else { return nil }
+        var stat = mz_zip_archive_file_stat()
+        guard mz_zip_reader_file_stat(zip, mz_uint(index), &stat) != 0 else {
+            throw ExtractError.unreadable("can't read \(name)")
+        }
+        guard stat.m_uncomp_size <= UInt64(Self.maxEntrySize) else {
+            throw ExtractError.unreadable("\(name) is \(stat.m_uncomp_size >> 20) MB, more than is read")
+        }
+        var size = 0
+        guard let bytes = mz_zip_reader_extract_to_heap(zip, mz_uint(index), &size, 0) else {
+            throw ExtractError.unreadable("can't read \(name)")
         }
+        defer { mz_free(bytes) }
+        return Array(UnsafeRawBufferPointer(start: bytes, count: size).bindMemory(to: UInt8.self))
     }
 }
 

Sources/DigUpKit/Rules.swift

@@ -21,7 +21,9 @@ public enum Rules {
     ]
     static let projectBundleExtensions: Set<String> = ["xcodeproj", "xcworkspace"]
 
-    static let docExtensions: Set<String> = ["txt", "text", "md", "markdown", "rtf", "docx", "doc", "odt", "html", "htm"]
+    static let docExtensions: Set<String> = [
+        "txt", "text", "md", "markdown", "rtf", "docx", "doc", "odt", "html", "htm", "xlsx", "pptx", "ods", "odp",
+    ]
 
     /// Source code and data files: off by default (your editor and ripgrep search them better, and they flood an index).
     static let codeExtensions: Set<String> = [