diff --git a/.gitignore b/.gitignore index e21ff61c..cdf85950 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,6 @@ shell/src-tauri/gen/ # What scripts/package-*.sh gather for the packages shell/src-tauri/helper/ shell/src-tauri/THIRD-PARTY-NOTICES.txt + +# The PST reader (pst/): its Cargo build +pst/target/ diff --git a/README.md b/README.md index 80528aa1..040d0c71 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,9 @@ Only the folders you choose: - audio of any length: the model listens to 30 seconds at a time, so DigUp takes overlapping 30-second windows and a match points to its moment - video of any length: a frame every few seconds, one per shot, plus the soundtrack the same way as audio +- on Windows, Outlook mail kept in .pst files: each mail by its subject, who sent it, its text and its attachments' + text, and Enter opens it in Outlook. A PST that Outlook has open can't be read; its new mail comes in once Outlook + is closed Outside the folders you pick for code search, it never reads code, and inside a code project (a folder with `.git`, `package.json` and the like) it reads only screenshots. It never reads keys, certificates, password files, hidden @@ -197,7 +200,7 @@ also finds them by what they show or say, and takes you to the page or moment. T **What's different on Windows and Linux?** The search is the same: the same model, the same results. Two things are missing for now. The text inside screenshots isn't read yet, so a screenshot is found by what it shows rather than by an exact code typed into the search. And doc files from old Word versions, RAW photos and SVG pictures are only read -on a Mac. +on a Mac. The other way round, Outlook mail in .pst files is only read on Windows, where classic Outlook opens it. **Why is the download 865 MB?** That's the model: the text part (310 MB) and the image and audio encoders (555 MB). The app itself is about 20 MB on a Mac, and on Windows and Linux 40 to 140 MB depending on the package, since those diff --git a/Sources/DigUpKit/Estimator.swift b/Sources/DigUpKit/Estimator.swift index 4a92cef1..839faab2 100644 --- a/Sources/DigUpKit/Estimator.swift +++ b/Sources/DigUpKit/Estimator.swift @@ -103,6 +103,11 @@ public enum Estimator { let seconds = duration(of: file.path) estimate.videoSeconds += seconds estimate.seconds += seconds * costs.videoSecond + case .mailbox, .mail: + // A PST's mails aren't counted before it's listed: about one per 30 KB, two chunks each. + let chunks = file.kind == .mail ? 2 : Int(max(2, file.size / 15_000)) + estimate.docChunks += chunks + estimate.seconds += Double(chunks) * costs.docChunk case .code: // A notebook's file is mostly its outputs, which aren't read: a guess at its cells' share. let characters = file.path.lowercased().hasSuffix(".ipynb") ? file.size / 8 : file.size diff --git a/Sources/DigUpKit/IndexStore.swift b/Sources/DigUpKit/IndexStore.swift index 649806f4..d569e5ee 100644 --- a/Sources/DigUpKit/IndexStore.swift +++ b/Sources/DigUpKit/IndexStore.swift @@ -377,10 +377,71 @@ public final class IndexStore { } public func finish(_ id: Int64, state: String, detail: String? = nil, info: [String: String] = [:]) throws { - let json = info.isEmpty ? nil : (try? JSONSerialization.data(withJSONObject: info, options: [.sortedKeys])) - .flatMap { String(data: $0, encoding: .utf8) } try db.run("UPDATE files SET state = ?, detail = ?, info = ?, indexed_at = ? WHERE id = ?", - [.text(state), .optional(detail), .optional(json), .double(Date().timeIntervalSince1970), .int(id)]) + [.text(state), .optional(detail), .optional(Self.json(info)), .double(Date().timeIntervalSince1970), + .int(id)]) + } + + static func json(_ info: [String: String]) -> String? { + info.isEmpty ? nil : (try? JSONSerialization.data(withJSONObject: info, options: [.sortedKeys])) + .flatMap { String(data: $0, encoding: .utf8) } + } + + // MARK: Mail + + /// Adds a mail of a PST (pending), named by its subject and its folder in the PST rather than by its path, and with + /// what results show of it (`Mail.Listed`) before it's read. + @discardableResult + func insertMail(_ file: Candidate, subject: String, folder: String, info: [String: String]) throws -> Int64 { + try db.run("INSERT INTO files(path, kind, size, mtime, inode, device, info) VALUES(?, ?, ?, ?, ?, ?, ?)", + [.text(file.path), .text(file.kind.rawValue), .int(file.size), .double(file.mtime), + .int(Int64(bitPattern: file.inode)), .int(Int64(bitPattern: file.device)), .optional(Self.json(info))]) + let id = db.lastInsertID + try addNameRow(file: id, name: subject, folder: folder) + return id + } + + /// A mail that changed in its PST, or waited for it: read again, under its subject as it is now. + func requeueMail(_ id: Int64, as file: Candidate, subject: String, folder: String, info: [String: String]) throws { + try deleteSegments(of: id, keepingName: false) + try addNameRow(file: id, name: subject, folder: folder) + try db.run(""" + UPDATE files SET size = ?, mtime = ?, state = 'pending', detail = NULL, info = ?, indexed_at = NULL + WHERE id = ? + """, [.int(file.size), .double(file.mtime), .optional(Self.json(info)), .int(id)]) + } + + /// A PST that was listed: its row takes the file's size and date, which tell the next sync whether to list it again. + func restat(_ id: Int64, as file: Candidate) throws { + try db.run("UPDATE files SET size = ?, mtime = ?, inode = ?, device = ? WHERE id = ?", + [.int(file.size), .double(file.mtime), .int(Int64(bitPattern: file.inode)), + .int(Int64(bitPattern: file.device)), .int(id)]) + } + + /// The mails of the PST at `mailbox`. + func mails(in mailbox: String) throws -> [FileRecord] { + var out: [FileRecord] = [] + let prefix = mailbox + Mail.separator + try db.query("SELECT \(Self.fileColumns) FROM files WHERE path >= ? AND path < ?", + [.text(prefix), .text(prefix + "\u{10FFFF}")]) { out.append(Self.record($0)) } + return out + } + + /// A PST moved or renamed: its mails' paths follow, and nothing is read again. + func moveMails(from old: String, to new: String) throws { + let prefix = old + Mail.separator + // substr counts characters (code points), not bytes. + try db.run("UPDATE files SET path = ? || substr(path, ?) WHERE path >= ? AND path < ?", + [.text(new), .int(Int64(old.unicodeScalars.count + 1)), .text(prefix), .text(prefix + "\u{10FFFF}")]) + } + + /// Mails of the PST at `mailbox` that wait for Outlook to let go of it. + func hasWaitingMail(in mailbox: String) throws -> Bool { + var found = false + let prefix = mailbox + Mail.separator + try db.query("SELECT 1 FROM files WHERE path >= ? AND path < ? AND state = 'skipped' AND detail = ? LIMIT 1", + [.text(prefix), .text(prefix + "\u{10FFFF}"), .text(Mail.waiting)]) { _ in found = true } + return found } /// Files read once more because DigUp reads them better now, each kind once per index (the schema stays as it was, @@ -444,10 +505,13 @@ public final class IndexStore { } private func addNameRow(file: Int64, path: String) throws { + try addNameRow(file: file, name: Self.nameText(path), folder: Self.folderText(path)) + } + + private func addNameRow(file: Int64, name: String, folder: String) throws { try db.run("INSERT INTO segments(file, kind, modality) VALUES(?, 'name', 'none')", [.int(file)]) try db.run("INSERT INTO fts(rowid, name, body) VALUES(?, ?, ?)", - [.int(db.lastInsertID), .text(SearchText.normalized(Self.nameText(path))), - .text(SearchText.normalized(Self.folderText(path)))]) + [.int(db.lastInsertID), .text(SearchText.normalized(name)), .text(SearchText.normalized(folder))]) } private func deleteSegments(of file: Int64, keepingName: Bool) throws { diff --git a/Sources/DigUpKit/Indexer.swift b/Sources/DigUpKit/Indexer.swift index 8c5bb405..6555d858 100644 --- a/Sources/DigUpKit/Indexer.swift +++ b/Sources/DigUpKit/Indexer.swift @@ -94,14 +94,29 @@ public final class Indexer { var byInode: [String: FileRecord] = [:] for file in existing { byPath[file.path] = file - byInode["\(file.device):\(file.inode)"] = file + if file.kind != .mail { byInode["\(file.device):\(file.inode)"] = file } // a mail's "inode" is its node id } let seen = Set(crawl.candidates.map(\.path)) var movedFrom = Set() var summary = SyncSummary(crawl: crawl) + // PSTs to list: new ones, changed ones, and those waiting for Outlook to let go (`Mail.waiting`). Listed before + // the transaction: a big PST's list takes seconds. A locked one fails at once, without reading anything. + var listings: [String: Result<[Mail.Listed], Error>] = [:] + for file in crawl.candidates where file.kind == .mailbox { + let row = byPath[file.path] + let moved = row == nil && byInode["\(file.device):\(file.inode)"].map { + !seen.contains($0.path) && $0.size == file.size && abs($0.mtime - file.mtime) < 0.001 + } == true + let changed = row.map { $0.size != file.size || abs($0.mtime - file.mtime) > 0.001 || $0.state == "skipped" } + if !moved, try changed ?? true || store.hasWaitingMail(in: file.path) { + listings[file.path] = Result { try Mail.list(file.path) } + } + } try store.db.transaction { for file in crawl.candidates { - if let row = byPath[file.path] { + if let row = byPath[file.path], row.kind == .mailbox, file.kind == .mailbox { + try syncMailbox(file, row: row, listing: listings[file.path], summary: &summary) + } else if let row = byPath[file.path] { if row.size != file.size || abs(row.mtime - file.mtime) > 0.001 || row.kind != file.kind { try store.requeue(row.id, as: file) summary.changed += 1 @@ -112,8 +127,11 @@ public final class Indexer { } else if let row = byInode["\(file.device):\(file.inode)"], !seen.contains(row.path), !movedFrom.contains(row.path), row.size == file.size, abs(row.mtime - file.mtime) < 0.001 { try store.move(row.id, to: file.path) + if row.kind == .mailbox { try store.moveMails(from: row.path, to: file.path) } movedFrom.insert(row.path) summary.moved += 1 + } else if file.kind == .mailbox { + try syncMailbox(file, row: nil, listing: listings[file.path], summary: &summary) } else { summary.queued.append(try store.insert(file)) summary.added += 1 @@ -123,8 +141,14 @@ public final class Indexer { // keep its rows rather than take that for "deleted". let vanished = roots.map { $0.standardizedFileURL.path } .filter { !FileManager.default.fileExists(atPath: $0) }.map { $0.hasSuffix("/") ? $0 : $0 + "/" } + let mailboxes = Set(crawl.candidates.filter { $0.kind == .mailbox }.map(\.path)) for row in existing where !seen.contains(row.path) && !movedFrom.contains(row.path) && !vanished.contains(where: { row.path.hasPrefix($0) }) { + // A mail goes with its PST: its PST's sync (`syncMailbox`) or move took care of it. + if row.kind == .mail, let mailbox = Mail.split(row.path)?.mailbox, + mailboxes.contains(mailbox) || movedFrom.contains(mailbox) { + continue + } try store.delete(row.id) summary.removed += 1 } @@ -132,6 +156,63 @@ public final class Indexer { return summary } + /// A PST in a sync. Listed (it's new, changed, or its mails wait for Outlook): new mails are queued, changed ones + /// and waiting ones queued again, gone ones removed. Not listed: unchanged. Locked by Outlook: it's left as it was, + /// waiting, and the next sync that can read it lists it. + private func syncMailbox(_ file: Candidate, row: FileRecord?, listing: Result<[Mail.Listed], Error>?, + summary: inout SyncSummary) throws { + guard let listing else { + summary.unchanged += 1 + return + } + var id = row?.id ?? 0 + if row == nil { + // No size or date until it's been listed: until then every sync tries again. + id = try store.insert(Candidate(path: file.path, kind: .mailbox, size: 0, mtime: 0, inode: file.inode, + device: file.device)) + summary.added += 1 + } + switch listing { + case .failure(Mail.Failure.locked): + try store.finish(id, state: "skipped", detail: Mail.waiting, info: store.info(of: id)) + case .failure(let error): + try store.restat(id, as: file) // not again until it changes + try store.finish(id, state: "failed", detail: "\(error)") + case .success(let mails): + try store.restat(id, as: file) + let name = ((file.path as NSString).lastPathComponent as NSString).deletingPathExtension + var known: [String: FileRecord] = [:] + for mail in try store.mails(in: file.path) { known[mail.path] = mail } + for mail in mails { + let path = Mail.path(mailbox: file.path, node: mail.node) + let candidate = Candidate(path: path, kind: .mail, size: 0, mtime: mail.modified, + inode: UInt64(mail.node), device: 0) + let info = ["subject": mail.subject, "from": mail.from, "date": mail.date ?? "", "folder": mail.folder] + .filter { !$0.value.isEmpty } + // Keyword search finds a mail by its subject, as a file by its name, and by its PST and folders. + let folder = ([name] + mail.folder.split(separator: "/").map(String.init)).joined(separator: " ") + if let row = known.removeValue(forKey: path) { + if abs(row.mtime - mail.modified) > 0.001 || row.state == "skipped" { + try store.requeueMail(row.id, as: candidate, subject: mail.subject, folder: folder, info: info) + summary.changed += 1 + summary.queued.append(row.id) + } else { + summary.unchanged += 1 + } + } else { + summary.queued.append(try store.insertMail(candidate, subject: mail.subject, folder: folder, + info: info)) + summary.added += 1 + } + } + for gone in known.values { + try store.delete(gone.id) + summary.removed += 1 + } + try store.finish(id, state: "done", info: ["mails": "\(mails.count)"]) + } + } + // MARK: Run /// Processes pending files, screenshots first, newest first (`only`: just these files). Long PDFs and documents get @@ -261,6 +342,9 @@ public final class Indexer { case .audio(_, let duration): measured.add("audioSecond", units: duration, seconds: seconds) case .video(_, _, let duration): measured.add("videoSecond", units: duration, seconds: seconds) case .code(let file): measured.add("codeStretch", units: Double(file.pieces.count), seconds: seconds) + case .mails(let mails): + let worded = mails.reduce(0) { $0 + $1.chunks.filter(SearchText.hasWords).count } + if worded > 0 { measured.add("docChunk", units: Double(worded), seconds: seconds) } case .nothing: break } } @@ -272,7 +356,7 @@ public final class Indexer { var step: Range? } - /// Images and screenshots go to the model 8 at a time; other files one by one. + /// Images and screenshots go to the model 8 at a time, mails of one PST too; other files one by one. static func batches(_ queue: [FileRecord]) -> [[FileRecord]] { var batches: [[FileRecord]] = [] var index = 0 @@ -283,6 +367,13 @@ public final class Indexer { queue[index + batch.count].kind == batch[0].kind { batch.append(queue[index + batch.count]) } + } else if batch[0].kind == .mail { + // Mails of one PST, 8 at a time: the tool opens the PST once for them. + let mailbox = Mail.split(batch[0].path)?.mailbox + while index + batch.count < queue.count, batch.count < 8, queue[index + batch.count].kind == .mail, + Mail.split(queue[index + batch.count].path)?.mailbox == mailbox { + batch.append(queue[index + batch.count]) + } } batches.append(batch) index += batch.count @@ -348,6 +439,15 @@ public final class Indexer { case audio(pieces: [MediaPiece], duration: Double) case video(frames: [MediaPiece], sound: [MediaPiece], duration: Double) case code(CodeExtractor.File) + case mails([ReadyMail]) + } + + /// A mail read from its PST: its text in chunks, and what results show of it. + struct ReadyMail: Sendable { + let slot: Int + let subject: String + let chunks: [String] + let info: [String: String] } struct ReadyImage: Sendable { @@ -462,6 +562,12 @@ public final class Indexer { prepared.payload = .video(frames: frames, sound: sound, duration: duration) case .code: prepared.payload = .code(try CodeExtractor.read(url, options: options)) + case .mailbox: + // Its mails are what's embedded; it's listed by the sync (here after the index was emptied for a new + // model). + prepared.outcomes[0] = Outcome(state: "done", detail: nil) + case .mail: + prepared.payload = .mails(try readMails(batch, options: options, into: folder, outcomes: &prepared.outcomes)) } } catch let error as ExtractError { if case .leftOut = error { @@ -481,6 +587,40 @@ public final class Indexer { return prepared } + /// Mails of one PST, read in one go. Those the PST no longer has fail; all of them wait while Outlook has it open. + private static func readMails(_ batch: [FileRecord], options: IndexOptions, into folder: URL, + outcomes: inout [Int: Outcome]) throws -> [ReadyMail] { + guard let mailbox = Mail.split(batch[0].path)?.mailbox else { throw ExtractError.unreadable("not a mail") } + startReading(mailbox) + let nodes = batch.map { Mail.split($0.path)?.node ?? 0 } + let read: [UInt32: Mail.Content] + do { + read = try Mail.read(mailbox, nodes: nodes, attachmentsTo: folder) + } catch Mail.Failure.locked { + for slot in batch.indices { outcomes[slot] = Outcome(state: "skipped", detail: Mail.waiting) } + return [] + } + var ready: [ReadyMail] = [] + for (slot, node) in nodes.enumerated() { + guard let mail = read[node] else { + outcomes[slot] = Outcome(state: "failed", detail: "not in the PST any more") + continue + } + // A mail is read in one step: a long one keeps its first `docStep` chunks (~64,000 characters), its + // attachments' text included. + let text = Mail.text(of: mail) + let chunks = DocExtractor.chunks(text, limit: options.docStep) + var info = ["subject": mail.subject, "from": mail.from, "date": mail.date ?? "", + "to": mail.recipients.filter { $0.kind == "to" }.map(\.name).joined(separator: ", "), + "attachments": mail.attachments.map(\.name).filter { !$0.isEmpty }.joined(separator: ", ")] + .filter { !$0.value.isEmpty } + info["chunks"] = "\(chunks.count)" + info["chars"] = "\(text.count)" + ready.append(ReadyMail(slot: slot, subject: mail.subject, chunks: chunks, info: info)) + } + return ready + } + // MARK: Commit (the indexing thread: the model and the database) private func commit(_ prepared: Prepared) throws -> [(String, String?)] { @@ -491,7 +631,9 @@ public final class Indexer { } var results = [(String, String?)](repeating: ("failed", nil), count: batch.count) for (slot, outcome) in prepared.outcomes { - try store.finish(batch[slot].id, state: outcome.state, detail: outcome.detail) + // A mail or a PST keeps what results show of it (its subject, sender and date) whatever its reading did. + let info = [.mail, .mailbox].contains(batch[slot].kind) ? try store.info(of: batch[slot].id) : [:] + try store.finish(batch[slot].id, state: outcome.state, detail: outcome.detail, info: info) results[slot] = (outcome.state, outcome.detail) } do { @@ -511,6 +653,8 @@ public final class Indexer { results[0] = try video(batch[0], frames: frames, sound: sound, duration: duration) case .code(let code): results[0] = try self.code(batch[0], code) + case .mails(let ready): + try mails(batch, ready: ready, results: &results) } } catch let error as WorkerError { throw error @@ -537,6 +681,35 @@ public final class Indexer { return ("done", "stopped before \(unit) \(place + 1): \(problem)") } + /// Mails of one PST: their chunks embedded together (the subject as their title), as a document's are. + private func mails(_ batch: [FileRecord], ready: [ReadyMail], results: inout [(String, String?)]) throws { + guard !ready.isEmpty else { return } + var texts: [String] = [], places: [(mail: Int, chunk: Int)] = [] + for (index, mail) in ready.enumerated() { + // Text that's mostly digits gets keyword search only. + for (chunk, text) in mail.chunks.enumerated() where SearchText.hasWords(text) { + texts.append(Prompts.document(title: mail.subject, text: text)) + places.append((index, chunk)) + } + } + let vectors = texts.isEmpty ? [] : try fullWorker().embedTexts(texts) + var vector: [Int: [Int: [Float]]] = [:] + for (place, made) in zip(places, vectors) where !made.isEmpty { vector[place.mail, default: [:]][place.chunk] = made } + try store.db.transaction { + for (index, mail) in ready.enumerated() { + let file = batch[mail.slot] + for (chunk, text) in mail.chunks.enumerated() { + try store.addSegment(file: file.id, kind: .chunk, modality: .text, loc: Double(chunk + 1), + excerpt: excerpt(text), text: text, vector: vector[index]?[chunk]) + } + // What the listing gave (the folder) stays; what reading it gave is added. + let info = try store.info(of: file.id).merging(mail.info) { $1 } + try store.finish(file.id, state: "done", info: info) + results[mail.slot] = ("done", "\(mail.chunks.count) chunks") + } + } + } + private func images(_ batch: [FileRecord], screenshot: Bool, ready: [ReadyImage], results: inout [(String, String?)]) throws { guard !ready.isEmpty else { return } diff --git a/Sources/DigUpKit/Mail.swift b/Sources/DigUpKit/Mail.swift new file mode 100644 index 00000000..0356e763 --- /dev/null +++ b/Sources/DigUpKit/Mail.swift @@ -0,0 +1,174 @@ +import Foundation + +/// Outlook mail stored in .pst files. The digup-pst tool (pst/, beside the helper) reads a PST without Outlook; each +/// mail becomes an entry of its own in the index, at a path inside its PST ("…/archive.pst::2097412": the mail's node +/// id in the PST), so search, results and the index's bookkeeping treat it as they treat a file. The PST itself is a +/// `mailbox`: syncing it lists its mails (fast, no bodies) and queues the new and changed ones (`Indexer.sync`). +/// +/// A PST that Outlook has open is locked: nothing can read it until Outlook lets go. Its mails wait (`waiting`) and +/// are taken up again by the first sync that can read it, so a locked PST never keeps indexing busy. +/// +/// Mails are indexed where the tool is: shipped with the Windows helper, where opening a mail in Outlook works +/// (`digup-pst open`). Elsewhere `$DIGUP_PST` names its folder (development, tests). +public enum Mail { + /// The tool: beside the helper, in $DIGUP_PST, then on PATH. + static let tool: String? = { + #if os(Windows) + let (name, separator): (String, Character) = ("digup-pst.exe", ";") + let path = ProcessInfo.processInfo.environment.first { $0.key.uppercased() == "PATH" }?.value ?? "" + #else + let (name, separator): (String, Character) = ("digup-pst", ":") + let path = ProcessInfo.processInfo.environment["PATH"] ?? "" + #endif + var folders: [String] = [] + if let chosen = ProcessInfo.processInfo.environment["DIGUP_PST"], !chosen.isEmpty { folders.append(chosen) } + if let executable = Bundle.main.executableURL { folders.append(executable.deletingLastPathComponent().path) } + folders += path.split(separator: separator).map(String.init) + return folders.map { $0 + "/" + name }.first { FileManager.default.isExecutableFile(atPath: $0) } + }() + + public static var isAvailable: Bool { tool != nil } + + /// Between a PST's path and a mail's node id in the path of the mail. + static let separator = "::" + + static func path(mailbox: String, node: UInt32) -> String { mailbox + separator + String(node) } + + /// The PST and the node id of a mail's path; nil for a file's. + public static func split(_ path: String) -> (mailbox: String, node: UInt32)? { + guard let range = path.range(of: separator, options: .backwards), let node = UInt32(path[range.upperBound...]) + else { return nil } + return (String(path[.. String { split(path)?.mailbox ?? path } + + /// Why a mail waits: its PST is open in Outlook. A sync that can read the PST queues it again. + static let waiting = "waiting: the PST is open in Outlook" + + enum Failure: Error, CustomStringConvertible { + case locked + var description: String { Mail.waiting } + } + + /// A mail as `list` gives it: enough to show it in results and to tell when it changed. + struct Listed: Sendable { + let node: UInt32 + let folder: String + let subject: String + let from: String + let date: String? + let modified: Double + } + + /// A mail with its text, as `read` gives it. + struct Content: Sendable { + let subject: String + let from: String + let recipients: [(kind: String, name: String)] + let date: String? + let body: String + let attachments: [(name: String, path: String?, embedded: String?)] + } + + /// The PST's mails, without their bodies. Seconds for a big PST. + static func list(_ mailbox: String) throws -> [Listed] { + try lines(["list", mailbox]).compactMap { line in + guard line["type"] as? String == "mail", let node = (line["node"] as? NSNumber)?.uint32Value else { return nil } + let date = line["received"] as? String ?? line["sent"] as? String + return Listed(node: node, folder: line["folder"] as? String ?? "", subject: line["subject"] as? String ?? "", + from: sender(line), date: date, + modified: (line["modified"] as? String ?? date).flatMap(seconds) ?? 0) + } + } + + /// The mails with these node ids, by node id, their attachments written into `folder`. A node that's gone from the + /// PST isn't in the result. + static func read(_ mailbox: String, nodes: [UInt32], attachmentsTo folder: URL) throws -> [UInt32: Content] { + var out: [UInt32: Content] = [:] + for line in try lines(["read", mailbox] + nodes.map(String.init) + ["--attachments", folder.path]) { + guard line["type"] as? String == "mail", let node = (line["node"] as? NSNumber)?.uint32Value else { continue } + let recipients = (line["recipients"] as? [[String: Any]] ?? []).map { recipient in + (kind: recipient["kind"] as? String ?? "to", name: person(recipient["name"], recipient["email"])) + } + let attachments = (line["attachments"] as? [[String: Any]] ?? []).map { item in + let embedded = item["embedded"] as? [String: Any] + return (name: item["name"] as? String ?? "", path: item["path"] as? String, + embedded: embedded.map { [$0["subject"] as? String, $0["body"] as? String] + .compactMap { $0 }.joined(separator: "\n") }) + } + out[node] = Content(subject: line["subject"] as? String ?? "", from: sender(line), recipients: recipients, + date: line["received"] as? String ?? line["sent"] as? String, + body: line["body"] as? String ?? "", attachments: attachments) + } + return out + } + + /// What's indexed of a mail: its subject, who sent it to whom and when, its body, then the text of its attachments + /// that DigUp reads (documents, spreadsheets, presentations, PDFs, attached mails). + static func text(of mail: Content) -> String { + var parts = [mail.subject] + if !mail.from.isEmpty { parts.append("From: \(mail.from)") } + for kind in ["to", "cc"] { + let names = mail.recipients.filter { $0.kind == kind }.map(\.name).filter { !$0.isEmpty } + if !names.isEmpty { parts.append("\(kind == "to" ? "To" : "Cc"): \(names.joined(separator: ", "))") } + } + if let date = mail.date { parts.append("Date: \(date)") } + var text = parts.joined(separator: "\n") + "\n\n" + mail.body + for attachment in mail.attachments { + guard let content = attachment.embedded ?? attachment.path.flatMap(attachmentText), !content.isEmpty + else { continue } + text += "\n\nAttachment: \(attachment.name)\n" + content + } + return text + } + + /// An attachment's text, when it's a kind DigUp reads for words. Pictures, audio and video are left out. + private static func attachmentText(_ path: String) -> String? { + let url = URL(fileURLWithPath: path) + let ext = url.pathExtension.lowercased() + if Rules.docExtensions.contains(ext) { return try? DocExtractor.text(of: url) } + if ext == "pdf", let pdf = try? PDFExtractor.pages(of: url, range: { 0.. String { + person(line["sender_name"], line["sender_email"]) + } + + private static func person(_ name: Any?, _ email: Any?) -> String { + let (name, email) = (name as? String ?? "", email as? String ?? "") + if name.isEmpty { return email } + return email.isEmpty || name == email ? name : "\(name) <\(email)>" + } + + private static func seconds(_ iso: String) -> Double? { + ISO8601DateFormatter().date(from: iso)?.timeIntervalSince1970 + } + + /// Runs the tool and parses its JSON lines. Exit 3: the PST is locked. + private static func lines(_ arguments: [String]) throws -> [[String: Any]] { + guard let tool else { throw ExtractError.unreadable("digup-pst isn't installed (Outlook mail needs it)") } + let process = Process() + process.executableURL = URL(fileURLWithPath: tool) + process.arguments = arguments + let output = Pipe() + process.standardOutput = output + process.standardError = FileHandle.nullDevice + process.standardInput = FileHandle.nullDevice + try process.run() + let data = output.fileHandleForReading.readDataToEndOfFile() + process.waitUntilExit() + if process.terminationStatus == 3 { throw Failure.locked } + guard process.terminationStatus == 0 else { + throw ExtractError.unreadable("digup-pst failed (\(process.terminationStatus))") + } + return data.split(separator: UInt8(ascii: "\n")).compactMap { + try? JSONSerialization.jsonObject(with: Data($0)) as? [String: Any] + } + } +} diff --git a/Sources/DigUpKit/Model.swift b/Sources/DigUpKit/Model.swift index 72cb6726..52921444 100644 --- a/Sources/DigUpKit/Model.swift +++ b/Sources/DigUpKit/Model.swift @@ -5,11 +5,13 @@ public enum FileKind: String, CaseIterable, Sendable, Codable { case screenshot, image, pdf, doc, audio, video /// Source code, a repo's docs and notebooks: only in the code index (`IndexOptions.code`). case code + /// An Outlook .pst file, and each mail in it (`Mail`): the mailbox is listed, its mails are embedded. + case mailbox, mail /// The order of the first indexing pass: what people look for most comes first. static let indexingOrderSQL = """ CASE kind WHEN 'screenshot' THEN 0 WHEN 'image' THEN 1 WHEN 'pdf' THEN 2 WHEN 'doc' THEN 3 - WHEN 'audio' THEN 4 WHEN 'video' THEN 5 ELSE 6 END + WHEN 'mail' THEN 4 WHEN 'audio' THEN 5 WHEN 'video' THEN 6 ELSE 7 END """ } diff --git a/Sources/DigUpKit/Platform/Apple/Files+Apple.swift b/Sources/DigUpKit/Platform/Apple/Files+Apple.swift index 3d594ba6..14898963 100644 --- a/Sources/DigUpKit/Platform/Apple/Files+Apple.swift +++ b/Sources/DigUpKit/Platform/Apple/Files+Apple.swift @@ -16,6 +16,7 @@ extension Rules { return options.includeCode ? .index(.doc) : .skip("code or data file") } if docExtensions.contains(ext) { return .index(.doc) } + if ext == "pst", Mail.isAvailable { return .index(.mailbox) } if codeExtensions.contains(ext) { return options.includeCode ? .index(.doc) : .skip("code or data file") } guard let type else { return .ignore } if type.conforms(to: .sourceCode) || type.conforms(to: .script) { diff --git a/Sources/DigUpKit/Platform/Portable/Files+Portable.swift b/Sources/DigUpKit/Platform/Portable/Files+Portable.swift index e5858613..a32fbf87 100644 --- a/Sources/DigUpKit/Platform/Portable/Files+Portable.swift +++ b/Sources/DigUpKit/Platform/Portable/Files+Portable.swift @@ -32,6 +32,7 @@ extension Rules { return options.includeCode ? .index(.doc) : .skip("code or data file") } if docExtensions.contains(ext) { return .index(.doc) } + if ext == "pst", Mail.isAvailable { return .index(.mailbox) } if codeExtensions.contains(ext) || sourceExtensions.contains(ext) { return options.includeCode ? .index(.doc) : .skip("code or data file") } diff --git a/Sources/DigUpKit/Searcher.swift b/Sources/DigUpKit/Searcher.swift index 4aa0b03d..e2485e10 100644 --- a/Sources/DigUpKit/Searcher.swift +++ b/Sources/DigUpKit/Searcher.swift @@ -360,6 +360,17 @@ public final class Searcher { /// The text of a page, a document's passage or a screenshot's OCR, as it's written: the preview shows the part of a /// document that matched. + /// The first chunk of a document or a mail (a mail's starts with its subject, sender, recipients and date): what a + /// preview shows when the file was found by its name. Nil when it hasn't been read yet. + public func firstText(ofFile path: String) throws -> String? { + var text: String? + try store.db.query(""" + SELECT coalesce(fts.original, fts.body) FROM files f JOIN segments s ON s.file = f.id + JOIN fts ON fts.rowid = s.id WHERE f.path = ? AND s.kind = 'chunk' ORDER BY s.loc LIMIT 1 + """, [.text(path)]) { text = $0.text(0) } + return text + } + public func text(ofSegment id: Int64) throws -> String? { var text: String? try store.db.query("SELECT coalesce(original, body) FROM fts WHERE rowid = ?", [.int(id)]) { text = $0.text(0) } @@ -416,8 +427,8 @@ public final class Searcher { static let kindWords: [String: FileKind] = [ "screenshot": .screenshot, "screenshots": .screenshot, "photo": .image, "photos": .image, "picture": .image, "pictures": .image, "image": .image, "images": .image, "pdf": .pdf, "pdfs": .pdf, "paper": .pdf, - "papers": .pdf, "document": .doc, "documents": .doc, "note": .doc, "notes": .doc, "video": .video, - "videos": .video, "clip": .video, "movie": .video, "audio": .audio, "song": .audio, "podcast": .audio, + "papers": .pdf, "document": .doc, "documents": .doc, "note": .doc, "notes": .doc, "mail": .mail, "mails": .mail, + "email": .mail, "emails": .mail, "video": .video, "videos": .video, "clip": .video, "movie": .video, "audio": .audio, "song": .audio, "podcast": .audio, "voice": .audio, "sound": .audio, "স্ক্রিনশট": .screenshot, "ছবি": .image, "পিডিএফ": .pdf, "নোট": .doc, "ভিডিও": .video, "অডিও": .audio, "গান": .audio, "রেকর্ডিং": .audio, diff --git a/Sources/digup/SearchService.swift b/Sources/digup/SearchService.swift index d5837660..3767f7f5 100644 --- a/Sources/digup/SearchService.swift +++ b/Sources/digup/SearchService.swift @@ -14,6 +14,8 @@ import Foundation /// {"op":"embed_ahead","q":…,"code":bool} → {"ok":true}: embeds and caches the text (the panel sends what's typed so /// far on every keystroke, so the pause's vector is usually ready) /// {"op":"text","segment":id,"code":bool} → {"text":…}: the part of a file a hit points at, for the preview +/// {"op":"text","segment":n,"code":bool} or {"op":"text","path":…} → {"text":…}: a segment's text, or a file's first +/// chunk (a mail's preview when its subject matched) /// {"op":"render","path":…,"kind":"pdf|video|image","loc":n,"pixels":n} → {"image":path}: a JPEG for a preview or a /// thumbnail (a PDF's page `loc`, a video's frame at `loc` seconds, or a /// picture a web view can't show, such as HEIC), made beside the requests @@ -153,11 +155,13 @@ final class SearchService: @unchecked Sendable { // its state is guarded by `l && FileManager.default.fileExists(atPath: $0.path) } } else { + // A mail's file is its PST (`Mail.file`). A PST itself isn't a result: its mails are. hits = try searcher.search(text, queryVector: vector, kinds: kinds, limit: limit, prefixLast: prefixLast) .filter { - (roots.isEmpty || FolderSelection.isSearched($0.path, folders: roots, skipped: skipped)) - && !types.contains(($0.path as NSString).pathExtension.lowercased()) - && FileManager.default.fileExists(atPath: $0.path) + let file = Mail.file(of: $0.path) + return $0.kind != .mailbox && (roots.isEmpty || FolderSelection.isSearched(file, folders: roots, skipped: skipped)) + && !types.contains((file as NSString).pathExtension.lowercased()) + && FileManager.default.fileExists(atPath: file) } } var reply: [String: Any] = [ @@ -171,8 +175,12 @@ final class SearchService: @unchecked Sendable { // its state is guarded by `l _ = vector(for: request["q"] as? String ?? "", code: request["code"] as? Bool ?? false) return ["ok": true] case "text": - guard let segment = (request["segment"] as? NSNumber)?.int64Value else { throw CLIError("which segment?") } let code = request["code"] as? Bool ?? false + // A file's first chunk: a mail's preview when it was found by its subject. + if let path = request["path"] as? String { + return ["text": try (code ? codeSearcher : searcher)?.firstText(ofFile: path) ?? NSNull()] + } + guard let segment = (request["segment"] as? NSNumber)?.int64Value else { throw CLIError("which segment?") } return ["text": try (code ? codeSearcher : searcher)?.text(ofSegment: segment) ?? NSNull()] case "reload": let hadVectors = searcher.vectorsLoaded diff --git a/Tests/DigUpKitTests/MailTests.swift b/Tests/DigUpKitTests/MailTests.swift new file mode 100644 index 00000000..e180558a --- /dev/null +++ b/Tests/DigUpKitTests/MailTests.swift @@ -0,0 +1,150 @@ +import Foundation +import Testing +@testable import DigUpKit + +/// Outlook mail in .pst files (`Mail`), with the PST Outlook made for pst/'s tests (30 mails in three folders, five +/// with an attachment). Needs the digup-pst tool: `cargo build --release` in pst/, then DIGUP_PST=pst/target/release. +@Suite(.enabled(if: Mail.isAvailable)) final class MailTests { + /// In Caches, not the temporary folder: the walk reports /private/var/… for /var/… (see LongFilesTests). + let folder: URL = { + let caches = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask)[0] + let url = caches.appendingPathComponent("DigUpTests-\(UUID().uuidString)") + try? FileManager.default.createDirectory(at: url.appendingPathComponent("files"), + withIntermediateDirectories: true) + return url + }() + + deinit { + try? FileManager.default.removeItem(at: folder) + } + + var files: URL { folder.appendingPathComponent("files") } + var pst: URL { files.appendingPathComponent("archive.pst") } + + /// pst/tests/data/outlook_made.pst, copied into the test's folder. + func copyPST(to url: URL) throws { + let source = URL(fileURLWithPath: #filePath).deletingLastPathComponent().deletingLastPathComponent() + .deletingLastPathComponent().appendingPathComponent("pst/tests/data/outlook_made.pst") + try FileManager.default.copyItem(at: source, to: url) + } + + func store() throws -> IndexStore { try IndexStore(directory: folder.appendingPathComponent("index")) } + + func indexer(_ store: IndexStore) throws -> Indexer { + try Indexer(store: store, options: IndexOptions(), makeEmbedder: { FakeEmbedder() }) + } + + /// Paths of what keyword search finds for `word`. + func found(_ store: IndexStore, _ word: String) throws -> [String] { + var out: [String] = [] + try store.db.query(""" + SELECT DISTINCT f.path FROM fts JOIN segments s ON s.id = fts.rowid JOIN files f ON f.id = s.file + WHERE fts MATCH ? ORDER BY f.path + """, [.text("\"\(word)\"")]) { out.append($0.text(0) ?? "") } + return out + } + + func mails(_ store: IndexStore) throws -> [(path: String, state: String, info: [String: String])] { + try store.mails(in: pst.path).map { ($0.path, $0.state, try store.info(of: $0.id)) }.sorted { $0.path < $1.path } + } + + @Test func pathsOfMails() { + #expect(Mail.path(mailbox: "C:\\Mail\\archive.pst", node: 2097412) == "C:\\Mail\\archive.pst::2097412") + #expect(Mail.split("C:\\Mail\\archive.pst::2097412")! == ("C:\\Mail\\archive.pst", 2097412)) + #expect(Mail.split("/Users/me/notes.txt") == nil) + #expect(Mail.split("/Users/me/odd::name.txt") == nil) + // What search checks is there on disk (and not skipped): a mail's PST. + #expect(Mail.file(of: "C:/Mail/archive.pst::2097412") == "C:/Mail/archive.pst") + #expect(Mail.file(of: "/Users/me/notes.txt") == "/Users/me/notes.txt") + } + + @Test func aPSTIsAMailbox() { + #expect(Rules.classify(pst, values: nil, options: IndexOptions()) == .index(.mailbox)) + } + + @Test func syncListsTheMailsAndRunReadsThem() throws { + try copyPST(to: pst) + let store = try store() + let indexer = try indexer(store) + let summary = try indexer.sync([files]) + #expect(summary.added == 31) // the PST and its 30 mails + #expect(summary.queued.count == 30) + var listed = try mails(store) + #expect(listed.count == 30) + #expect(listed.allSatisfy { $0.state == "pending" && $0.info["subject"]?.hasPrefix("PST test ") == true }) + #expect(Set(listed.compactMap { $0.info["folder"] }) == ["/Boite de test", "/Clients", "/Clients/Dupont"]) + // Before it's read, a mail is found by its subject. + #expect(try found(store, "PST test 07").count == 1) + + #expect(try indexer.run()["done"] == 30) + listed = try mails(store) + #expect(listed.allSatisfy { $0.state == "done" }) + let seven = try #require(listed.first { $0.info["subject"] == "PST test 07" }) + #expect(seven.info["folder"] == "/Clients") // from the listing, kept + #expect(seven.info["attachments"] == "piece_jointe.txt") + #expect(try found(store, "DEV-2024-0007") == [seven.path]) + // Attachments are read for their text: mails 0, 7, 14, 21 and 28 have one. + #expect(try found(store, "Contenu de la piece jointe").count == 5) + #expect(try store.info(of: try #require(store.files(under: [files.path]).first { $0.kind == .mailbox }).id) + == ["mails": "30"]) + + // Nothing changed: nothing is listed or queued again. + let again = try indexer.sync([files]) + #expect(again.added == 0 && again.changed == 0 && again.removed == 0 && again.queued.isEmpty) + } + + @Test func aMovedPSTKeepsItsMails() throws { + try copyPST(to: pst) + let store = try store() + let indexer = try indexer(store) + _ = try indexer.sync([files]) + try indexer.run() + let renamed = files.appendingPathComponent("archive 2024.pst") + try FileManager.default.moveItem(at: pst, to: renamed) + let summary = try indexer.sync([files]) + #expect(summary.moved == 1 && summary.removed == 0 && summary.queued.isEmpty) + let moved = try store.mails(in: renamed.path) + #expect(moved.count == 30 && moved.allSatisfy { $0.state == "done" }) + #expect(try found(store, "DEV-2024-0007").first?.hasPrefix(renamed.path + "::") == true) + } + + @Test func mailsWaitingForOutlookAreReadOnceThePSTCanBe() throws { + try copyPST(to: pst) + let store = try store() + let indexer = try indexer(store) + _ = try indexer.sync([files]) + try indexer.run() + // As if Outlook had held the PST when these were read. + let waiting = Array(try store.mails(in: pst.path).prefix(3)) + for mail in waiting { try store.finish(mail.id, state: "skipped", detail: Mail.waiting, info: store.info(of: mail.id)) } + #expect(try store.pendingCount() == 0) // waiting isn't pending: nothing spins while Outlook has it + let summary = try indexer.sync([files]) // the PST didn't change, but its mails wait: it's listed + #expect(Set(summary.queued) == Set(waiting.map(\.id))) + #expect(try indexer.run()["done"] == 3) + } + + @Test func aMailsPreviewIsItsBeginning() throws { + try copyPST(to: pst) + let store = try store() + let indexer = try indexer(store) + _ = try indexer.sync([files]) + let seven = try #require(try store.mails(in: pst.path).first { try store.info(of: $0.id)["subject"] == "PST test 07" }) + let searcher = try Searcher(store: store, loadVectors: false) + #expect(try searcher.firstText(ofFile: seven.path) == nil) // listed, not read yet + try indexer.run() + let text = try #require(try searcher.firstText(ofFile: seven.path)) + #expect(text.hasPrefix("PST test 07\n")) + #expect(text.contains("reference DEV-2024-0007")) + } + + @Test func aDeletedPSTTakesItsMailsAlong() throws { + try copyPST(to: pst) + let store = try store() + let indexer = try indexer(store) + _ = try indexer.sync([files]) + try FileManager.default.removeItem(at: pst) + let summary = try indexer.sync([files]) + #expect(summary.removed == 31) + #expect(try store.files(under: [files.path]).isEmpty) + } +} diff --git a/pst/.cargo/config.toml b/pst/.cargo/config.toml new file mode 100644 index 00000000..38f7da93 --- /dev/null +++ b/pst/.cargo/config.toml @@ -0,0 +1,6 @@ +# Windows: the C runtime linked into digup-pst.exe, so it starts on a PC without the Visual C++ runtime installed. +[target.x86_64-pc-windows-msvc] +rustflags = ["-C", "target-feature=+crt-static"] + +[target.aarch64-pc-windows-msvc] +rustflags = ["-C", "target-feature=+crt-static"] diff --git a/pst/Cargo.lock b/pst/Cargo.lock new file mode 100644 index 00000000..00b153ad --- /dev/null +++ b/pst/Cargo.lock @@ -0,0 +1,624 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "codepage" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdff162541cd8b79de82e2edcc7eff3a8c2a6dc3d75152636028f96d93de3b26" +dependencies = [ + "encoding_rs", +] + +[[package]] +name = "compressed-rtf" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51b7f2aaf6791c6517669240b82ef12c1283b39772f0cad3bc5c73f19f6e7f16" +dependencies = [ + "byteorder", + "thiserror", +] + +[[package]] +name = "core_detect" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" + +[[package]] +name = "digup-pst" +version = "0.1.0" +dependencies = [ + "anyhow", + "codepage", + "compressed-rtf", + "encoding_rs", + "html2text", + "outlook-pst", + "serde_json", + "windows", +] + +[[package]] +name = "encoding_rs" +version = "0.8.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e985e0451871ad22fb8d2b6b076e2028a502a0d3950998c2c5c0a4f9b5d9679" +dependencies = [ + "cfg-if", + "core_detect", + "multiversion_no_op", + "rustversion", + "scopeguard", + "simdutf8", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "html2text" +version = "0.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7475d5c5606bb86be8d7986962ba34b233490985cfa76a20d6a23e1eb9563b1c" +dependencies = [ + "html5ever", + "tendril", + "thiserror", + "unicode-width", +] + +[[package]] +name = "html5ever" +version = "0.40.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "456a1a377e608e555d22ddab27ac0114bc7a7b4199078108e34c2aeae6c9b130" +dependencies = [ + "log", + "markup5ever", + "memchr", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.190" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "markup5ever" +version = "0.40.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ab3dc68ac4a0f5719e560136778c1ee716e296030d75dbd4484e37e39e3a842" +dependencies = [ + "log", + "tendril", + "web_atoms", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "multiversion_no_op" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" + +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "outlook-pst" +version = "1.2.0" +source = "git+https://github.com/Private-Recall/outlook-pst-rs.git?rev=32e4a88acd76b0bc643f8c475cc36c56a8708b94#32e4a88acd76b0bc643f8c475cc36c56a8708b94" +dependencies = [ + "byteorder", + "thiserror", + "tracing", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "phf" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "010378780309880b08997fae13be7834dba947d36393bd372f2b1556deb2a2f6" +dependencies = [ + "phf_shared", + "serde", +] + +[[package]] +name = "phf_codegen" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41b585a510fb76fdebead6897982ef2a03a21d8e6cbcca904999742a4afc6ffe" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aeb62e0959d5a1bebc965f4d15d9e2b7cea002b6b0f5ba8cde6cc26738467100" +dependencies = [ + "fastrand", + "phf_shared", +] + +[[package]] +name = "phf_shared" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6fd9027e2d9319be6349febd1db4e8d02aa544921200c9b777720ac34a3aa89" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "siphasher" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256" + +[[package]] +name = "smallvec" +version = "1.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b" + +[[package]] +name = "string_cache" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffa8a5dbe8b3f0bbe29d4c3225daafaeead63afdc1b65fc4c01a1384166038e6" +dependencies = [ + "new_debug_unreachable", + "parking_lot", + "phf_shared", + "precomputed-hash", +] + +[[package]] +name = "string_cache_codegen" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "928dcdf75e47626b3617a976ec205d9f057584c371c1f23b782129268d0e6edc" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tendril" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fed54709c5b3a53d09bb1c113ea4f5ceafd1e772ddcb0030a82e1d56c087b08" +dependencies = [ + "new_debug_unreachable", +] + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "web_atoms" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7572660c8890448ba236b7376f27e389c6a7e1c70195622faced601f855c0ada" +dependencies = [ + "phf", + "phf_codegen", + "string_cache", + "string_cache_codegen", +] + +[[package]] +name = "windows" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd04d41d93c4992d421894c18c8b43496aa748dd4c081bac0dc93eb0489272b6" +dependencies = [ + "windows-core", + "windows-targets", +] + +[[package]] +name = "windows-core" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba6d44ec8c2591c134257ce647b7ea6b20335bf6379a27dac5f1641fcf59f99" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-result", + "windows-strings", + "windows-targets", +] + +[[package]] +name = "windows-implement" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-strings" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" +dependencies = [ + "windows-result", + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/pst/Cargo.toml b/pst/Cargo.toml new file mode 100644 index 00000000..98e45381 --- /dev/null +++ b/pst/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "digup-pst" +version = "0.1.0" +description = "Outlook mail stored in .pst files, for DigUp's helper: list and read mails without Outlook, open one in Outlook" +edition = "2021" +license = "MIT" +publish = false + +[dependencies] +# microsoft/outlook-pst-rs with its PR #62 (messages without a subnode tree, ~half of some real PSTs), pinned to that +# PR's commit until it's merged and released upstream; then the crates.io release. +outlook-pst = { git = "https://github.com/Private-Recall/outlook-pst-rs.git", rev = "32e4a88acd76b0bc643f8c475cc36c56a8708b94" } +compressed-rtf = "1" +anyhow = "1" +serde_json = "1" +html2text = "0.17" +encoding_rs = "0.8" +codepage = "0.1" + +# Opening a mail in Outlook: its COM automation (classic Outlook only; the new Outlook has none). +[target.'cfg(windows)'.dependencies] +windows = { version = "0.58", features = ["Win32_Foundation", "Win32_System_Com", "Win32_System_Ole", + "Win32_System_Variant", "Win32_UI_WindowsAndMessaging"] } + +[profile.release] +debug = false diff --git a/pst/src/main.rs b/pst/src/main.rs new file mode 100644 index 00000000..a520e188 --- /dev/null +++ b/pst/src/main.rs @@ -0,0 +1,147 @@ +//! digup-pst: Outlook mail stored in .pst files, for DigUp's helper. It reads PSTs without Outlook, MAPI or COM, with +//! microsoft/outlook-pst-rs (MIT), and on Windows opens a mail in classic Outlook through its COM automation. +//! +//! Usage: +//! digup-pst list FILE.pst one JSON line per mail (no body), then a summary line +//! digup-pst read FILE.pst NODE... [--attachments DIR] +//! those mails with their body as plain text, and with --attachments +//! their attachments written into DIR +//! digup-pst open FILE.pst NODE shows the mail in Outlook (Windows) +//! +//! A mail is known by its node id in the PST (NODE, as `list` gives it); its Outlook EntryID is built from it and the +//! PST's record key when it's opened, since Outlook can give a PST a new record key (a copy attached beside the original). +//! +//! Lines: {"type":"store"...}, then {"type":"mail"...} or {"type":"error"...}, then {"type":"summary"...}. Error lines +//! carry the library's error kind and a node id, never mail content. +//! Exit codes: 0 ok, 1 error, 3 the PST is locked (Outlook has it open: try again later), 4 classic Outlook isn't +//! there, 5 the PST isn't open in Outlook, 6 Outlook can't find the mail. + +mod open; +mod reader; +mod text; + +use std::{ + io::{BufWriter, Write}, + path::{Path, PathBuf}, + process::ExitCode, + time::Instant, +}; + +use anyhow::{anyhow, Context, Result}; +use reader::{is_locked, open, Backend, Opened, Options, Walker}; +use serde_json::{json, Value}; + +enum Command { + List, + Read { nodes: Vec, attachments: Option }, + Open { node: u32 }, +} + +const USAGE: &str = "usage: digup-pst list FILE.pst | read FILE.pst NODE... [--attachments DIR] | open FILE.pst NODE"; + +fn parse_args() -> Result<(Command, PathBuf)> { + let mut args = std::env::args().skip(1); + let command = args.next().context(USAGE)?; + let file: PathBuf = args.next().context(USAGE)?.into(); + let rest: Vec = args.collect(); + let node = |s: &str| s.parse::().with_context(|| format!("not a node id: {s}")); + let command = match command.as_str() { + "list" if rest.is_empty() => Command::List, + "read" => { + let (mut nodes, mut attachments) = (Vec::new(), None); + let mut rest = rest.iter(); + while let Some(arg) = rest.next() { + if arg == "--attachments" { + attachments = Some(rest.next().context("--attachments needs a folder")?.into()); + } else { + nodes.push(node(arg)?); + } + } + Command::Read { nodes, attachments } + } + "open" if rest.len() == 1 => Command::Open { node: node(&rest[0])? }, + _ => return Err(anyhow!(USAGE)), + }; + Ok((command, file)) +} + +fn list(backend: B) -> Result<()> { + let started = Instant::now(); + let mut walker = Walker::new(backend, Options { attachments_dir: None, bodies: false })?; + let mut out = BufWriter::new(std::io::stdout().lock()); + writeln!(out, "{}", walker.store_line())?; + let mut failed = None; + walker.walk(&mut |line: Value| { + if let Err(err) = writeln!(out, "{line}") { + failed.get_or_insert(err); + } + })?; + if let Some(err) = failed { + return Err(err.into()); + } + let s = &walker.stats; + writeln!(out, "{}", json!({ + "type": "summary", "folders": s.folders, "mails": s.mails, "other_items": s.other_items, "errors": s.errors, + "seconds": (started.elapsed().as_secs_f64() * 100.0).round() / 100.0, + }))?; + out.flush()?; + Ok(()) +} + +fn read(backend: B, nodes: &[u32], attachments: Option<&Path>) -> Result<()> { + if let Some(dir) = attachments { + std::fs::create_dir_all(dir)?; + } + let mut walker = Walker::new(backend, Options { attachments_dir: attachments, bodies: true })?; + let mut out = BufWriter::new(std::io::stdout().lock()); + writeln!(out, "{}", walker.store_line())?; + let mut failed = None; + walker.read(nodes, &mut |line: Value| { + if let Err(err) = writeln!(out, "{line}") { + failed.get_or_insert(err); + } + }); + if let Some(err) = failed { + return Err(err.into()); + } + writeln!(out, "{}", json!({"type": "summary", "mails": walker.stats.mails, "errors": walker.stats.errors}))?; + out.flush()?; + Ok(()) +} + +fn main() -> ExitCode { + let (command, file) = match parse_args() { + Ok(parsed) => parsed, + Err(err) => { + eprintln!("{err:#}"); + return ExitCode::from(1); + } + }; + if let Command::Open { node } = command { + return open::show(&file, node); + } + let result = match open(&file) { + Ok(Opened::Unicode(b)) => run(b, &command), + Ok(Opened::Ansi(b)) => run(b, &command), + Err(err) if is_locked(&err) => { + println!("{}", json!({"type": "error", "error": "locked", "detail": "the PST is open in Outlook"})); + return ExitCode::from(3); + } + Err(err) => Err(anyhow!(err).context("cannot open the PST")), + }; + match result { + Ok(()) => ExitCode::SUCCESS, + Err(err) => { + eprintln!("{err:#}"); + ExitCode::from(1) + } + } +} + +fn run(backend: B, command: &Command) -> Result<()> { + match command { + Command::List => list(backend), + Command::Read { nodes, attachments } => read(backend, nodes, attachments.as_deref()), + Command::Open { .. } => unreachable!(), + } +} diff --git a/pst/src/open.rs b/pst/src/open.rs new file mode 100644 index 00000000..ed89c996 --- /dev/null +++ b/pst/src/open.rs @@ -0,0 +1,194 @@ +//! Showing a mail in classic Outlook, through its COM automation (late bound, as a script would): the store whose file +//! is the PST, the mail by its EntryID in that store, its window brought to the front. Outlook starts if it isn't +//! running, with its default profile. Nothing is changed in the profile: a PST that isn't open in Outlook is reported +//! (exit 5), not added. + +use std::{path::Path, process::ExitCode}; + +#[cfg(not(windows))] +pub fn show(_file: &Path, _node: u32) -> ExitCode { + eprintln!("opening a mail needs classic Outlook on Windows"); + ExitCode::from(4) +} + +#[cfg(windows)] +pub fn show(file: &Path, node: u32) -> ExitCode { + match windows_outlook::show(file, node) { + Ok(()) => ExitCode::SUCCESS, + Err(failure) => { + eprintln!("{}", failure.message); + ExitCode::from(failure.code) + } + } +} + +#[cfg(windows)] +mod windows_outlook { + use std::{mem::ManuallyDrop, path::Path, thread::sleep, time::Duration}; + + use windows::{ + core::{IUnknown, Interface, BSTR, GUID, HRESULT, PCWSTR, VARIANT}, + Win32::{ + System::Com::{ + CLSIDFromProgID, CoCreateInstance, CoInitializeEx, IDispatch, CLSCTX_LOCAL_SERVER, + COINIT_APARTMENTTHREADED, DISPATCH_FLAGS, DISPATCH_METHOD, DISPATCH_PROPERTYGET, DISPPARAMS, EXCEPINFO, + }, + UI::WindowsAndMessaging::{AllowSetForegroundWindow, ASFW_ANY}, + }, + }; + + pub struct Failure { + pub code: u8, + pub message: String, + } + + fn fail(code: u8, message: impl Into) -> Failure { + Failure { code, message: message.into() } + } + + /// Outlook answers "busy" while it starts or shows a dialog: calls are tried again for a while. + const RPC_E_CALL_REJECTED: HRESULT = HRESULT(0x8001_0001_u32 as i32); + const RPC_E_SERVERCALL_RETRYLATER: HRESULT = HRESULT(0x8001_010A_u32 as i32); + const TYPE_E_TYPEMISMATCH: HRESULT = HRESULT(0x8002_8CA0_u32 as i32); + const LOCALE_USER_DEFAULT: u32 = 0x0400; + /// PR_RECORD_KEY of a store: the PST's own id, which its mails' EntryIDs carry. + const RECORD_KEY: &str = "http://schemas.microsoft.com/mapi/proptag/0x0FF90102"; + + struct Object(IDispatch); + + impl Object { + fn call(&self, name: &str, flags: DISPATCH_FLAGS, args: &[VARIANT]) -> windows::core::Result { + let wide: Vec = name.encode_utf16().chain([0]).collect(); + let names = [PCWSTR(wide.as_ptr())]; + let mut id = 0i32; + // Arguments go in reverse order. + let mut args: Vec = args.iter().rev().cloned().collect(); + let params = DISPPARAMS { + rgvarg: args.as_mut_ptr(), + cArgs: args.len() as u32, + ..Default::default() + }; + let mut tries = 0; + loop { + let mut exception = EXCEPINFO::default(); + let result = unsafe { + self.0.GetIDsOfNames(&GUID::zeroed(), names.as_ptr(), 1, LOCALE_USER_DEFAULT, &mut id).and_then( + |()| { + let mut value = VARIANT::default(); + self.0 + .Invoke(id, &GUID::zeroed(), LOCALE_USER_DEFAULT, flags, ¶ms, Some(&mut value), + Some(&mut exception), None) + .map(|()| value) + }, + ) + }; + // Outlook says what went wrong in the exception's description ("a dialog box is open"…). + let description = ManuallyDrop::into_inner(exception.bstrDescription).to_string(); + drop(ManuallyDrop::into_inner(exception.bstrSource)); + drop(ManuallyDrop::into_inner(exception.bstrHelpFile)); + let result = result.map_err(|err| { + if description.is_empty() { err } else { windows::core::Error::new(err.code(), description) } + }); + match result { + Err(err) if (err.code() == RPC_E_CALL_REJECTED || err.code() == RPC_E_SERVERCALL_RETRYLATER) + && tries < 60 => { + tries += 1; + sleep(Duration::from_millis(500)); + } + other => return other, + } + } + } + + fn get(&self, name: &str) -> windows::core::Result { + self.call(name, DISPATCH_PROPERTYGET, &[]) + } + + fn method(&self, name: &str, args: &[VARIANT]) -> windows::core::Result { + self.call(name, DISPATCH_METHOD, args) + } + + /// The object a call returned. Outlook hands objects back as VT_DISPATCH, which windows-core's conversions + /// don't take (they want VT_UNKNOWN), so the pointer is read here. + fn object(value: VARIANT) -> windows::core::Result { + const VT_UNKNOWN: u16 = 13; + const VT_DISPATCH: u16 = 9; + unsafe { + let raw = &value.as_raw().Anonymous.Anonymous; + let pointer = raw.Anonymous.punkVal; + if (raw.vt == VT_DISPATCH || raw.vt == VT_UNKNOWN) && !pointer.is_null() { + if let Some(unknown) = IUnknown::from_raw_borrowed(&pointer) { + return Ok(Object(unknown.cast::()?)); + } + } + } + Err(windows::core::Error::from_hresult(TYPE_E_TYPEMISMATCH)) + } + + fn text(value: &VARIANT) -> String { + BSTR::try_from(value).map(|b| b.to_string()).unwrap_or_default() + } + } + + /// The same file as Outlook spells it? Paths compared without case and with one kind of slash. + fn same_file(a: &str, b: &Path) -> bool { + let norm = |s: &str| s.replace('/', "\\").trim_end_matches('\\').to_lowercase(); + let b = std::fs::canonicalize(b).map(|p| p.to_string_lossy().trim_start_matches(r"\\?\").to_string()) + .unwrap_or_else(|_| b.to_string_lossy().to_string()); + norm(a) == norm(&b) || norm(a) == norm(&b.trim_start_matches(r"\\?\").to_string()) + } + + pub fn show(file: &Path, node: u32) -> Result<(), Failure> { + unsafe { + let _ = CoInitializeEx(None, COINIT_APARTMENTTHREADED); + } + let clsid = unsafe { CLSIDFromProgID(windows::core::w!("Outlook.Application")) } + .map_err(|_| fail(4, "classic Outlook isn't installed"))?; + let app: IDispatch = unsafe { CoCreateInstance(&clsid, None, CLSCTX_LOCAL_SERVER) } + .map_err(|err| fail(4, format!("Outlook didn't start: {err}")))?; + let app = Object(app); + let com = |err: windows::core::Error| fail(1, format!("Outlook: {err}")); + let session = Object::object(app.method("GetNamespace", &[VARIANT::from(BSTR::from("MAPI"))]).map_err(com)?) + .map_err(com)?; + // Started by this call, Outlook isn't logged on yet: its default profile, as when it's opened from the Start menu. + if Object::text(&session.get("CurrentProfileName").map_err(com)?).is_empty() { + session.method("Logon", &[]).map_err(com)?; + } + + let stores = Object::object(session.get("Stores").map_err(com)?).map_err(com)?; + let count = i32::try_from(&stores.get("Count").map_err(com)?).unwrap_or(0); + let mut found = None; + for index in 1..=count { + let Ok(store) = stores.method("Item", &[VARIANT::from(index)]).and_then(Object::object) else { continue }; + // Exchange and IMAP stores have no file, or an .ost one. + let path = store.get("FilePath").map(|v| Object::text(&v)).unwrap_or_default(); + if !path.is_empty() && same_file(&path, file) { + found = Some(store); + break; + } + } + let store = found.ok_or_else(|| fail(5, "this PST isn't open in Outlook"))?; + + // The EntryID: four zero bytes, the store's record key, the node id (little-endian), in hex. + let accessor = Object::object(store.get("PropertyAccessor").map_err(com)?).map_err(com)?; + let key = accessor.method("GetProperty", &[VARIANT::from(BSTR::from(RECORD_KEY))]).map_err(com)?; + let key = Object::text(&accessor.method("BinaryToString", &[key]).map_err(com)?); + let entry_id = format!("00000000{}{}", key.to_uppercase(), + node.to_le_bytes().iter().map(|b| format!("{b:02X}")).collect::()); + let store_id = store.get("StoreID").map_err(com)?; + let mail = session + .method("GetItemFromID", &[VARIANT::from(BSTR::from(entry_id.as_str())), store_id]) + .and_then(Object::object) + .map_err(|_| fail(6, "Outlook can't find this mail in the PST"))?; + + // The shell started this program from the foreground: let Outlook come in front of it. + unsafe { + let _ = AllowSetForegroundWindow(ASFW_ANY); + } + mail.method("Display", &[VARIANT::from(false)]).map_err(com)?; + if let Ok(window) = mail.get("GetInspector").and_then(Object::object) { + let _ = window.method("Activate", &[]); + } + Ok(()) + } +} diff --git a/pst/src/reader.rs b/pst/src/reader.rs new file mode 100644 index 00000000..db2e8bbb --- /dev/null +++ b/pst/src/reader.rs @@ -0,0 +1,569 @@ +//! Walking a PST, or reading some of its mails, and turning each mail into a JSON object. +//! +//! The crate reads attachments only from concrete store and message types (Unicode or ANSI PST), +//! so the walk is generic over a small `Backend` trait implemented for both. + +use std::{collections::BTreeMap, io, path::Path, rc::Rc}; + +use outlook_pst::{ + ltp::{prop_context::PropertyValue, table_context::TableContext}, + messaging::{ + attachment::{AnsiAttachment, Attachment, AttachmentData, UnicodeAttachment}, + folder::Folder, + message::{AnsiMessage, Message, UnicodeMessage}, + store::{AnsiStore, EntryId, Store, UnicodeStore}, + }, + ndb::node_id::NodeId, + AnsiPstFile, UnicodePstFile, +}; +use serde_json::{json, Value}; + +use crate::text::{collapse_blank_lines, decode_8bit, filetime_to_iso, html_to_text, rtf_to_text}; + +// MAPI property ids. +const PR_MESSAGE_CLASS: u16 = 0x001A; +const PR_SUBJECT: u16 = 0x0037; +const PR_CLIENT_SUBMIT_TIME: u16 = 0x0039; +const PR_SENT_REPRESENTING_NAME: u16 = 0x0042; +const PR_SENT_REPRESENTING_EMAIL_ADDRESS: u16 = 0x0065; +const PR_TRANSPORT_MESSAGE_HEADERS: u16 = 0x007D; +const PR_DISPLAY_CC: u16 = 0x0E03; +const PR_DISPLAY_TO: u16 = 0x0E04; +const PR_SENDER_NAME: u16 = 0x0C1A; +const PR_SENDER_EMAIL_ADDRESS: u16 = 0x0C1F; +const PR_RECIPIENT_TYPE: u16 = 0x0C15; +const PR_MESSAGE_DELIVERY_TIME: u16 = 0x0E06; +const PR_ATTACH_SIZE: u16 = 0x0E20; +const PR_BODY: u16 = 0x1000; +const PR_RTF_COMPRESSED: u16 = 0x1009; +const PR_HTML: u16 = 0x1013; +const PR_DISPLAY_NAME: u16 = 0x3001; +const PR_EMAIL_ADDRESS: u16 = 0x3003; +const PR_LAST_MODIFICATION_TIME: u16 = 0x3008; +const PR_ATTACH_FILENAME: u16 = 0x3704; +const PR_ATTACH_METHOD: u16 = 0x3705; +const PR_ATTACH_LONG_FILENAME: u16 = 0x3707; +const PR_ATTACH_CONTENT_ID: u16 = 0x3712; +const PR_SMTP_ADDRESS: u16 = 0x39FE; +const PR_INTERNET_CPID: u16 = 0x3FDE; +const PR_MESSAGE_CODEPAGE: u16 = 0x3FFD; +const PR_SENDER_SMTP_ADDRESS: u16 = 0x5D01; +const PR_SENT_REPRESENTING_SMTP_ADDRESS: u16 = 0x5D02; +const PR_ATTACHMENT_HIDDEN: u16 = 0x7FFE; + +const ATTACH_BY_VALUE: i32 = 1; + +/// Options of a run. +pub struct Options<'a> { + /// Write attachment bytes there (`ENTRYID_N.ext`); None: list attachments only. + pub attachments_dir: Option<&'a Path>, + /// Include body text in the JSON lines. + pub bodies: bool, +} + +/// Counters for the summary line. +#[derive(Default)] +pub struct Stats { + pub folders: usize, + pub items: usize, + pub mails: usize, + pub other_items: usize, + pub errors: usize, + pub body_kinds: BTreeMap<&'static str, usize>, + pub body_chars: usize, + pub attachments: usize, + pub embedded_messages: usize, + pub error_kinds: BTreeMap, +} + +/// What differs between Unicode and ANSI PSTs. +pub trait Backend { + type Msg: Message + 'static; + fn store(&self) -> Rc; + fn open_message(&self, entry_id: &EntryId) -> io::Result>; + fn open_attachment(&self, message: &Rc, sub_node: NodeId) -> io::Result>; +} + +pub struct UnicodeBackend(pub Rc); +pub struct AnsiBackend(pub Rc); + +impl Backend for UnicodeBackend { + type Msg = UnicodeMessage; + fn store(&self) -> Rc { + self.0.clone() + } + fn open_message(&self, entry_id: &EntryId) -> io::Result> { + UnicodeMessage::read(self.0.clone(), entry_id, None) + } + fn open_attachment(&self, message: &Rc, sub_node: NodeId) -> io::Result> { + Ok(UnicodeAttachment::read(message.clone(), sub_node, None)?) + } +} + +impl Backend for AnsiBackend { + type Msg = AnsiMessage; + fn store(&self) -> Rc { + self.0.clone() + } + fn open_message(&self, entry_id: &EntryId) -> io::Result> { + AnsiMessage::read(self.0.clone(), entry_id, None) + } + fn open_attachment(&self, message: &Rc, sub_node: NodeId) -> io::Result> { + Ok(AnsiAttachment::read(message.clone(), sub_node, None)?) + } +} + +/// Opens a PST, Unicode first then ANSI. A file Outlook has open fails with a lock error. +pub enum Opened { + Unicode(UnicodeBackend), + Ansi(AnsiBackend), +} + +pub fn open(path: &Path) -> io::Result { + match UnicodePstFile::open(path) { + Ok(file) => Ok(Opened::Unicode(UnicodeBackend(UnicodeStore::read(Rc::new(file))?))), + Err(err) if is_locked(&err) => Err(err), + Err(_) => Ok(Opened::Ansi(AnsiBackend(AnsiStore::read(Rc::new(AnsiPstFile::open(path)?))?))), + } +} + +/// Windows' sharing and lock violations: Outlook has the file open. +pub fn is_locked(err: &io::Error) -> bool { + matches!(err.raw_os_error(), Some(32) | Some(33)) || err.kind() == io::ErrorKind::PermissionDenied +} + +fn string(value: Option<&PropertyValue>, code_page: Option) -> Option { + match value? { + PropertyValue::String8(v) => Some(decode_8bit(v.buffer(), code_page)), + PropertyValue::Unicode(v) => Some(String::from_utf16_lossy(v.buffer())), + _ => None, + } + .map(|s| s.trim_end_matches('\0').to_string()) +} + +fn int(value: Option<&PropertyValue>) -> Option { + match value? { + PropertyValue::Integer32(v) => Some(*v), + _ => None, + } +} + +fn time(value: Option<&PropertyValue>) -> Option { + match value? { + PropertyValue::Time(v) => filetime_to_iso(*v), + _ => None, + } +} + +/// The subject, without the 2-character prefix marker MAPI may store in front of it. +fn subject(value: Option<&PropertyValue>, code_page: Option) -> String { + let s = string(value, code_page).unwrap_or_default(); + match s.chars().next() { + Some('\u{1}') => s.chars().skip(2).collect(), + _ => s, + } +} + +/// Internet headers (PR_TRANSPORT_MESSAGE_HEADERS), unfolded, by lower-case name (first value wins). +fn headers(raw: &str) -> BTreeMap { + let mut out = BTreeMap::new(); + let mut current: Option<(String, String)> = None; + for line in raw.lines() { + if line.starts_with([' ', '\t']) { + if let Some((_, value)) = current.as_mut() { + value.push(' '); + value.push_str(line.trim()); + } + continue; + } + if let Some((name, value)) = current.take() { + out.entry(name).or_insert(value); + } + if line.is_empty() { + break; + } + if let Some((name, value)) = line.split_once(':') { + current = Some((name.trim().to_ascii_lowercase(), value.trim().to_string())); + } + } + if let Some((name, value)) = current { + out.entry(name).or_insert(value); + } + out +} + +/// "Jean Dupont , b@y.fr, Bob" as (name, email) pairs; a part without an address +/// (exports from Lotus Notes and others) keeps its name with an empty email. +fn addresses(list: &str) -> Vec<(String, String)> { + let mut out = Vec::new(); + let mut depth = 0; + let mut quoted = false; + let mut start = 0; + let bytes = list.as_bytes(); + let mut parts = Vec::new(); + for (i, &b) in bytes.iter().enumerate() { + match b { + b'"' => quoted = !quoted, + b'<' if !quoted => depth += 1, + b'>' if !quoted => depth -= 1, + b',' | b';' if !quoted && depth == 0 => { + parts.push(&list[start..i]); + start = i + 1; + } + _ => {} + } + } + parts.push(&list[start..]); + for part in parts { + let part = part.trim(); + let (name, email) = match (part.rfind('<'), part.rfind('>')) { + (Some(a), Some(b)) if a < b => (part[..a].trim().trim_matches('"').trim(), part[a + 1..b].trim()), + _ => ("", part), + }; + if email.contains('@') { + out.push((name.to_string(), email.to_string())); + } else if !part.is_empty() { + out.push((part.trim_matches('"').trim().to_string(), String::new())); + } + } + out +} + +fn is_address(s: &str) -> bool { + s.contains('@') && !s.contains(' ') +} + +type Props<'a> = dyn Fn(u16) -> Option<&'a PropertyValue> + 'a; + +fn code_page_of(get: &Props) -> Option { + int(get(PR_INTERNET_CPID)).or_else(|| int(get(PR_MESSAGE_CODEPAGE))).map(|v| v as u32) +} + +/// Body text and where it came from: plain text, else HTML, else RTF. +fn body(get: &Props) -> (&'static str, String) { + let cp = code_page_of(get); + if let Some(text) = string(get(PR_BODY), cp).filter(|t| !t.trim().is_empty()) { + return ("text", collapse_blank_lines(&text)); + } + let html = match get(PR_HTML) { + Some(PropertyValue::Binary(v)) => Some(decode_8bit(v.buffer(), cp.or(Some(65001)))), + other => string(other, cp), + }; + if let Some(html) = html.filter(|h| !h.trim().is_empty()) { + return ("html", html_to_text(&html)); + } + if let Some(PropertyValue::Binary(v)) = get(PR_RTF_COMPRESSED) { + if let Ok(rtf) = compressed_rtf::decompress_rtf(v.buffer()) { + return ("rtf", rtf_to_text(&rtf)); + } + } + ("none", String::new()) +} + +/// Rows of a table as maps from property id to value. +fn table_rows(table: &dyn TableContext) -> Vec<(u32, BTreeMap)> { + let context = table.context(); + table + .rows_matrix() + .filter_map(|row| { + let columns = row.columns(context).ok()?; + let mut values = BTreeMap::new(); + for (column, value) in context.columns().iter().zip(columns) { + if let Some(value) = value.as_ref() { + if let Ok(v) = table.read_column(value, column.prop_type()) { + values.insert(column.prop_id(), v); + } + } + } + Some((u32::from(row.id()), values)) + }) + .collect() +} + +/// Recipients from the recipient table, else from the Internet headers, else names only from +/// PR_DISPLAY_TO / PR_DISPLAY_CC. +fn recipients(message: &dyn Message, get: &Props, hdrs: &BTreeMap, code_page: Option) + -> Vec { + let from_table = match message.recipient_table() { + Some(table) => table_recipients(table.as_ref(), code_page), + None => Vec::new(), + }; + if !from_table.is_empty() { + return from_table; + } + let mut out = Vec::new(); + for (kind, header) in [("to", "to"), ("cc", "cc")] { + for (name, email) in hdrs.get(header).map(|v| addresses(v)).unwrap_or_default() { + out.push(json!({"kind": kind, "name": name, "email": email})); + } + } + if !out.is_empty() { + return out; + } + for (kind, id) in [("to", PR_DISPLAY_TO), ("cc", PR_DISPLAY_CC)] { + for name in string(get(id), code_page).unwrap_or_default().split(';') { + let name = name.trim(); + if !name.is_empty() { + out.push(json!({"kind": kind, "name": name, "email": ""})); + } + } + } + out +} + +fn table_recipients(table: &dyn TableContext, code_page: Option) -> Vec { + table_rows(table) + .into_iter() + .map(|(_, row)| { + let kind = match int(row.get(&PR_RECIPIENT_TYPE)) { + Some(2) => "cc", + Some(3) => "bcc", + _ => "to", + }; + let email = string(row.get(&PR_SMTP_ADDRESS), code_page) + .or_else(|| string(row.get(&PR_EMAIL_ADDRESS), code_page)) + .unwrap_or_default(); + json!({"kind": kind, "name": string(row.get(&PR_DISPLAY_NAME), code_page).unwrap_or_default(), + "email": email}) + }) + .collect() +} + +pub fn entry_id_hex(record_key: &[u8; 16], node: u32) -> String { + let mut bytes = vec![0u8; 4]; + bytes.extend_from_slice(record_key); + bytes.extend_from_slice(&node.to_le_bytes()); + bytes.iter().map(|b| format!("{b:02X}")).collect() +} + +fn safe_name(name: &str) -> String { + let ext = Path::new(name).extension().and_then(|e| e.to_str()).unwrap_or(""); + let ext: String = ext.chars().filter(|c| c.is_ascii_alphanumeric()).take(10).collect(); + if ext.is_empty() { String::new() } else { format!(".{}", ext.to_ascii_lowercase()) } +} + +pub struct Walker<'a, B: Backend> { + pub backend: B, + pub options: Options<'a>, + pub stats: Stats, + record_key: [u8; 16], +} + +impl<'a, B: Backend> Walker<'a, B> { + pub fn new(backend: B, options: Options<'a>) -> io::Result { + let record_key = *backend.store().properties().make_entry_id(NodeId::from(0u32))?.record_key(); + Ok(Self { backend, options, stats: Stats::default(), record_key }) + } + + pub fn store_line(&self) -> Value { + let store = self.backend.store(); + json!({"type": "store", "display_name": store.properties().display_name().unwrap_or_default(), + "record_key": self.record_key.iter().map(|b| format!("{b:02X}")).collect::()}) + } + + fn error(&mut self, folder: &str, node: Option, err: &dyn std::fmt::Display, emit: &mut dyn FnMut(Value)) { + self.stats.errors += 1; + let kind: String = err.to_string().chars().take(80).collect(); + *self.stats.error_kinds.entry(kind.clone()).or_default() += 1; + emit(json!({"type": "error", "folder": folder, "node": node, "error": kind})); + } + + /// Walks the folders Outlook shows (the IPM subtree), calling `emit` with one JSON value per mail. + pub fn walk(&mut self, emit: &mut dyn FnMut(Value)) -> io::Result<()> { + let store = self.backend.store(); + let ipm = store.properties().ipm_sub_tree_entry_id()?; + let root = store.open_folder(&ipm)?; + self.folder(root, "", emit); + Ok(()) + } + + /// Reads the mails with these node ids, in this order, wherever they are (their folder isn't looked up: `list` + /// gave it). A node that's gone, or isn't a mail, gets an error line. + pub fn read(&mut self, nodes: &[u32], emit: &mut dyn FnMut(Value)) { + let store = self.backend.store(); + for &node in nodes { + let (mails, errors) = (self.stats.mails, self.stats.errors); + let opened = store + .properties() + .make_entry_id(NodeId::from(node)) + .and_then(|entry_id| self.backend.open_message(&entry_id)); + match opened { + Ok(message) => self.message(&message, node, "", emit), + Err(err) => self.error("", Some(node), &err, emit), + } + if self.stats.mails == mails && self.stats.errors == errors { + self.error("", Some(node), &"not a mail", emit); + } + } + } + + fn folder(&mut self, folder: Rc, path: &str, emit: &mut dyn FnMut(Value)) { + self.stats.folders += 1; + let store = self.backend.store(); + if let Some(contents) = folder.contents_table() { + let nodes: Vec = contents.rows_matrix().map(|row| u32::from(row.id())).collect(); + for node in nodes { + self.stats.items += 1; + let opened = store + .properties() + .make_entry_id(NodeId::from(node)) + .and_then(|entry_id| self.backend.open_message(&entry_id)); + match opened { + Ok(message) => self.message(&message, node, path, emit), + Err(err) => self.error(path, Some(node), &err, emit), + } + } + } + if let Some(hierarchy) = folder.hierarchy_table() { + let nodes: Vec = hierarchy.rows_matrix().map(|row| u32::from(row.id())).collect(); + for node in nodes { + let sub = store.properties().make_entry_id(NodeId::from(node)).and_then(|id| store.open_folder(&id)); + match sub { + Ok(sub) => { + let name = sub.properties().display_name().unwrap_or_default(); + self.folder(sub, &format!("{path}/{name}"), emit); + } + Err(err) => self.error(path, Some(node), &err, emit), + } + } + } + } + + fn message(&mut self, message: &Rc, node: u32, folder: &str, emit: &mut dyn FnMut(Value)) { + let props = message.properties(); + let get = |id: u16| props.get(id); + let cp = code_page_of(&get); + let class = string(get(PR_MESSAGE_CLASS), cp).unwrap_or_default(); + if !class.to_ascii_uppercase().starts_with("IPM.NOTE") { + self.stats.other_items += 1; + return; + } + self.stats.mails += 1; + let entry_id = entry_id_hex(&self.record_key, node); + let (kind, text) = body(&get); + *self.stats.body_kinds.entry(kind).or_default() += 1; + self.stats.body_chars += text.chars().count(); + let hdrs = headers(&string(get(PR_TRANSPORT_MESSAGE_HEADERS), cp).unwrap_or_default()); + let header_from = hdrs.get("from").map(|v| addresses(v)).unwrap_or_default().into_iter().next(); + let header_from_email = header_from.as_ref().map(|(_, e)| e.clone()).filter(|e| !e.is_empty()); + let sender_email = [PR_SENDER_SMTP_ADDRESS, PR_SENDER_EMAIL_ADDRESS, PR_SENT_REPRESENTING_SMTP_ADDRESS, + PR_SENT_REPRESENTING_EMAIL_ADDRESS] + .iter() + .filter_map(|id| string(get(*id), cp)) + .find(|s| is_address(s)) + .or(header_from_email) + .unwrap_or_default(); + let sender_name = string(get(PR_SENDER_NAME), cp) + .or_else(|| string(get(PR_SENT_REPRESENTING_NAME), cp)) + .filter(|s| !s.is_empty()) + .or_else(|| header_from.as_ref().map(|(n, _)| n.clone()).filter(|n| !n.is_empty())) + .unwrap_or_default(); + let attachments = self.attachments(message, &entry_id, folder, node, emit); + let mut line = json!({ + "type": "mail", + "entry_id": entry_id, + "node": node, + "folder": folder, + "class": class, + "subject": subject(get(PR_SUBJECT), cp), + "sender_name": sender_name, + "sender_email": sender_email, + "recipients": recipients(message.as_ref(), &get, &hdrs, cp), + "sent": time(get(PR_CLIENT_SUBMIT_TIME)), + "received": time(get(PR_MESSAGE_DELIVERY_TIME)), + "modified": time(get(PR_LAST_MODIFICATION_TIME)), + "body_kind": kind, + "attachments": attachments, + }); + if self.options.bodies { + line["body"] = Value::String(text); + } + emit(line); + } + + fn attachments(&mut self, message: &Rc, entry_id: &str, folder: &str, node: u32, + emit: &mut dyn FnMut(Value)) -> Vec { + let Some(table) = message.attachment_table() else { return Vec::new() }; + let rows: Vec = table.rows_matrix().map(|row| u32::from(row.id())).collect(); + let mut out = Vec::new(); + for (n, sub_node) in rows.into_iter().enumerate() { + let attachment = match self.backend.open_attachment(message, NodeId::from(sub_node)) { + Ok(a) => a, + Err(err) => { + self.error(folder, Some(node), &format!("attachment: {err}"), emit); + continue; + } + }; + self.stats.attachments += 1; + let props = attachment.properties(); + let name = string(props.get(PR_ATTACH_LONG_FILENAME), None) + .or_else(|| string(props.get(PR_ATTACH_FILENAME), None)) + .or_else(|| string(props.get(PR_DISPLAY_NAME), None)) + .unwrap_or_default(); + let method = int(props.get(PR_ATTACH_METHOD)).unwrap_or(0); + let content_id = string(props.get(PR_ATTACH_CONTENT_ID), None); + let hidden = matches!(props.get(PR_ATTACHMENT_HIDDEN), Some(PropertyValue::Boolean(true))); + let mut item = json!({"name": name, "method": method, "size": int(props.get(PR_ATTACH_SIZE)), + "inline": hidden || content_id.is_some()}); + match attachment.data() { + Some(AttachmentData::Binary(data)) if method == ATTACH_BY_VALUE => { + item["bytes"] = json!(data.buffer().len()); + if let Some(dir) = self.options.attachments_dir { + let path = dir.join(format!("{entry_id}_{n}{}", safe_name(&name))); + match std::fs::write(&path, data.buffer()) { + Ok(()) => item["path"] = json!(path.to_string_lossy()), + Err(err) => self.error(folder, Some(node), &format!("attachment write: {err}"), emit), + } + } + } + Some(AttachmentData::Message(embedded)) => { + self.stats.embedded_messages += 1; + let eprops = embedded.properties(); + let eget = |id: u16| eprops.get(id); + let ecp = code_page_of(&eget); + let (ekind, etext) = body(&eget); + let mut e = json!({"subject": subject(eget(PR_SUBJECT), ecp), + "sender_name": string(eget(PR_SENDER_NAME), ecp).unwrap_or_default(), + "sent": time(eget(PR_CLIENT_SUBMIT_TIME)), "body_kind": ekind}); + if self.options.bodies { + e["body"] = Value::String(etext); + } + item["embedded"] = e; + } + _ => {} + } + out.push(item); + } + out + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn header_parsing() { + let raw = "Received: from x\r\n\tby y\r\nFrom: \"Dupont, Jean\" \r\nTo: a@b.fr,\r\n Claire ; Bob\r\nSubject: Devis\r\n\r\nbody: no"; + let h = headers(raw); + assert_eq!(addresses(&h["from"]), vec![("Dupont, Jean".to_string(), "jean.dupont@x.fr".to_string())]); + assert_eq!(addresses(&h["to"]), vec![(String::new(), "a@b.fr".to_string()), + ("Claire".to_string(), "claire@c.fr".to_string()), + ("Bob".to_string(), String::new())]); + assert_eq!(h["received"], "from x by y"); + assert!(!h.contains_key("body")); + } + + #[test] + fn addresses_need_an_at_sign() { + assert!(is_address("jean@x.fr")); + assert!(!is_address("Michelle Cash")); + assert!(!is_address("/O=ENRON/OU=NA/CN=RECIPIENTS/CN=MCASH")); + } + + #[test] + fn entry_id_layout() { + let key = [0xFFu8, 0xD6, 0xC3, 0xDB, 0xF5, 0x3B, 0x93, 0x4B, 0xBC, 0x64, 0x68, 0x3F, 0x7D, 0xB2, 0xD9, 0xFE]; + assert_eq!(entry_id_hex(&key, 0x0020_0024), "00000000FFD6C3DBF53B934BBC64683F7DB2D9FE24002000"); + } +} diff --git a/pst/src/text.rs b/pst/src/text.rs new file mode 100644 index 00000000..1570ccdb --- /dev/null +++ b/pst/src/text.rs @@ -0,0 +1,259 @@ +//! Turning MAPI values into text: dates, 8-bit strings, HTML and RTF bodies. + +/// FILETIME (100 ns ticks since 1601-01-01 UTC) as ISO 8601 UTC. Outlook's "no date" (year 4501) +/// and zero are None. +pub fn filetime_to_iso(ticks: i64) -> Option { + if ticks <= 0 { + return None; + } + let secs = ticks / 10_000_000 - 11_644_473_600; + let (days, rem) = (secs.div_euclid(86_400), secs.rem_euclid(86_400)); + let (y, m, d) = civil_from_days(days); + if y >= 4500 { + return None; + } + Some(format!("{y:04}-{m:02}-{d:02}T{:02}:{:02}:{:02}Z", rem / 3600, rem % 3600 / 60, rem % 60)) +} + +/// Days since 1970-01-01 to a proleptic Gregorian date (Howard Hinnant's algorithm). +fn civil_from_days(z: i64) -> (i64, u32, u32) { + let z = z + 719_468; + let era = z.div_euclid(146_097); + let doe = z.rem_euclid(146_097); + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let d = (doy - (153 * mp + 2) / 5 + 1) as u32; + let m = if mp < 10 { mp + 3 } else { mp - 9 } as u32; + let y = yoe + era * 400; + (if m <= 2 { y + 1 } else { y }, m, d) +} + +/// Bytes in a Windows code page (1252 when unknown). +pub fn decode_8bit(bytes: &[u8], code_page: Option) -> String { + let encoding = code_page + .and_then(|cp| u16::try_from(cp).ok()) + .and_then(codepage::to_encoding) + .unwrap_or(encoding_rs::WINDOWS_1252); + encoding.decode(bytes).0.into_owned() +} + +/// HTML as plain text (no styling, no link targets), without hard wrapping. +pub fn html_to_text(html: &str) -> String { + html2text::config::plain_no_decorate() + .link_footnotes(false) + .string_from_read(html.as_bytes(), 100_000) + .unwrap_or_default() + .lines() + .map(str::trim_end) + .collect::>() + .join("\n") + .trim() + .to_string() +} + +/// Destinations whose content is never text. +const SKIPPED_DESTINATIONS: &[&str] = &[ + "fonttbl", "colortbl", "stylesheet", "info", "pict", "object", "themedata", "colorschememapping", + "latentstyles", "datastore", "xmlnstbl", "listtable", "listoverridetable", "rsidtbl", "generator", + "header", "footer", "headerl", "headerr", "footerl", "footerr", "fldinst", "filetbl", "revtbl", +]; + +/// RTF as plain text. Ignorable destinations (`{\*...}`) are skipped, which also drops the HTML +/// tags of RTF that encapsulates HTML (`\fromhtml1`) and keeps its readable text. +pub fn rtf_to_text(rtf: &str) -> String { + #[derive(Clone, Copy)] + struct Group { + skip: bool, + uc: usize, + } + let bytes = rtf.as_bytes(); + let mut out = String::new(); + let mut pending: Vec = Vec::new(); // \'hh bytes, decoded together (multi-byte code pages) + let mut code_page: Option = None; + let mut stack = vec![Group { skip: false, uc: 1 }]; + let mut fallback_skip = 0usize; // characters to drop after a \uN + let mut i = 0; + + fn flush(out: &mut String, pending: &mut Vec, code_page: Option) { + if !pending.is_empty() { + out.push_str(&decode_8bit(pending, code_page)); + pending.clear(); + } + } + + while i < bytes.len() { + let skip = stack.last().map(|g| g.skip).unwrap_or(false); + match bytes[i] { + b'{' => { + let top = *stack.last().unwrap_or(&Group { skip: false, uc: 1 }); + stack.push(top); + i += 1; + } + b'}' => { + if stack.len() > 1 { + stack.pop(); + } + i += 1; + } + b'\\' if i + 1 < bytes.len() => { + let c = bytes[i + 1]; + if c == b'\'' && i + 3 < bytes.len() { + if let Ok(byte) = u8::from_str_radix(&rtf[i + 2..i + 4], 16) { + if fallback_skip > 0 { + fallback_skip -= 1; + } else if !skip { + pending.push(byte); + } + } + i += 4; + } else if c.is_ascii_alphabetic() { + let start = i + 1; + let mut j = start; + while j < bytes.len() && bytes[j].is_ascii_alphabetic() { + j += 1; + } + let word = &rtf[start..j]; + let num_start = j; + if j < bytes.len() && bytes[j] == b'-' { + j += 1; + } + while j < bytes.len() && bytes[j].is_ascii_digit() { + j += 1; + } + let param: Option = rtf[num_start..j].parse().ok(); + if j < bytes.len() && bytes[j] == b' ' { + j += 1; + } + i = j; + if !word.starts_with("u") || word == "uc" || param.is_none() { + flush(&mut out, &mut pending, code_page); + } + match word { + "ansicpg" => code_page = param.map(|p| p as u32), + "par" | "line" | "sect" | "row" if !skip => out.push('\n'), + "tab" | "cell" if !skip => out.push('\t'), + "emdash" if !skip => out.push('—'), + "endash" if !skip => out.push('–'), + "bullet" if !skip => out.push('•'), + "lquote" | "rquote" if !skip => out.push('\''), + "ldblquote" | "rdblquote" if !skip => out.push('"'), + "uc" => { + if let (Some(g), Some(p)) = (stack.last_mut(), param) { + g.uc = p.max(0) as usize; + } + } + "u" => { + if let Some(p) = param { + if !skip { + flush(&mut out, &mut pending, code_page); + let code = if p < 0 { p + 65_536 } else { p } as u32; + if let Some(ch) = char::from_u32(code) { + out.push(ch); + } + } + fallback_skip = stack.last().map(|g| g.uc).unwrap_or(1); + } + } + w if SKIPPED_DESTINATIONS.contains(&w) => { + if let Some(g) = stack.last_mut() { + g.skip = true; + } + } + _ => {} + } + } else { + flush(&mut out, &mut pending, code_page); + match c { + b'*' => { + if let Some(g) = stack.last_mut() { + g.skip = true; + } + } + b'\\' | b'{' | b'}' if !skip => out.push(c as char), + b'~' if !skip => out.push(' '), + b'\n' | b'\r' if !skip => out.push('\n'), + _ => {} + } + i += 2; + } + } + b'\r' | b'\n' => i += 1, + _ => { + // One UTF-8 character of plain text. + let ch = rtf[i..].chars().next().unwrap_or(' '); + if fallback_skip > 0 { + fallback_skip -= 1; + } else if !skip { + flush(&mut out, &mut pending, code_page); + out.push(ch); + } + i += ch.len_utf8(); + } + } + } + flush(&mut out, &mut pending, code_page); + collapse_blank_lines(&out) +} + +/// Trim line ends and keep at most one empty line in a row. +pub fn collapse_blank_lines(text: &str) -> String { + let mut out = Vec::new(); + let mut blank = 0; + for line in text.lines().map(str::trim_end) { + if line.trim().is_empty() { + blank += 1; + if blank > 1 { + continue; + } + } else { + blank = 0; + } + out.push(line); + } + out.join("\n").trim().to_string() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn filetimes() { + // 2000-06-09T14:45:00Z + assert_eq!(filetime_to_iso(126_050_355_000_000_000).as_deref(), Some("2000-06-09T14:45:00Z")); + assert_eq!(filetime_to_iso(0), None); + // 4501-01-01, Outlook's "none" + assert_eq!(filetime_to_iso(915_151_392_000_000_000), None); + } + + #[test] + fn code_pages() { + assert_eq!(decode_8bit(b"r\xe9union", None), "réunion"); + assert_eq!(decode_8bit(b"r\xc3\xa9union", Some(65001)), "réunion"); + assert_eq!(decode_8bit(&[0x82, 0xa0], Some(932)), "あ"); + } + + #[test] + fn html() { + let text = html_to_text("

Bonjour Jean,

\ +

Voir le devis.

"); + assert!(text.contains("Bonjour Jean,"), "{text}"); + assert!(text.contains("le devis"), "{text}"); + assert!(!text.contains("color"), "{text}"); + } + + #[test] + fn rtf_plain() { + let rtf = r"{\rtf1\ansi\ansicpg1252{\fonttbl{\f0 Calibri;}}{\colortbl;\red0\green0\blue0;}\f0 Bonjour,\par Le devis est valid\'e9.\par\pard \u8364? 42\par}"; + assert_eq!(rtf_to_text(rtf), "Bonjour,\nLe devis est validé.\n€ 42"); + } + + #[test] + fn rtf_encapsulated_html() { + let rtf = r"{\rtf1\ansi\ansicpg1252\fromhtml1 {\*\htmltag19 }{\*\htmltag50 }\htmlrtf {\htmlrtf0 Bonjour\htmlrtf }{\*\htmltag84
}\htmlrtf \par \htmlrtf0 Le devis\htmlrtf }{\*\htmltag58 }}"; + let text = rtf_to_text(rtf); + assert!(text.contains("Bonjour") && text.contains("Le devis"), "{text}"); + assert!(!text.contains('<'), "{text}"); + } +} diff --git a/pst/tests/data/outlook_made.expected.tsv b/pst/tests/data/outlook_made.expected.tsv new file mode 100644 index 00000000..6a8a8484 --- /dev/null +++ b/pst/tests/data/outlook_made.expected.tsv @@ -0,0 +1,31 @@ +# EntryID, subject and folder of each mail, as listed by classic Outlook 16.0.0.20430 (Folder.GetTable) on 2026-10-14 +00000000FFD6C3DBF53B934BBC64683F7DB2D9FE24002000 PST test 00 /Boite de test +00000000FFD6C3DBF53B934BBC64683F7DB2D9FE44002000 PST test 01 /Clients +00000000FFD6C3DBF53B934BBC64683F7DB2D9FE64002000 PST test 02 /Clients/Dupont +00000000FFD6C3DBF53B934BBC64683F7DB2D9FE84002000 PST test 03 /Boite de test +00000000FFD6C3DBF53B934BBC64683F7DB2D9FEA4002000 PST test 04 /Clients +00000000FFD6C3DBF53B934BBC64683F7DB2D9FEC4002000 PST test 05 /Clients/Dupont +00000000FFD6C3DBF53B934BBC64683F7DB2D9FEE4002000 PST test 06 /Boite de test +00000000FFD6C3DBF53B934BBC64683F7DB2D9FE04012000 PST test 07 /Clients +00000000FFD6C3DBF53B934BBC64683F7DB2D9FE24012000 PST test 08 /Clients/Dupont +00000000FFD6C3DBF53B934BBC64683F7DB2D9FE44012000 PST test 09 /Boite de test +00000000FFD6C3DBF53B934BBC64683F7DB2D9FE64012000 PST test 10 /Clients +00000000FFD6C3DBF53B934BBC64683F7DB2D9FE84012000 PST test 11 /Clients/Dupont +00000000FFD6C3DBF53B934BBC64683F7DB2D9FEA4012000 PST test 12 /Boite de test +00000000FFD6C3DBF53B934BBC64683F7DB2D9FEC4012000 PST test 13 /Clients +00000000FFD6C3DBF53B934BBC64683F7DB2D9FEE4012000 PST test 14 /Clients/Dupont +00000000FFD6C3DBF53B934BBC64683F7DB2D9FE04022000 PST test 15 /Boite de test +00000000FFD6C3DBF53B934BBC64683F7DB2D9FE24022000 PST test 16 /Clients +00000000FFD6C3DBF53B934BBC64683F7DB2D9FE44022000 PST test 17 /Clients/Dupont +00000000FFD6C3DBF53B934BBC64683F7DB2D9FE64022000 PST test 18 /Boite de test +00000000FFD6C3DBF53B934BBC64683F7DB2D9FE84022000 PST test 19 /Clients +00000000FFD6C3DBF53B934BBC64683F7DB2D9FEA4022000 PST test 20 /Clients/Dupont +00000000FFD6C3DBF53B934BBC64683F7DB2D9FEC4022000 PST test 21 /Boite de test +00000000FFD6C3DBF53B934BBC64683F7DB2D9FEE4022000 PST test 22 /Clients +00000000FFD6C3DBF53B934BBC64683F7DB2D9FE04032000 PST test 23 /Clients/Dupont +00000000FFD6C3DBF53B934BBC64683F7DB2D9FE24032000 PST test 24 /Boite de test +00000000FFD6C3DBF53B934BBC64683F7DB2D9FE44032000 PST test 25 /Clients +00000000FFD6C3DBF53B934BBC64683F7DB2D9FE64032000 PST test 26 /Clients/Dupont +00000000FFD6C3DBF53B934BBC64683F7DB2D9FE84032000 PST test 27 /Boite de test +00000000FFD6C3DBF53B934BBC64683F7DB2D9FEA4032000 PST test 28 /Clients +00000000FFD6C3DBF53B934BBC64683F7DB2D9FEC4032000 PST test 29 /Clients/Dupont diff --git a/pst/tests/data/outlook_made.pst b/pst/tests/data/outlook_made.pst new file mode 100644 index 00000000..b503d804 Binary files /dev/null and b/pst/tests/data/outlook_made.pst differ diff --git a/pst/tests/outlook_made.rs b/pst/tests/outlook_made.rs new file mode 100644 index 00000000..91a3fdf3 --- /dev/null +++ b/pst/tests/outlook_made.rs @@ -0,0 +1,110 @@ +//! Reads tests/data/outlook_made.pst, a PST that classic Outlook 16.0 created (30 synthetic mails in three folders, 5 +//! attachments, plain and HTML bodies), and checks it against what Outlook itself listed (outlook_made.expected.tsv). + +use std::{collections::BTreeMap, path::PathBuf, process::Command}; + +use serde_json::Value; + +fn data(name: &str) -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/data").join(name) +} + +fn run(args: &[&str]) -> (Option, Vec) { + let output = Command::new(env!("CARGO_BIN_EXE_digup-pst")).args(args).output().expect("run digup-pst"); + let lines = String::from_utf8(output.stdout) + .unwrap() + .lines() + .map(|l| serde_json::from_str(l).expect("one JSON value per line")) + .collect(); + (output.status.code(), lines) +} + +fn mails(lines: &[Value]) -> BTreeMap { + lines + .iter() + .filter(|l| l["type"] == "mail") + .map(|l| (l["entry_id"].as_str().unwrap().to_string(), l.clone())) + .collect() +} + +fn pst() -> String { + data("outlook_made.pst").to_string_lossy().to_string() +} + +#[test] +fn list_matches_outlook() { + let (code, lines) = run(&["list", &pst()]); + assert_eq!(code, Some(0)); + let mails = mails(&lines); + let expected: Vec> = std::fs::read_to_string(data("outlook_made.expected.tsv")) + .unwrap() + .lines() + .filter(|l| !l.starts_with('#')) + .map(|l| l.split('\t').map(String::from).collect()) + .collect(); + assert_eq!(expected.len(), 30); + assert_eq!(mails.len(), 30); + for row in &expected { + let mail = mails.get(&row[0]).unwrap_or_else(|| panic!("missing {}", row[0])); + assert_eq!(mail["subject"], row[1].as_str()); + assert_eq!(mail["folder"], row[2].as_str()); + assert!(mail.get("body").is_none()); + assert!(mail["node"].as_u64().is_some()); + } + let summary = lines.last().unwrap(); + assert_eq!(summary["type"], "summary"); + assert_eq!(summary["mails"], 30); + assert_eq!(summary["errors"], 0); +} + +#[test] +fn read_gives_plain_text_bodies_and_attachments() { + let (_, listed) = run(&["list", &pst()]); + let nodes: BTreeMap = listed + .iter() + .filter(|l| l["type"] == "mail") + .map(|l| (l["subject"].as_str().unwrap().to_string(), l["node"].as_u64().unwrap())) + .collect(); + let dir = std::env::temp_dir().join(format!("digup-pst-test-{}", std::process::id())); + let (seven, twelve) = (nodes["PST test 07"].to_string(), nodes["PST test 12"].to_string()); + let (code, lines) = run(&["read", &pst(), &seven, &twelve, "--attachments", dir.to_str().unwrap()]); + assert_eq!(code, Some(0)); + let read: Vec<&Value> = lines.iter().filter(|l| l["type"] == "mail").collect(); + assert_eq!(read.len(), 2); + assert_eq!(read[0]["subject"], "PST test 07"); // in the order asked + assert_eq!(read[1]["subject"], "PST test 12"); + for mail in &read { + // Outlook stored every body as HTML, even the ones written as plain text. + assert_eq!(mail["body_kind"], "html"); + let body = mail["body"].as_str().unwrap(); + assert!(body.contains("reference DEV-2024-00"), "{body}"); + assert!(!body.contains('<'), "{body}"); + } + let attachment = &read[0]["attachments"][0]; // mails 0, 7, 14, 21 and 28 have one + assert_eq!(attachment["name"], "piece_jointe.txt"); + let saved = std::fs::read_to_string(attachment["path"].as_str().unwrap()).unwrap(); + assert_eq!(saved, "Contenu de la piece jointe de test."); + assert!(read[1]["attachments"].as_array().unwrap().is_empty()); + std::fs::remove_dir_all(dir).ok(); +} + +#[test] +fn read_reports_nodes_that_are_not_mails() { + let (code, lines) = run(&["read", &pst(), "99"]); + assert_eq!(code, Some(0)); + assert_eq!(lines.iter().filter(|l| l["type"] == "error" && l["node"] == 99).count(), 1); + assert_eq!(lines.last().unwrap()["mails"], 0); +} + +#[test] +fn missing_file_and_bad_arguments_fail_cleanly() { + assert_eq!(run(&["list", &data("absent.pst").to_string_lossy()]).0, Some(1)); + assert_eq!(run(&["read", &pst(), "seven"]).0, Some(1)); + assert_eq!(run(&["frobnicate", &pst()]).0, Some(1)); +} + +#[cfg(not(windows))] +#[test] +fn opening_needs_windows() { + assert_eq!(run(&["open", &pst(), "2097412"]).0, Some(4)); +} diff --git a/scripts/check-windows-package.sh b/scripts/check-windows-package.sh index 348eb51b..5a95bfb0 100755 --- a/scripts/check-windows-package.sh +++ b/scripts/check-windows-package.sh @@ -18,7 +18,7 @@ win() { cygpath -m "$1"; } echo "installing $setup" MSYS_NO_PATHCONV=1 "$setup" /S installed=$(cygpath -u "$LOCALAPPDATA")/DigUp -ls "$installed/DigUp.exe" "$installed/helper/digup.exe" > /dev/null +ls "$installed/DigUp.exe" "$installed/helper/digup.exe" "$installed/helper/digup-pst.exe" > /dev/null # The licenses of what the installer carries, beside the app. grep -q "GNU LESSER GENERAL PUBLIC LICENSE" "$installed/THIRD-PARTY-NOTICES.txt" || { echo "no license notices"; exit 1; } @@ -72,4 +72,9 @@ PATH="/c/Windows/System32:/c/Windows:/c/Windows/System32/Wbem" "$helper" status grep -Eq '^vectors: 5 in 5 segments' "$work/status.txt" || { echo "not every file has a vector"; exit 1; } PATH="/c/Windows/System32:/c/Windows:/c/Windows/System32/Wbem" DIGUP_MODELS="$models" \ "$helper" search "how long to bake the cake" --index "$(win "$work/index")" -n 3 + +echo "digup-pst with only Windows on PATH, on the PST Outlook made" +PATH="/c/Windows/System32:/c/Windows:/c/Windows/System32/Wbem" "$installed/helper/digup-pst.exe" list \ + "$(win pst/tests/data/outlook_made.pst)" | tail -1 | tee "$work/pst.txt" +grep -q '"mails":30' "$work/pst.txt" || { echo "digup-pst didn't read the test PST"; exit 1; } echo "ok" diff --git a/scripts/notices.py b/scripts/notices.py index 26baf860..7ac776e5 100644 --- a/scripts/notices.py +++ b/scripts/notices.py @@ -2,7 +2,8 @@ """Writes the third-party notices DigUp's Windows and Linux packages carry: the license of every library they ship, from the libraries' own license files (llama.cpp's and PDFium's from Vendor/, ffmpeg's, dav1d's and zlib's from the small ffmpeg scripts/build-ffmpeg.sh makes, Swift's from the toolchain or scripts/licenses), and of every Rust crate the app -is built from (Cargo's own record of them, normal dependencies only: build tools and macros don't end up in the app). +and, on Windows, digup-pst (pst/) are built from (Cargo's own record of them, normal dependencies only: build tools and +macros don't end up in the programs). scripts/notices.py linux|windows @@ -69,13 +70,13 @@ def miniz_license(): sys.exit("notices: Vendor/miniz/LICENSE is missing (run scripts/fetch-vendor.sh)") -def rust_crates(platform): - """The crates in the app's binary for the platform: Cargo's resolve graph from the app, through normal - dependencies, without proc-macro crates (they run while compiling).""" +def rust_crates(platform, folder): + """The crates in a program's binary for the platform: Cargo's resolve graph from the crate in `folder`, through + normal dependencies, without proc-macro crates (they run while compiling).""" target = {"linux": "x86_64-unknown-linux-gnu", "windows": "x86_64-pc-windows-msvc"}[platform] metadata = json.loads(subprocess.run( ["cargo", "metadata", "--format-version", "1", "--locked", "--filter-platform", target], - cwd=ROOT / "shell/src-tauri", check=True, capture_output=True, encoding="utf-8").stdout) + cwd=ROOT / folder, check=True, capture_output=True, encoding="utf-8").stdout) packages = {package["id"]: package for package in metadata["packages"]} nodes = {node["id"]: node for node in metadata["resolve"]["nodes"]} root = metadata["resolve"]["root"] @@ -96,23 +97,19 @@ def rust_crates(platform): return sorted((packages[i] for i in seen), key=lambda package: (package["name"], package["version"])) -def rust_section(platform): - crates = rust_crates(platform) +def rust_section(platform, folder, title, intro): + crates = rust_crates(platform, folder) by_text = {} # a license file's text, its spacing aside → the text and the crates that ship it (printed once) without = [] for crate in crates: - folder = Path(crate["manifest_path"]).parent - files = sorted(path for path in folder.iterdir() if path.is_file() and LICENSE_FILE.match(path.name)) + files = license_files(Path(crate["manifest_path"]).parent) label = f"{crate['name']} {crate['version']} ({crate.get('license') or 'see its files'})" if not files: without.append((label, crate)) for path in files: text = read(path) by_text.setdefault(" ".join(text.split()), (text, []))[1].append(label) - parts = [RULE, f"Rust crates in the app ({len(crates)}), each under its own license", RULE, "", - "The app around DigUp's search on Windows and Linux is written in Rust with Tauri (https://tauri.app). " - "These are the crates it is built from, with the license files they ship; a text several crates share " - "is printed once, after their names.", ""] + parts = [RULE, f"{title} ({len(crates)}), each under its own license", RULE, "", intro, ""] for text, labels in sorted(by_text.values(), key=lambda item: item[1][0]): parts += ["-" * 80, *labels, "-" * 80, "", text, ""] if without: @@ -126,6 +123,15 @@ def rust_section(platform): return "\n".join(parts) +def license_files(folder): + """A crate's license files; for a crate of a workspace that keeps them at its top (outlook-pst), the workspace's.""" + for place in [folder, folder.parent, folder.parent.parent]: + files = sorted(path for path in place.iterdir() if path.is_file() and LICENSE_FILE.match(path.name)) + if files or (place / "Cargo.lock").is_file(): + return files + return [] + + def main(): if len(sys.argv) != 3 or sys.argv[1] not in ("linux", "windows"): sys.exit(__doc__) @@ -193,7 +199,17 @@ def main(): "its copyright file in usr/share/doc inside the AppImage. " "Their source is Ubuntu's: https://packages.ubuntu.com/jammy. The .deb uses your system's " "own copies instead.")] - parts.append(rust_section(platform)) + parts.append(rust_section( + platform, "shell/src-tauri", "Rust crates in the app", + "The app around DigUp's search on Windows and Linux is written in Rust with Tauri (https://tauri.app). These " + "are the crates it is built from, with the license files they ship; a text several crates share is printed " + "once, after their names.")) + if windows: + parts.append(rust_section( + platform, "pst", "Rust crates in digup-pst", + "digup-pst, in DigUp's helper folder, reads Outlook's .pst files with Microsoft's outlook-pst " + "(https://github.com/microsoft/outlook-pst-rs) and opens a mail in Outlook. It is written in Rust too; " + "these are the crates it is built from, with the license files they ship.")) out.write_text("\n".join(parts) + "\n", encoding="utf-8") print(f"{out}: {out.stat().st_size // 1024} KB") diff --git a/scripts/package-windows.sh b/scripts/package-windows.sh index bd618bc3..051c9958 100755 --- a/scripts/package-windows.sh +++ b/scripts/package-windows.sh @@ -1,6 +1,7 @@ #!/bin/bash # Builds DigUp's Windows installer: the Tauri shell (shell/) with DigUp's helper, llama.cpp, PDFium, the Swift and C++ -# runtimes and ffmpeg in its helper folder, as an NSIS installer that installs for the current user (no admin prompt). +# runtimes, ffmpeg and digup-pst (pst/, Outlook's .pst files) in its helper folder, as an NSIS installer that installs +# for the current user (no admin prompt). # Run it in Git for Windows' bash inside a Visual Studio developer environment, after scripts/fetch-vendor.sh, # scripts/build-llama.sh and scripts/build-ffmpeg.sh (that one in MSYS2's MINGW64 shell). Needs Swift, Rust, Python and # Node (for Tauri's CLI): @@ -11,6 +12,7 @@ cd "$(dirname "$0")/.." TAURI_CLI=2.12.2 swift build -c release --product digup +(cd pst && cargo build --release --locked) # digup-pst: Outlook's .pst files # What the installer puts beside DigUp.exe, in helper\ ("DigUp.exe" and "digup.exe" can't share a folder): the helper # finds the DLLs, llama.cpp's backends and ffmpeg in its own folder. The C++ runtime goes with them, so nothing needs @@ -25,6 +27,7 @@ cp .build/release/digup.exe Vendor/llama/lib/*.dll Vendor/pdfium/bin/pdfium.dll cp "$swift_runtime"/*.dll "$helper/" cp "$crt"/*.dll "$helper/" # over the Swift runtime's copies of the same: the newer C++ runtime cp Vendor/ffmpeg/bin/ffmpeg.exe Vendor/ffmpeg/bin/ffprobe.exe "$helper/" # the small static build, no DLLs +cp pst/target/release/digup-pst.exe "$helper/" # With the release's signing key (TAURI_SIGNING_PRIVATE_KEY, and its password) the packages that update themselves are # signed for in-app updates (a .sig beside each). Installed copies check those signatures with the public key in diff --git a/shell/src-tauri/src/main.rs b/shell/src-tauri/src/main.rs index 034ffd45..d3cfd4d7 100644 --- a/shell/src-tauri/src/main.rs +++ b/shell/src-tauri/src/main.rs @@ -412,6 +412,35 @@ fn open_code(app: AppHandle, shell: State<'_, Arc>, path: String, line: u app.opener().open_path(path, None::<&str>).map_err(|error| error.to_string()) } +/// Opens a mail result in classic Outlook: digup-pst, beside the helper, asks Outlook to show it (starting Outlook when +/// it isn't running). The path is the mail's PST and its node id there ("…\archive.pst::2097412"). +#[tauri::command] +async fn open_mail(app: AppHandle, shell: State<'_, Arc>, path: String) -> Result<(), String> { + let (pst, node) = path.rsplit_once("::").map(|(pst, node)| (pst.to_string(), node.to_string())).ok_or("not a mail")?; + let tool = shell.options.helper.with_file_name(if cfg!(windows) { "digup-pst.exe" } else { "digup-pst" }); + hide_panel(app); + let output = tauri::async_runtime::spawn_blocking(move || { + let mut command = std::process::Command::new(&tool); + command.args(["open", &pst, &node]).stdin(std::process::Stdio::null()); + #[cfg(windows)] + { + use std::os::windows::process::CommandExt; + command.creation_flags(0x0800_0000); // CREATE_NO_WINDOW + } + command.output().map_err(|error| format!("can't run {}: {error}", tool.display())) + }) + .await + .map_err(|error| error.to_string())??; + match output.status.code() { + Some(0) => Ok(()), + Some(4) => Err("Opening a mail needs classic Outlook.".into()), + Some(5) => Err("This PST isn't open in Outlook. Open it there (File › Open & Export › Open Outlook Data File), \ + then try again.".into()), + Some(6) => Err("Outlook can't find this mail in the PST any more.".into()), + _ => Err(format!("Outlook didn't show the mail: {}", String::from_utf8_lossy(&output.stderr).trim())), + } +} + /// Ctrl+C on a result: the file on the clipboard, to paste in a file manager, a mail or a chat. #[tauri::command] fn copy_file(window: tauri::WebviewWindow, path: String) -> Result<(), String> { @@ -645,7 +674,8 @@ fn main() { ) .setup(move |app| setup(app, options)) .invoke_handler(tauri::generate_handler![ - ask, estimate, setup_info, pick_folders, apply_setup_folders, finish_setup, open_path, open_code, reveal, hide_panel, panel_height, + ask, estimate, setup_info, pick_folders, apply_setup_folders, finish_setup, open_path, open_code, open_mail, + reveal, hide_panel, panel_height, panel_area, settings_info, set_open_at_login, set_center_panel, set_processor_only, debug_query, debug_report, window_info, app_state, open_window, open_settings, choose_folders, index_now, quit_app, close_window, pause_indexing, remove_folder, set_skipped, set_skip_types, set_index_hours, code_setup_info, set_code_folders, choose_code_folders, diff --git a/shell/ui/panel.js b/shell/ui/panel.js index d6e415ab..944d0111 100644 --- a/shell/ui/panel.js +++ b/shell/ui/panel.js @@ -141,7 +141,7 @@ function openSelected() { function revealSelected() { const hit = selectedHit(); - if (hit) invoke("reveal", { path: hit.path }).catch(failed); + if (hit) invoke("reveal", { path: fileOf(hit) }).catch(failed); } /// The same search in the window, for a bigger look (only one kind, if given). @@ -218,7 +218,7 @@ function row(hit) { item.append(thumbnail(hit)); const text = element("div", "row-text"); const title = element("div", "row-title"); - title.append(element("span", "name", nameOf(hit.path))); + title.append(element("span", "name", titleOf(hit))); const place = placeOf(hit); if (place) title.append(element("span", "badge", place)); const sub = element("div", "row-sub"); @@ -332,9 +332,13 @@ function picture(hit) { } default: { const passage = element("div", "passage" + (hit.kind === "code" ? " code" : "")); - passage.textContent = (hit.excerpt ?? "").replace(/[«»]/g, ""); - ask({ op: "text", segment: hit.segment_id, code: state.isCode }) - .then((reply) => { if (reply.text) passage.innerHTML = highlighted(reply.text, state.text); }) + const nameOnly = hit.kind === "mail" && hit.segment !== "chunk"; + passage.textContent = nameOnly ? "" : (hit.excerpt ?? "").replace(/[«»]/g, ""); + previewText(hit, state.isCode) + .then((reply) => { + if (reply.text) passage.innerHTML = highlighted(reply.text, state.text); + else if (nameOnly) passage.textContent = "Not read yet: found by its subject."; + }) .catch(() => {}); return passage; } @@ -343,7 +347,7 @@ function picture(hit) { function facts(hit) { const box = element("div", "facts"); - box.append(element("div", "title", nameOf(hit.path)), element("div", "line", folderOf(hit)), + box.append(element("div", "title", titleOf(hit)), element("div", "line", folderOf(hit)), element("div", "line", details(hit))); const reason = element("div", "reason"); reason.innerHTML = ICONS.found; @@ -405,7 +409,7 @@ function renderBig() { const hit = selectedHit(); if (!hit) return closeBig(); const head = element("div", "head"); - head.append(element("span", "name", nameOf(hit.path))); + head.append(element("span", "name", titleOf(hit))); const place = placeOf(hit); if (place) head.append(element("span", "place", place)); let view; diff --git a/shell/ui/results.js b/shell/ui/results.js index 63d39e96..7689987d 100644 --- a/shell/ui/results.js +++ b/shell/ui/results.js @@ -16,9 +16,9 @@ const FILE_MANAGER = isMac ? "Finder" : isWindows ? "Explorer" : "Files"; // Pictures every web view shows as they are; anything else (HEIC, TIFF, RAW…) comes as JPEG from the helper. const SHOWN_AS_IS = new Set(["jpg", "jpeg", "png", "gif", "webp", "bmp"]); const GROUP_OF = { screenshot: "Screenshots", image: "Images", pdf: "Documents", doc: "Documents", video: "Video", - audio: "Audio", code: "Code" }; + audio: "Audio", code: "Code", mail: "Mail" }; const KIND_TITLE = { screenshot: "Screenshot", image: "Image", pdf: "PDF", doc: "Document", audio: "Audio", - video: "Video", code: "Code" }; + video: "Video", code: "Code", mail: "Mail" }; /// The home folder, for "~/Pictures" (filled in as the page starts). let homeFolder = ""; @@ -45,6 +45,7 @@ function placeOf(hit) { /// default apps (where those apps can be told), anything else plainly. function openResult(hit) { if (hit.kind === "code") return invoke("open_code", { path: hit.path, line: codeLine(hit) }); + if (hit.kind === "mail") return invoke("open_mail", { path: hit.path }); return invoke("open_path", { path: hit.path, ...placeToOpen(hit) }); } @@ -55,7 +56,7 @@ function copyResult(event, hit, navigated) { const fieldText = field?.selectionStart != null && field.selectionStart !== field.selectionEnd; // (WebKit may count the field's selected text in the page's.) const textSelected = fieldText ? !navigated : Boolean(window.getSelection()?.toString()); - if (!hit || textSelected) return false; + if (!hit || textSelected || hit.kind === "mail") return false; // a mail is no file of its own event.preventDefault(); invoke("copy_file", { path: hit.path }).catch(() => {}); return true; @@ -113,9 +114,19 @@ function reasonOf(hit) { } } -/// "Screenshot · Oct 7, 2026 · 1440 × 900 · 62 KB", "PDF · … · 463 pages, 120 read so far · 23 MB" +/// "Screenshot · Oct 7, 2026 · 1440 × 900 · 62 KB", "PDF · … · 463 pages, 120 read so far · 23 MB", +/// "Mail · Jean Dupont · Mar 3, 2024 · 2 attachments" function details(hit) { const info = hit.info || {}; + if (hit.kind === "mail") { + const parts = ["Mail"]; + if (info.from) parts.push(info.from); + const sent = Date.parse(info.date ?? ""); + parts.push(dateOf(Number.isNaN(sent) ? hit.modified : sent / 1000)); + const attached = info.attachments ? info.attachments.split(", ").length : 0; + if (attached) parts.push(attached === 1 ? "1 attachment" : `${attached} attachments`); + return parts.join(" · "); + } const parts = [KIND_TITLE[hit.kind] || "File", dateOf(hit.modified)]; if (info.width && info.height) parts.push(`${info.width} × ${info.height}`); const readTo = info.read_to != null ? Number(info.read_to) : null; @@ -133,14 +144,27 @@ function nameOf(path) { return path.split(/[\\/]/).pop(); } +/// What a result is called: a file's name, a mail's subject. +function titleOf(hit) { + if (hit.kind === "mail") return hit.info?.subject || "(no subject)"; + return nameOf(hit.path); +} + +/// The file a result is in: itself, or a mail's PST ("…\archive.pst::2097412" is a mail of archive.pst). +function fileOf(hit) { + return hit.kind === "mail" ? hit.path.slice(0, hit.path.lastIndexOf("::")) : hit.path; +} + function extensionOf(path) { const name = nameOf(path); const dot = name.lastIndexOf("."); return dot > 0 ? name.slice(dot + 1).toLowerCase() : ""; } -/// Where the file is: "~/Downloads/Screenshots"; code says where it is in its repo: "billing/payments". +/// Where the file is: "~/Downloads/Screenshots"; code says where it is in its repo: "billing/payments"; a mail, its PST +/// and its folder there: "archive.pst › /Inbox/Clients". function folderOf(hit) { + if (hit.kind === "mail") return [nameOf(fileOf(hit)), hit.info?.folder].filter(Boolean).join(" › "); const folder = hit.path.slice(0, Math.max(0, hit.path.search(/[\\/][^\\/]*$/))); const repo = hit.info?.repo; if (hit.kind === "code" && repo && hit.path.startsWith(repo + "/")) { @@ -237,6 +261,13 @@ function kindIcon(kind) { return icon; } +/// The text a preview shows: the passage that matched, or for a mail found by its subject or folder, its beginning +/// (subject, sender, recipients, date, then the body). Null text: the mail isn't read yet. +function previewText(hit, code = false) { + if (hit.kind === "mail" && hit.segment !== "chunk") return ask({ op: "text", path: hit.path }); + return ask({ op: "text", segment: hit.segment_id, code }); +} + /// The helper's JPEG of a page, a frame or a picture, asked for once per file and place. const renders = new Map(); function rendered(path, kind, loc, pixels) { @@ -266,4 +297,5 @@ const ICONS = { video: '', image: '', screenshot: '', + mail: '', }; diff --git a/shell/ui/window.js b/shell/ui/window.js index db28c086..74b686c0 100644 --- a/shell/ui/window.js +++ b/shell/ui/window.js @@ -14,17 +14,19 @@ const FILTERS = [ { key: "video", title: "Video", kinds: ["video"], icon: "video" }, { key: "audio", title: "Audio", kinds: ["audio"], icon: "audio" }, ]; +/// Shown when the index has mail (Outlook .pst files, on Windows), after the others: Ctrl+8. +const MAIL_FILTER = { key: "mail", title: "Mail", kinds: ["mail"], icon: "mail" }; const FILTER_OF_KIND = { screenshot: "screenshots", image: "images", pdf: "documents", doc: "documents", video: "video", - audio: "audio" }; + audio: "audio", mail: "mail" }; const EXAMPLES = [ ["a screenshot of an error message", "screenshots"], ["receipt", "documents"], ["whiteboard notes", "images"], ["someone laughing", "audio"], ["a dog on the beach", "video"], ["slides about the budget", "documents"], ]; /// What the home counts, in this order: "136 screenshots, 104 images, 25 PDFs…" -const KIND_ORDER = ["screenshot", "image", "pdf", "doc", "audio", "video", "code"]; +const KIND_ORDER = ["screenshot", "image", "pdf", "doc", "mail", "audio", "video", "code"]; const COUNTED = { screenshot: ["screenshot", "screenshots"], image: ["image", "images"], pdf: ["PDF", "PDFs"], doc: ["document", "documents"], audio: ["audio file", "audio files"], video: ["video", "videos"], - code: ["code file", "code files"] }; + code: ["code file", "code files"], mail: ["mail", "mails"] }; const SEARCH_ICON = ''; const $ = (id) => document.getElementById(id); @@ -80,7 +82,7 @@ function run() { // Results of an earlier query stay until these are in; with none to show, a spinner. renderBar(); if (!state.hasResults) renderContent(); - const filter = FILTERS.find((item) => item.key === state.filter); + const filter = filters().find((item) => item.key === state.filter); state.timer = setTimeout(async () => { try { const all = await ask({ op: "search", q: text, code, meaning: true, limit: LIMIT }); @@ -161,6 +163,11 @@ function renderBar() { $kinds.append(chip(filter.title, filter.icon, code ? null : state.counts[filter.key], selected, () => setFilter(selected ? null : filter.key), index + 2)); }); + if (hasMail()) { + const selected = state.filter === MAIL_FILTER.key && !code; + $kinds.append(chip(MAIL_FILTER.title, MAIL_FILTER.icon, code ? null : state.counts.mail, selected, + () => setFilter(selected ? null : MAIL_FILTER.key), 8)); + } if (state.info.hasCode) { $kinds.append(element("span", "divider")); $kinds.append(chip("Code", "code", code ? state.counts.code : null, code, @@ -169,6 +176,15 @@ function renderBar() { } /// All · Screenshots 4 · Images 9 · …: what the query found of each kind; a click shows only that kind. +function hasMail() { + return (state.indexed?.mail ?? 0) > 0; +} + +/// The filters shown: Mail too when there's mail. +function filters() { + return hasMail() ? [...FILTERS, MAIL_FILTER] : FILTERS; +} + function chip(title, icon, count, selected, action, number) { const button = element("button", "chip" + (selected ? " selected" : "")); if (icon) button.insertAdjacentHTML("beforeend", ICONS[icon]); @@ -215,7 +231,7 @@ function cell(hit) { const item = element("div", "cell" + (hit.path === state.selection ? " selected" : "")); item.dataset.path = hit.path; item.setAttribute("role", "option"); - item.setAttribute("aria-label", `${KIND_TITLE[hit.kind] || "File"}, ${nameOf(hit.path)}, ${reasonOf(hit)}`); + item.setAttribute("aria-label", `${KIND_TITLE[hit.kind] || "File"}, ${titleOf(hit)}, ${reasonOf(hit)}`); const picture = element("div", "picture"); picture.append(cellPicture(hit)); if (hit.kind !== "image" && hit.kind !== "screenshot") { @@ -225,7 +241,7 @@ function cell(hit) { } const place = placeOf(hit); if (place) picture.append(element("span", "place on-image", place)); - item.append(picture, element("div", "name", "‎" + nameOf(hit.path)), folderLine(folderOf(hit))); + item.append(picture, element("div", "name", "‎" + titleOf(hit)), folderLine(folderOf(hit))); wire(item, hit); return item; } @@ -242,7 +258,7 @@ function cellPicture(hit) { thumbnails.observe(image); return image; } - if (hit.kind === "doc" && hit.excerpt) { + if ((hit.kind === "doc" || (hit.kind === "mail" && hit.segment === "chunk")) && hit.excerpt) { const card = element("div", "text-card"); card.dir = "auto"; card.innerHTML = marked(hit.excerpt.slice(0, 600)); @@ -319,7 +335,7 @@ function codeRow(hit, weaker) { icon.innerHTML = ICONS.code; const text = element("div", "text"); const head = element("div", "head"); - head.append(element("span", "title", "‎" + nameOf(hit.path))); + head.append(element("span", "title", "‎" + titleOf(hit))); const place = placeOf(hit); if (place) head.append(element("span", "place", place)); head.append(folderLine(folderOf(hit), "span")); @@ -350,7 +366,7 @@ function wire(item, hit) { } // Elsewhere: pressed and moved a little, the system's drag (which needs the button down). item.onmousedown = (event) => { - if (event.button !== 0 || isLinux) return; + if (event.button !== 0 || isLinux || hit.kind === "mail") return; // a mail is no file to drag event.preventDefault(); const [x, y] = [event.clientX, event.clientY]; const moved = (move) => { @@ -370,7 +386,10 @@ function wire(item, hit) { item.oncontextmenu = (event) => { event.preventDefault(); select(hit.path); - showMenu(event.clientX, event.clientY, [ + showMenu(event.clientX, event.clientY, hit.kind === "mail" ? [ + ["Open in Outlook", () => open(hit)], + [`Show PST in ${FILE_MANAGER}`, () => reveal(hit)], + ] : [ ["Open", () => open(hit)], [`Show in ${FILE_MANAGER}`, () => reveal(hit)], null, @@ -408,7 +427,7 @@ function open(hit) { } function reveal(hit) { - invoke("reveal", { path: hit.path }).catch(failed); + invoke("reveal", { path: fileOf(hit) }).catch(failed); } function failed(error) { @@ -426,7 +445,7 @@ function home() { const shown = [...EXAMPLES, ...(state.info.hasCode ? [["code: where the settings are saved", "code"]] : [])]; for (const [text, kind] of shown) { const example = element("button", "example"); - example.innerHTML = ICONS[FILTERS.find((filter) => filter.key === kind)?.icon ?? kind]; + example.innerHTML = ICONS[filters().find((filter) => filter.key === kind)?.icon ?? kind]; example.append(text); example.title = `Search for “${text}”`; example.onmousedown = (event) => event.preventDefault(); @@ -564,7 +583,7 @@ function previewPicture(hit) { addPlace(box, span(hit)); return box; } - if (hit.segment === "chunk") { + if (hit.segment === "chunk" || hit.kind === "mail") { box.append(passage(hit)); const parts = Number(hit.info?.chunks || 0); if (hit.loc != null && parts > 1) addPlace(box, `Part ${Math.trunc(hit.loc)} of ${parts}`); @@ -644,7 +663,7 @@ function renderBig() { const sheet = element("div", "sheet"); const head = element("div", "head"); const title = element("div", "title"); - title.append(element("span", "name", nameOf(hit.path))); + title.append(element("span", "name", titleOf(hit))); const place = placeOf(hit); if (place) title.append(element("span", "place", place)); const opening = element("button", "primary", "Open"); @@ -777,9 +796,14 @@ function passage(hit) { page.dir = isRightToLeft(text) ? "rtl" : "ltr"; page.innerHTML = highlighted(passageAround(text, [...found, ...queryWords(state.text)]), state.text, found); }; - show((hit.excerpt ?? "").replace(/[«»]/g, "")); - ask({ op: "text", segment: hit.segment_id, code: false }) - .then((reply) => { if (reply.text) show(reply.text); }) + // A mail found by its subject or folder shows its beginning; one not read yet says so. + const nameOnly = hit.kind === "mail" && hit.segment !== "chunk"; + show(nameOnly ? "" : (hit.excerpt ?? "").replace(/[«»]/g, "")); + previewText(hit) + .then((reply) => { + if (reply.text) show(reply.text); + else if (nameOnly) show("Not read yet: found by its subject."); + }) .catch(() => {}); page.ondblclick = () => open(hit); return page; @@ -819,8 +843,8 @@ function codeLines(hit) { function facts(hit) { const box = element("div", "facts"); - const title = element("div", "title", "‎" + nameOf(hit.path)); - title.title = nameOf(hit.path); + const title = element("div", "title", "‎" + titleOf(hit)); + title.title = titleOf(hit); const folder = folderLine(folderOf(hit), "button"); folder.title = `Show in ${FILE_MANAGER}`; folder.onclick = () => reveal(hit); @@ -1000,6 +1024,11 @@ document.addEventListener("keydown", (event) => { setFilter("code"); return; } + if (number === 8 && hasMail()) { + event.preventDefault(); + setFilter(MAIL_FILTER.key); + return; + } switch (event.key.toLowerCase()) { case ",": event.preventDefault();