From 0ef87f221b4a4311339417b6704eb4db2529733c Mon Sep 17 00:00:00 2001 From: Oliver Drobnik Date: Wed, 7 Oct 2026 18:23:11 +0200 Subject: [PATCH 1/2] Download only the parts a tool uses SwiftMail's fetchMessages(using:) downloads every part of a message, attachments included. Listing, post fetch, attachment downloads and PDF export all went through it, so listing a folder or reading the text of an email with a 77 MB video downloaded the video. - listMessages fetches header data only, on an extra connection like search. - fetchMessage and exportPDF fetch the structure, then only the text and HTML bodies; attachments are still listed by name and type. - downloadAttachment picks the part from the structure and downloads only that part. LeanFetchLiveTests compares the results with full downloads on a real account; it runs only with POST_LIVE_TEST_SERVER set. Co-Authored-By: Claude Opus 5.5 --- Sources/PostServer/PostServer.swift | 181 +++++++++++--------- Tests/PostCLITests/LeanFetchLiveTests.swift | 116 +++++++++++++ 2 files changed, 219 insertions(+), 78 deletions(-) create mode 100644 Tests/PostCLITests/LeanFetchLiveTests.swift diff --git a/Sources/PostServer/PostServer.swift b/Sources/PostServer/PostServer.swift index 3d56e95..b352e53 100644 --- a/Sources/PostServer/PostServer.swift +++ b/Sources/PostServer/PostServer.swift @@ -302,13 +302,14 @@ public actor PostServer { throw PostServerError.invalidLimit(limit) } - return try await withServer(serverId: serverId) { server in - let status = try await server.selectMailbox(mailbox) + return try await withExtraConnection(serverId: serverId) { connection in + let status = try await connection.selectMailbox(mailbox) guard let latest = status.latest(limit) else { return [] } - let headers = try await collectHeaders(from: server.fetchMessages(using: latest)) + // Header data only; the listing shows nothing from the bodies. + let headers = try await fetchHeaders(for: latest.toArray(), using: connection) return headers.sorted { $0.uid > $1.uid } } } @@ -327,7 +328,11 @@ public actor PostServer { _ = try await server.selectMailbox(mailbox) var messages: [MessageDetail] = [] - for try await message in server.fetchMessages(using: set) { + for info in try await fetchInfos(for: set.toArray(), using: server) { + guard let uid = info.uid else { continue } + // Download the text and HTML bodies only; attachments are listed by name and type. + let outline = Message(header: info, parts: try await server.fetchStructure(uid)) + let message = try await downloading(outline.bodies, of: outline, uid: uid, using: server) let additionalHeaders = await fetchAdditionalHeaders(for: message, using: server) messages.append(messageDetail(from: message, additionalHeaders: additionalHeaders)) @@ -771,37 +776,30 @@ public actor PostServer { return try await withServer(serverId: serverId) { server in _ = try await server.selectMailbox(mailbox) - let set = MessageIdentifierSet(UID(UInt32(uid))) - - for try await message in server.fetchMessages(using: set) { - if let cid { - let normalizedCID = Self.normalizedContentID(cid) - guard let match = message.cids.first(where: { part in - guard let contentId = part.contentId else { return false } - return Self.normalizedContentID(contentId) == normalizedCID - }) else { - throw PostServerError.attachmentContentIdNotFound(contentId: cid, uid: uid) - } + let messageUID = UID(UInt32(uid)) - guard let data = match.decodedData() ?? match.data else { - throw PostServerError.attachmentDataMissing(filename: match.suggestedFilename) - } + // Choose the part from the message's structure, then download only that part. + guard let message = try await fetchOutline(uid: messageUID, using: server) else { + throw PostServerError.messageNotFound(uid: uid, mailbox: mailbox) + } - return AttachmentData( - filename: match.filename ?? match.suggestedFilename, - contentType: match.contentType, - data: data.base64EncodedString(), - size: data.count - ) + var part: MessagePart + if let cid { + let normalizedCID = Self.normalizedContentID(cid) + guard let match = message.cids.first(where: { part in + guard let contentId = part.contentId else { return false } + return Self.normalizedContentID(contentId) == normalizedCID + }) else { + throw PostServerError.attachmentContentIdNotFound(contentId: cid, uid: uid) } - + part = match + } else { let attachments = message.attachments guard !attachments.isEmpty else { throw PostServerError.noAttachments(uid: uid) } - let part: MessagePart if let filename { guard let match = attachments.first(where: { ($0.filename ?? $0.suggestedFilename).lowercased() == filename.lowercased() @@ -812,20 +810,20 @@ public actor PostServer { } else { part = attachments[0] } + } - guard let data = part.decodedData() ?? part.data else { - throw PostServerError.attachmentDataMissing(filename: part.suggestedFilename) - } + part.data = try await server.fetchPart(section: part.section, of: messageUID) - return AttachmentData( - filename: part.filename ?? part.suggestedFilename, - contentType: part.contentType, - data: data.base64EncodedString(), - size: data.count - ) + guard let data = part.decodedData() ?? part.data else { + throw PostServerError.attachmentDataMissing(filename: part.suggestedFilename) } - throw PostServerError.messageNotFound(uid: uid, mailbox: mailbox) + return AttachmentData( + filename: part.filename ?? part.suggestedFilename, + contentType: part.contentType, + data: data.base64EncodedString(), + size: data.count + ) } } @@ -873,32 +871,33 @@ public actor PostServer { return try await withServer(serverId: serverId) { server in _ = try await server.selectMailbox(mailbox) - let set = MessageIdentifierSet(UID(UInt32(uid))) + let messageUID = UID(UInt32(uid)) - for try await message in server.fetchMessages(using: set) { - let html: String - - if let htmlBody = message.htmlBody, !htmlBody.isEmpty { - html = htmlBody - } else if let textBody = message.textBody, !textBody.isEmpty { - html = MarkdownToHTML.convert(textBody) - } else { - throw PostServerError.emptyBody(uid: uid) - } - - let subject = message.subject ?? "message-\(uid)" - let filename = Self.sanitizeFilename(subject) + ".pdf" - - let pdfData = try await HTMLToPDF.render(html: html, logger: logger) - return AttachmentData( - filename: filename, - contentType: "application/pdf", - data: pdfData.base64EncodedString(), - size: pdfData.count - ) + guard let outline = try await fetchOutline(uid: messageUID, using: server) else { + throw PostServerError.messageNotFound(uid: uid, mailbox: mailbox) } + // The PDF is rendered from the text or HTML body; attachments are not downloaded. + let message = try await downloading(outline.bodies, of: outline, uid: messageUID, using: server) + + let html: String + if let htmlBody = message.htmlBody, !htmlBody.isEmpty { + html = htmlBody + } else if let textBody = message.textBody, !textBody.isEmpty { + html = MarkdownToHTML.convert(textBody) + } else { + throw PostServerError.emptyBody(uid: uid) + } + + let subject = message.subject ?? "message-\(uid)" + let filename = Self.sanitizeFilename(subject) + ".pdf" - throw PostServerError.messageNotFound(uid: uid, mailbox: mailbox) + let pdfData = try await HTMLToPDF.render(html: html, logger: logger) + return AttachmentData( + filename: filename, + contentType: "application/pdf", + data: pdfData.base64EncodedString(), + size: pdfData.count + ) } } @@ -1659,16 +1658,6 @@ public actor PostServer { return message.header.additionalFields } - internal func collectHeaders(from stream: AsyncThrowingStream) async throws -> [MessageHeader] { - var headers: [MessageHeader] = [] - - for try await message in stream { - headers.append(messageHeader(from: message)) - } - - return headers - } - internal func messageHeader(from message: Message) -> MessageHeader { messageHeader(from: message.header) } @@ -1686,19 +1675,55 @@ public actor PostServer { ) } - /// Fetches the header data for `uids` in chunks, so a large result set never becomes one + /// Fetches the header data for `identifiers` in chunks, so a large set never becomes one /// oversized response. - internal func fetchHeaders(for uids: [UID], using connection: any MailboxCommandRunning) async throws -> [MessageHeader] { + internal func fetchInfos( + for identifiers: [T], + using connection: any MailboxCommandRunning + ) async throws -> [MessageInfo] { let chunkSize = 200 - var headers: [MessageHeader] = [] - var start = uids.startIndex - while start < uids.endIndex { - let end = uids.index(start, offsetBy: chunkSize, limitedBy: uids.endIndex) ?? uids.endIndex - let infos = try await connection.fetchHeaderInfos(using: MessageIdentifierSet(Array(uids[start..( + for identifiers: [T], + using connection: any MailboxCommandRunning + ) async throws -> [MessageHeader] { + try await fetchInfos(for: identifiers, using: connection).map { messageHeader(from: $0) } + } + + /// The message's header data and part structure, without the data of any part. + /// + /// SwiftMail's `fetchMessages(using:)` downloads every part, attachments included. Starting + /// from the outline, each tool downloads only the parts it uses (see `downloading`). + internal func fetchOutline(uid: UID, using server: IMAPServer) async throws -> Message? { + guard let info = try await server.fetchMessageInfo(for: uid) else { + return nil + } + return Message(header: info, parts: try await server.fetchStructure(uid)) + } + + /// `message` with the data of `parts` downloaded; every other part stays without data. + internal func downloading( + _ parts: [MessagePart], + of message: Message, + uid: UID, + using server: IMAPServer + ) async throws -> Message { + let sections = Set(parts.map(\.section)) + var filled = message.parts + for index in filled.indices where sections.contains(filled[index].section) { + filled[index].data = try await server.fetchPart(section: filled[index].section, of: uid) + } + return Message(header: message.header, parts: filled) } internal func messageDetail(from message: Message, additionalHeaders: [String: String]? = nil) -> MessageDetail { diff --git a/Tests/PostCLITests/LeanFetchLiveTests.swift b/Tests/PostCLITests/LeanFetchLiveTests.swift new file mode 100644 index 0000000..0d44b8a --- /dev/null +++ b/Tests/PostCLITests/LeanFetchLiveTests.swift @@ -0,0 +1,116 @@ +import XCTest +import SwiftMail +@testable import PostServer + +/// Listing, `post fetch`, attachment downloads and PDF export now download only the parts they +/// use instead of every part. Their results must equal what full downloads produce. +/// +/// Runs against a real account and only reads: set POST_LIVE_TEST_SERVER to a server ID from +/// ~/.post.json; optionally POST_LIVE_TEST_LEAN_MAILBOX (default INBOX), and +/// POST_LIVE_TEST_ATTACHMENT=: for a message with an attachment. +final class LeanFetchLiveTests: XCTestCase { + private func liveServerID() throws -> String { + guard let serverId = ProcessInfo.processInfo.environment["POST_LIVE_TEST_SERVER"], !serverId.isEmpty else { + throw XCTSkip("Set POST_LIVE_TEST_SERVER to run against a real account") + } + return serverId + } + + /// A separate connection that downloads messages the previous way: every part. + private func referenceServer(for serverId: String, configuration: PostConfiguration) async throws -> IMAPServer { + let credentials = try configuration.resolveCredentials(forServer: serverId) + let server = IMAPServer(host: credentials.host, port: credentials.port) + try await server.connect() + try await server.login(username: credentials.username, password: credentials.password) + return server + } + + private func fullMessages(_ set: MessageIdentifierSet, using server: IMAPServer) async throws -> [Message] { + var messages: [Message] = [] + for try await message in server.fetchMessages(using: set) { + messages.append(message) + } + return messages + } + + /// JSON with sorted keys and without `dropping`, for comparing values that aren't Equatable. + private func json(_ value: some Encodable, dropping keys: Set = []) throws -> String { + let encoded = try JSONEncoder().encode(value) + var object = try JSONSerialization.jsonObject(with: encoded) + if var dictionary = object as? [String: Any] { + keys.forEach { dictionary.removeValue(forKey: $0) } + object = dictionary + } + let data = try JSONSerialization.data(withJSONObject: object, options: [.sortedKeys]) + return String(decoding: data, as: UTF8.self) + } + + func testListingAndFetchMatchFullDownloads() async throws { + let serverId = try liveServerID() + let mailbox = ProcessInfo.processInfo.environment["POST_LIVE_TEST_LEAN_MAILBOX"] ?? "INBOX" + let configuration = try PostConfiguration.load() + let post = PostServer(configuration: configuration) + let reference = try await referenceServer(for: serverId, configuration: configuration) + + let selection = try await reference.selectMailbox(mailbox) + guard let latest = selection.latest(8) else { + throw XCTSkip("\(mailbox) is empty") + } + let full = try await fullMessages(latest, using: reference) + + // Listing: header data only. + let listed = try await post.listMessages(serverId: serverId, mailbox: mailbox, limit: 8) + var expectedHeaders: [MessageHeader] = [] + for message in full { + expectedHeaders.append(await post.messageHeader(from: message)) + } + XCTAssertEqual( + try listed.sorted { $0.uid > $1.uid }.map { try json($0) }, + try expectedHeaders.sorted { $0.uid > $1.uid }.map { try json($0) } + ) + + // post fetch: text and HTML bodies only. Additional headers are fetched separately and + // not part of the comparison. + let uids = full.compactMap(\.uid).map { String($0.value) }.joined(separator: ",") + let details = try await post.fetchMessage(serverId: serverId, uids: uids, mailbox: mailbox) + var expectedDetails: [MessageDetail] = [] + for message in full { + expectedDetails.append(await post.messageDetail(from: message)) + } + XCTAssertEqual(details.count, expectedDetails.count) + XCTAssertEqual( + try details.sorted { $0.uid < $1.uid }.map { try json($0, dropping: ["additionalHeaders"]) }, + try expectedDetails.sorted { $0.uid < $1.uid }.map { try json($0, dropping: ["additionalHeaders"]) } + ) + + await post.shutdown() + try? await reference.disconnect() + } + + func testAttachmentMatchesFullDownload() async throws { + let serverId = try liveServerID() + guard let spec = ProcessInfo.processInfo.environment["POST_LIVE_TEST_ATTACHMENT"], + let separator = spec.lastIndex(of: ":"), + let uid = Int(spec[spec.index(after: separator)...]) else { + throw XCTSkip("Set POST_LIVE_TEST_ATTACHMENT=: for a message with an attachment") + } + let mailbox = String(spec[..(UID(UInt32(uid))), using: reference) + let expectedPart = try XCTUnwrap(full.first?.attachments.first, "message \(uid) has no attachment") + let expectedData = try XCTUnwrap(expectedPart.decodedData() ?? expectedPart.data) + + let downloaded = try await post.downloadAttachment(serverId: serverId, uid: uid, mailbox: mailbox) + XCTAssertEqual(downloaded.filename, expectedPart.filename ?? expectedPart.suggestedFilename) + XCTAssertEqual(downloaded.contentType, expectedPart.contentType) + XCTAssertEqual(downloaded.size, expectedData.count) + XCTAssertTrue(downloaded.data == expectedData.base64EncodedString(), "attachment data differs") + + await post.shutdown() + try? await reference.disconnect() + } +} From adb85fbf77f17ad4b9be97886a64999a0cc35142 Mon Sep 17 00:00:00 2001 From: Oliver Drobnik Date: Wed, 7 Oct 2026 18:31:35 +0200 Subject: [PATCH 2/2] Fetch requested UID sets compactly and check additional headers fetchMessage expanded the requested UID set into an array and fetched it 200 UIDs at a time, so a range like 1-1000000 meant a million-element array and 5,000 FETCH commands. It now sends the set as given in one UID FETCH, and the server answers only for messages that exist. SwiftMail's fetchMessages expanded such ranges as well, in chunked(size:). The header fetch uses SwiftMail's default info set, which includes the full header section, so additional headers are still filled. The doc comment that said otherwise is corrected; the live test now compares additional headers too and fetches a wide range. Co-Authored-By: Claude Opus 5.5 --- .../PostServer/MailboxCommandRunning.swift | 4 +- Sources/PostServer/PostServer.swift | 4 +- Tests/PostCLITests/LeanFetchLiveTests.swift | 46 +++++++++++++------ 3 files changed, 38 insertions(+), 16 deletions(-) diff --git a/Sources/PostServer/MailboxCommandRunning.swift b/Sources/PostServer/MailboxCommandRunning.swift index 4eb0150..b85cfd8 100644 --- a/Sources/PostServer/MailboxCommandRunning.swift +++ b/Sources/PostServer/MailboxCommandRunning.swift @@ -50,7 +50,9 @@ extension MailboxCommandRunning { ) } - /// Envelope, flags and dates only, no message bodies. + /// Header data without any message body: SwiftMail's default info set, which is envelope, + /// internal date, flags, body structure and the full header section (so + /// `MessageInfo.additionalFields` is filled, as `fetchMessages(using:)` filled it). func fetchHeaderInfos( using identifierSet: SwiftMail.MessageIdentifierSet ) async throws -> [SwiftMail.MessageInfo] { diff --git a/Sources/PostServer/PostServer.swift b/Sources/PostServer/PostServer.swift index b352e53..48d38a3 100644 --- a/Sources/PostServer/PostServer.swift +++ b/Sources/PostServer/PostServer.swift @@ -328,7 +328,9 @@ public actor PostServer { _ = try await server.selectMailbox(mailbox) var messages: [MessageDetail] = [] - for info in try await fetchInfos(for: set.toArray(), using: server) { + // One FETCH with the set as given: a range like 1-1000000 stays compact, and the + // server answers only for the messages that exist. + for info in try await server.fetchHeaderInfos(using: set) { guard let uid = info.uid else { continue } // Download the text and HTML bodies only; attachments are listed by name and type. let outline = Message(header: info, parts: try await server.fetchStructure(uid)) diff --git a/Tests/PostCLITests/LeanFetchLiveTests.swift b/Tests/PostCLITests/LeanFetchLiveTests.swift index 0d44b8a..2444dc4 100644 --- a/Tests/PostCLITests/LeanFetchLiveTests.swift +++ b/Tests/PostCLITests/LeanFetchLiveTests.swift @@ -33,16 +33,11 @@ final class LeanFetchLiveTests: XCTestCase { return messages } - /// JSON with sorted keys and without `dropping`, for comparing values that aren't Equatable. - private func json(_ value: some Encodable, dropping keys: Set = []) throws -> String { - let encoded = try JSONEncoder().encode(value) - var object = try JSONSerialization.jsonObject(with: encoded) - if var dictionary = object as? [String: Any] { - keys.forEach { dictionary.removeValue(forKey: $0) } - object = dictionary - } - let data = try JSONSerialization.data(withJSONObject: object, options: [.sortedKeys]) - return String(decoding: data, as: UTF8.self) + /// JSON with sorted keys, for comparing values that aren't Equatable. + private func json(_ value: some Encodable) throws -> String { + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + return String(decoding: try encoder.encode(value), as: UTF8.self) } func testListingAndFetchMatchFullDownloads() async throws { @@ -69,8 +64,7 @@ final class LeanFetchLiveTests: XCTestCase { try expectedHeaders.sorted { $0.uid > $1.uid }.map { try json($0) } ) - // post fetch: text and HTML bodies only. Additional headers are fetched separately and - // not part of the comparison. + // post fetch: text and HTML bodies only, additional headers included. let uids = full.compactMap(\.uid).map { String($0.value) }.joined(separator: ",") let details = try await post.fetchMessage(serverId: serverId, uids: uids, mailbox: mailbox) var expectedDetails: [MessageDetail] = [] @@ -78,15 +72,39 @@ final class LeanFetchLiveTests: XCTestCase { expectedDetails.append(await post.messageDetail(from: message)) } XCTAssertEqual(details.count, expectedDetails.count) + // A message may have nothing left after the noise filter, but the sample must have some. + XCTAssertTrue(details.contains { !($0.additionalHeaders ?? [:]).isEmpty }, "additional headers missing") XCTAssertEqual( - try details.sorted { $0.uid < $1.uid }.map { try json($0, dropping: ["additionalHeaders"]) }, - try expectedDetails.sorted { $0.uid < $1.uid }.map { try json($0, dropping: ["additionalHeaders"]) } + try details.sorted { $0.uid < $1.uid }.map { try json($0) }, + try expectedDetails.sorted { $0.uid < $1.uid }.map { try json($0) } ) await post.shutdown() try? await reference.disconnect() } + /// A wide UID range must go to the server as one compact FETCH, not be expanded into one + /// request per 200 UIDs. + func testWideRangeFetchesOnlyExistingMessages() async throws { + let serverId = try liveServerID() + let mailbox = ProcessInfo.processInfo.environment["POST_LIVE_TEST_LEAN_MAILBOX"] ?? "INBOX" + let post = PostServer(configuration: try PostConfiguration.load()) + + let newest = try await post.listMessages(serverId: serverId, mailbox: mailbox, limit: 3).map(\.uid).sorted() + guard let lowest = newest.first, let highest = newest.last else { + throw XCTSkip("\(mailbox) is empty") + } + + let start = ContinuousClock.now + let details = try await post.fetchMessage(serverId: serverId, uids: "\(lowest)-\(highest + 1_000_000)", mailbox: mailbox) + let elapsed = ContinuousClock.now - start + + XCTAssertEqual(details.map(\.uid).sorted(), newest) + XCTAssertLessThan(elapsed, .seconds(30)) + + await post.shutdown() + } + func testAttachmentMatchesFullDownload() async throws { let serverId = try liveServerID() guard let spec = ProcessInfo.processInfo.environment["POST_LIVE_TEST_ATTACHMENT"],