diff --git a/Sources/PostServer/MailboxCommandRunning.swift b/Sources/PostServer/MailboxCommandRunning.swift index 4eb0150..b85cfd8 100644 --- a/Sources/PostServer/MailboxCommandRunning.swift +++ b/Sources/PostServer/MailboxCommandRunning.swift @@ -50,7 +50,9 @@ extension MailboxCommandRunning { ) } - /// Envelope, flags and dates only, no message bodies. + /// Header data without any message body: SwiftMail's default info set, which is envelope, + /// internal date, flags, body structure and the full header section (so + /// `MessageInfo.additionalFields` is filled, as `fetchMessages(using:)` filled it). func fetchHeaderInfos( using identifierSet: SwiftMail.MessageIdentifierSet ) async throws -> [SwiftMail.MessageInfo] { diff --git a/Sources/PostServer/PostServer.swift b/Sources/PostServer/PostServer.swift index 3d56e95..48d38a3 100644 --- a/Sources/PostServer/PostServer.swift +++ b/Sources/PostServer/PostServer.swift @@ -302,13 +302,14 @@ public actor PostServer { throw PostServerError.invalidLimit(limit) } - return try await withServer(serverId: serverId) { server in - let status = try await server.selectMailbox(mailbox) + return try await withExtraConnection(serverId: serverId) { connection in + let status = try await connection.selectMailbox(mailbox) guard let latest = status.latest(limit) else { return [] } - let headers = try await collectHeaders(from: server.fetchMessages(using: latest)) + // Header data only; the listing shows nothing from the bodies. + let headers = try await fetchHeaders(for: latest.toArray(), using: connection) return headers.sorted { $0.uid > $1.uid } } } @@ -327,7 +328,13 @@ public actor PostServer { _ = try await server.selectMailbox(mailbox) var messages: [MessageDetail] = [] - for try await message in server.fetchMessages(using: set) { + // One FETCH with the set as given: a range like 1-1000000 stays compact, and the + // server answers only for the messages that exist. + for info in try await server.fetchHeaderInfos(using: set) { + guard let uid = info.uid else { continue } + // Download the text and HTML bodies only; attachments are listed by name and type. + let outline = Message(header: info, parts: try await server.fetchStructure(uid)) + let message = try await downloading(outline.bodies, of: outline, uid: uid, using: server) let additionalHeaders = await fetchAdditionalHeaders(for: message, using: server) messages.append(messageDetail(from: message, additionalHeaders: additionalHeaders)) @@ -771,37 +778,30 @@ public actor PostServer { return try await withServer(serverId: serverId) { server in _ = try await server.selectMailbox(mailbox) - let set = MessageIdentifierSet(UID(UInt32(uid))) - - for try await message in server.fetchMessages(using: set) { - if let cid { - let normalizedCID = Self.normalizedContentID(cid) - guard let match = message.cids.first(where: { part in - guard let contentId = part.contentId else { return false } - return Self.normalizedContentID(contentId) == normalizedCID - }) else { - throw PostServerError.attachmentContentIdNotFound(contentId: cid, uid: uid) - } + let messageUID = UID(UInt32(uid)) - guard let data = match.decodedData() ?? match.data else { - throw PostServerError.attachmentDataMissing(filename: match.suggestedFilename) - } + // Choose the part from the message's structure, then download only that part. + guard let message = try await fetchOutline(uid: messageUID, using: server) else { + throw PostServerError.messageNotFound(uid: uid, mailbox: mailbox) + } - return AttachmentData( - filename: match.filename ?? match.suggestedFilename, - contentType: match.contentType, - data: data.base64EncodedString(), - size: data.count - ) + var part: MessagePart + if let cid { + let normalizedCID = Self.normalizedContentID(cid) + guard let match = message.cids.first(where: { part in + guard let contentId = part.contentId else { return false } + return Self.normalizedContentID(contentId) == normalizedCID + }) else { + throw PostServerError.attachmentContentIdNotFound(contentId: cid, uid: uid) } - + part = match + } else { let attachments = message.attachments guard !attachments.isEmpty else { throw PostServerError.noAttachments(uid: uid) } - let part: MessagePart if let filename { guard let match = attachments.first(where: { ($0.filename ?? $0.suggestedFilename).lowercased() == filename.lowercased() @@ -812,20 +812,20 @@ public actor PostServer { } else { part = attachments[0] } + } - guard let data = part.decodedData() ?? part.data else { - throw PostServerError.attachmentDataMissing(filename: part.suggestedFilename) - } + part.data = try await server.fetchPart(section: part.section, of: messageUID) - return AttachmentData( - filename: part.filename ?? part.suggestedFilename, - contentType: part.contentType, - data: data.base64EncodedString(), - size: data.count - ) + guard let data = part.decodedData() ?? part.data else { + throw PostServerError.attachmentDataMissing(filename: part.suggestedFilename) } - throw PostServerError.messageNotFound(uid: uid, mailbox: mailbox) + return AttachmentData( + filename: part.filename ?? part.suggestedFilename, + contentType: part.contentType, + data: data.base64EncodedString(), + size: data.count + ) } } @@ -873,32 +873,33 @@ public actor PostServer { return try await withServer(serverId: serverId) { server in _ = try await server.selectMailbox(mailbox) - let set = MessageIdentifierSet(UID(UInt32(uid))) + let messageUID = UID(UInt32(uid)) - for try await message in server.fetchMessages(using: set) { - let html: String - - if let htmlBody = message.htmlBody, !htmlBody.isEmpty { - html = htmlBody - } else if let textBody = message.textBody, !textBody.isEmpty { - html = MarkdownToHTML.convert(textBody) - } else { - throw PostServerError.emptyBody(uid: uid) - } - - let subject = message.subject ?? "message-\(uid)" - let filename = Self.sanitizeFilename(subject) + ".pdf" - - let pdfData = try await HTMLToPDF.render(html: html, logger: logger) - return AttachmentData( - filename: filename, - contentType: "application/pdf", - data: pdfData.base64EncodedString(), - size: pdfData.count - ) + guard let outline = try await fetchOutline(uid: messageUID, using: server) else { + throw PostServerError.messageNotFound(uid: uid, mailbox: mailbox) } + // The PDF is rendered from the text or HTML body; attachments are not downloaded. + let message = try await downloading(outline.bodies, of: outline, uid: messageUID, using: server) + + let html: String + if let htmlBody = message.htmlBody, !htmlBody.isEmpty { + html = htmlBody + } else if let textBody = message.textBody, !textBody.isEmpty { + html = MarkdownToHTML.convert(textBody) + } else { + throw PostServerError.emptyBody(uid: uid) + } + + let subject = message.subject ?? "message-\(uid)" + let filename = Self.sanitizeFilename(subject) + ".pdf" - throw PostServerError.messageNotFound(uid: uid, mailbox: mailbox) + let pdfData = try await HTMLToPDF.render(html: html, logger: logger) + return AttachmentData( + filename: filename, + contentType: "application/pdf", + data: pdfData.base64EncodedString(), + size: pdfData.count + ) } } @@ -1659,16 +1660,6 @@ public actor PostServer { return message.header.additionalFields } - internal func collectHeaders(from stream: AsyncThrowingStream) async throws -> [MessageHeader] { - var headers: [MessageHeader] = [] - - for try await message in stream { - headers.append(messageHeader(from: message)) - } - - return headers - } - internal func messageHeader(from message: Message) -> MessageHeader { messageHeader(from: message.header) } @@ -1686,19 +1677,55 @@ public actor PostServer { ) } - /// Fetches the header data for `uids` in chunks, so a large result set never becomes one + /// Fetches the header data for `identifiers` in chunks, so a large set never becomes one /// oversized response. - internal func fetchHeaders(for uids: [UID], using connection: any MailboxCommandRunning) async throws -> [MessageHeader] { + internal func fetchInfos( + for identifiers: [T], + using connection: any MailboxCommandRunning + ) async throws -> [MessageInfo] { let chunkSize = 200 - var headers: [MessageHeader] = [] - var start = uids.startIndex - while start < uids.endIndex { - let end = uids.index(start, offsetBy: chunkSize, limitedBy: uids.endIndex) ?? uids.endIndex - let infos = try await connection.fetchHeaderInfos(using: MessageIdentifierSet(Array(uids[start..( + for identifiers: [T], + using connection: any MailboxCommandRunning + ) async throws -> [MessageHeader] { + try await fetchInfos(for: identifiers, using: connection).map { messageHeader(from: $0) } + } + + /// The message's header data and part structure, without the data of any part. + /// + /// SwiftMail's `fetchMessages(using:)` downloads every part, attachments included. Starting + /// from the outline, each tool downloads only the parts it uses (see `downloading`). + internal func fetchOutline(uid: UID, using server: IMAPServer) async throws -> Message? { + guard let info = try await server.fetchMessageInfo(for: uid) else { + return nil + } + return Message(header: info, parts: try await server.fetchStructure(uid)) + } + + /// `message` with the data of `parts` downloaded; every other part stays without data. + internal func downloading( + _ parts: [MessagePart], + of message: Message, + uid: UID, + using server: IMAPServer + ) async throws -> Message { + let sections = Set(parts.map(\.section)) + var filled = message.parts + for index in filled.indices where sections.contains(filled[index].section) { + filled[index].data = try await server.fetchPart(section: filled[index].section, of: uid) + } + return Message(header: message.header, parts: filled) } internal func messageDetail(from message: Message, additionalHeaders: [String: String]? = nil) -> MessageDetail { diff --git a/Tests/PostCLITests/LeanFetchLiveTests.swift b/Tests/PostCLITests/LeanFetchLiveTests.swift new file mode 100644 index 0000000..2444dc4 --- /dev/null +++ b/Tests/PostCLITests/LeanFetchLiveTests.swift @@ -0,0 +1,134 @@ +import XCTest +import SwiftMail +@testable import PostServer + +/// Listing, `post fetch`, attachment downloads and PDF export now download only the parts they +/// use instead of every part. Their results must equal what full downloads produce. +/// +/// Runs against a real account and only reads: set POST_LIVE_TEST_SERVER to a server ID from +/// ~/.post.json; optionally POST_LIVE_TEST_LEAN_MAILBOX (default INBOX), and +/// POST_LIVE_TEST_ATTACHMENT=: for a message with an attachment. +final class LeanFetchLiveTests: XCTestCase { + private func liveServerID() throws -> String { + guard let serverId = ProcessInfo.processInfo.environment["POST_LIVE_TEST_SERVER"], !serverId.isEmpty else { + throw XCTSkip("Set POST_LIVE_TEST_SERVER to run against a real account") + } + return serverId + } + + /// A separate connection that downloads messages the previous way: every part. + private func referenceServer(for serverId: String, configuration: PostConfiguration) async throws -> IMAPServer { + let credentials = try configuration.resolveCredentials(forServer: serverId) + let server = IMAPServer(host: credentials.host, port: credentials.port) + try await server.connect() + try await server.login(username: credentials.username, password: credentials.password) + return server + } + + private func fullMessages(_ set: MessageIdentifierSet, using server: IMAPServer) async throws -> [Message] { + var messages: [Message] = [] + for try await message in server.fetchMessages(using: set) { + messages.append(message) + } + return messages + } + + /// JSON with sorted keys, for comparing values that aren't Equatable. + private func json(_ value: some Encodable) throws -> String { + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + return String(decoding: try encoder.encode(value), as: UTF8.self) + } + + func testListingAndFetchMatchFullDownloads() async throws { + let serverId = try liveServerID() + let mailbox = ProcessInfo.processInfo.environment["POST_LIVE_TEST_LEAN_MAILBOX"] ?? "INBOX" + let configuration = try PostConfiguration.load() + let post = PostServer(configuration: configuration) + let reference = try await referenceServer(for: serverId, configuration: configuration) + + let selection = try await reference.selectMailbox(mailbox) + guard let latest = selection.latest(8) else { + throw XCTSkip("\(mailbox) is empty") + } + let full = try await fullMessages(latest, using: reference) + + // Listing: header data only. + let listed = try await post.listMessages(serverId: serverId, mailbox: mailbox, limit: 8) + var expectedHeaders: [MessageHeader] = [] + for message in full { + expectedHeaders.append(await post.messageHeader(from: message)) + } + XCTAssertEqual( + try listed.sorted { $0.uid > $1.uid }.map { try json($0) }, + try expectedHeaders.sorted { $0.uid > $1.uid }.map { try json($0) } + ) + + // post fetch: text and HTML bodies only, additional headers included. + let uids = full.compactMap(\.uid).map { String($0.value) }.joined(separator: ",") + let details = try await post.fetchMessage(serverId: serverId, uids: uids, mailbox: mailbox) + var expectedDetails: [MessageDetail] = [] + for message in full { + expectedDetails.append(await post.messageDetail(from: message)) + } + XCTAssertEqual(details.count, expectedDetails.count) + // A message may have nothing left after the noise filter, but the sample must have some. + XCTAssertTrue(details.contains { !($0.additionalHeaders ?? [:]).isEmpty }, "additional headers missing") + XCTAssertEqual( + try details.sorted { $0.uid < $1.uid }.map { try json($0) }, + try expectedDetails.sorted { $0.uid < $1.uid }.map { try json($0) } + ) + + await post.shutdown() + try? await reference.disconnect() + } + + /// A wide UID range must go to the server as one compact FETCH, not be expanded into one + /// request per 200 UIDs. + func testWideRangeFetchesOnlyExistingMessages() async throws { + let serverId = try liveServerID() + let mailbox = ProcessInfo.processInfo.environment["POST_LIVE_TEST_LEAN_MAILBOX"] ?? "INBOX" + let post = PostServer(configuration: try PostConfiguration.load()) + + let newest = try await post.listMessages(serverId: serverId, mailbox: mailbox, limit: 3).map(\.uid).sorted() + guard let lowest = newest.first, let highest = newest.last else { + throw XCTSkip("\(mailbox) is empty") + } + + let start = ContinuousClock.now + let details = try await post.fetchMessage(serverId: serverId, uids: "\(lowest)-\(highest + 1_000_000)", mailbox: mailbox) + let elapsed = ContinuousClock.now - start + + XCTAssertEqual(details.map(\.uid).sorted(), newest) + XCTAssertLessThan(elapsed, .seconds(30)) + + await post.shutdown() + } + + func testAttachmentMatchesFullDownload() async throws { + let serverId = try liveServerID() + guard let spec = ProcessInfo.processInfo.environment["POST_LIVE_TEST_ATTACHMENT"], + let separator = spec.lastIndex(of: ":"), + let uid = Int(spec[spec.index(after: separator)...]) else { + throw XCTSkip("Set POST_LIVE_TEST_ATTACHMENT=: for a message with an attachment") + } + let mailbox = String(spec[..(UID(UInt32(uid))), using: reference) + let expectedPart = try XCTUnwrap(full.first?.attachments.first, "message \(uid) has no attachment") + let expectedData = try XCTUnwrap(expectedPart.decodedData() ?? expectedPart.data) + + let downloaded = try await post.downloadAttachment(serverId: serverId, uid: uid, mailbox: mailbox) + XCTAssertEqual(downloaded.filename, expectedPart.filename ?? expectedPart.suggestedFilename) + XCTAssertEqual(downloaded.contentType, expectedPart.contentType) + XCTAssertEqual(downloaded.size, expectedData.count) + XCTAssertTrue(downloaded.data == expectedData.base64EncodedString(), "attachment data differs") + + await post.shutdown() + try? await reference.disconnect() + } +}