Stream and parser performance

src/vmime/header.cpp

@@ -42,6 +42,48 @@ header::~header() {
 }
 
 
+void header::parseImpl(
+	parsingContext& ctx,
+	const shared_ptr <utility::parserInputStreamAdapter>& parser,
+	const size_t position,
+	const size_t end,
+	size_t* newPosition
+) {
+
+	// Fields cannot extend beyond the first empty line, which ends the
+	// header: only extract up to it, not the body which follows
+	size_t headerEnd = end;
+
+	for (size_t pos = position ; pos < end ; ) {
+
+		parser->seek(pos);
+
+		const byte_t c = parser->peekByte();
+
+		if (c == '\n') {
+
+			headerEnd = pos + 1;
+			break;
+
+		} else if (c == '\r' && pos + 1 < end && parser->matchBytes("\r\n", 2)) {
+
+			headerEnd = pos + 2;
+			break;
+		}
+
+		const size_t eol = parser->findNext("\n", pos, end);
+
+		if (eol == npos) {
+			break;
+		}
+
+		pos = eol + 1;
+	}
+
+	component::parseImpl(ctx, parser, position, headerEnd, newPosition);
+}
+
+
 /*
 
  RFC #822:

src/vmime/header.hpp

@@ -339,6 +339,14 @@ class VMIME_EXPORT header : public component {
 protected:
 
 	// Component parsing & assembling
+	void parseImpl(
+		parsingContext& ctx,
+		const shared_ptr <utility::parserInputStreamAdapter>& parser,
+		const size_t position,
+		const size_t end,
+		size_t* newPosition = NULL
+	);
+
 	void parseImpl(
 		parsingContext& ctx,
 		const string& buffer,

src/vmime/net/maildir/maildirMessage.cpp

@@ -39,7 +39,7 @@
 #include "vmime/exception.hpp"
 #include "vmime/platform.hpp"
 
-#include "vmime/utility/outputStreamAdapter.hpp"
+#include "vmime/utility/outputStreamStringAdapter.hpp"
 #include "vmime/utility/stringUtils.hpp"
 
 
@@ -341,17 +341,28 @@ void maildirMessage::fetch(const shared_ptr <maildirFolder>& msgFolder, const fe
 
 			while (!is->eof()) {
 
+				// Only search the empty line which ends the header in the new data
+				// (and the last bytes before, which may be the start of it), not
+				// again in all the data read so far
+				const size_t searchStart = (contents.length() >= 2 ? contents.length() - 2 : 0);
+
 				const size_t read = is->read(buffer, sizeof(buffer));
 				vmime::utility::stringUtils::appendBytesToString(contents, buffer, read);
 
-				const size_t sep1 = contents.rfind("\r\n\r\n");
-				const size_t sep2 = contents.rfind("\n\n");
+				size_t headerEnd = contents.find("\n\n", searchStart);
 
-				if (sep1 != string::npos) {
-					contents.erase(contents.begin() + sep1 + 4, contents.end());
-					break;
-				} else if (sep2 != string::npos) {
-					contents.erase(contents.begin() + sep2 + 2, contents.end());
+				if (headerEnd != string::npos) {
+					headerEnd += 2;
+				}
+
+				const size_t sep = contents.find("\n\r\n", searchStart);
+
+				if (sep != string::npos && sep + 3 < headerEnd) {
+					headerEnd = sep + 3;
+				}
+
+				if (headerEnd != string::npos) {
+					contents.erase(headerEnd);
 					break;
 				}
 			}
@@ -389,13 +400,13 @@ shared_ptr <header> maildirMessage::getOrCreateHeader() {
 
 shared_ptr <vmime::message> maildirMessage::getParsedMessage() {
 
-	std::ostringstream oss;
-	utility::outputStreamAdapter os(oss);
+	string buffer;
+	utility::outputStreamStringAdapter os(buffer);
 
 	extract(os);
 
 	shared_ptr <vmime::message> msg = make_shared <vmime::message>();
-	msg->parse(oss.str());
+	msg->parse(buffer);
 
 	return msg;
 }

src/vmime/net/pop3/POP3Message.cpp

@@ -33,11 +33,8 @@
 #include "vmime/net/pop3/POP3Folder.hpp"
 #include "vmime/net/pop3/POP3Store.hpp"
 
-#include "vmime/utility/outputStreamAdapter.hpp"
 #include "vmime/utility/outputStreamStringAdapter.hpp"
 
-#include <sstream>
-
 
 namespace vmime {
 namespace net {
@@ -262,13 +259,13 @@ void POP3Message::setFlags(const int /* flags */, const int /* mode */) {
 
 shared_ptr <vmime::message> POP3Message::getParsedMessage() {
 
-	std::ostringstream oss;
-	utility::outputStreamAdapter os(oss);
+	string buffer;
+	utility::outputStreamStringAdapter os(buffer);
 
 	extract(os);
 
 	shared_ptr <vmime::message> msg = make_shared <vmime::message>();
-	msg->parse(oss.str());
+	msg->parse(buffer);
 
 	return msg;
 }

src/vmime/utility/inputStreamStringAdapter.cpp

@@ -64,18 +64,22 @@ void inputStreamStringAdapter::reset() {
 
 size_t inputStreamStringAdapter::read(byte_t* const data, const size_t count) {
 
+	// Copy from bytes rather than chars: with the same source and destination
+	// types, std::copy() is a memmove() instead of a byte-by-byte loop
+	const byte_t* const buffer = reinterpret_cast <const byte_t*>(m_buffer.data());
+
 	const size_t remaining = m_end - m_pos;
 
 	if (remaining <= count) {
 
-		std::copy(m_buffer.begin() + m_pos, m_buffer.begin() + m_end, data);
+		std::copy(buffer + m_pos, buffer + m_end, data);
 		m_pos = m_end;
 
 		return remaining;
 
 	} else {
 
-		std::copy(m_buffer.begin() + m_pos, m_buffer.begin() + m_pos + count, data);
+		std::copy(buffer + m_pos, buffer + m_pos + count, data);
 		m_pos += count;
 
 		return count;

src/vmime/utility/parserInputStreamAdapter.cpp

@@ -62,28 +62,23 @@ const string parserInputStreamAdapter::extract(const size_t begin, const size_t
 
 	const size_t initialPos = m_stream->getPosition();
 
-	byte_t *buffer = NULL;
-
 	try {
 
-		buffer = new byte_t[end - begin + 1];
+		// Read directly into the string: going through a byte_t buffer costs
+		// a second copy, which is done byte by byte (types differ)
+		string str(end - begin, '\0');
 
 		m_stream->seek(begin);
 
-		const size_t readBytes = m_stream->read(buffer, end - begin);
-		buffer[readBytes] = '\0';
+		const size_t readBytes = m_stream->read(reinterpret_cast <byte_t*>(&str[0]), str.length());
+		str.resize(readBytes);
 
 		m_stream->seek(initialPos);
 
-		string str(buffer, buffer + readBytes);
-		delete [] buffer;
-
 		return str;
 
 	} catch (...) {
 
-		delete [] buffer;
-
 		m_stream->seek(initialPos);
 		throw;
 	}

tests/net/maildir/maildirStoreTest.cpp

@@ -158,6 +158,8 @@ VMIME_TEST_SUITE_BEGIN(maildirStoreTest)
 		VMIME_TEST(testListMessages_KMail)
 		VMIME_TEST(testListMessages_Courier)
 
+		VMIME_TEST(testFetchHeader)
+
 		VMIME_TEST(testRenameFolder_KMail)
 		VMIME_TEST(testRenameFolder_Courier)
 
@@ -325,6 +327,55 @@ VMIME_TEST_SUITE_BEGIN(maildirStoreTest)
 	}
 
 
+	void testFetchHeader() {
+
+		// The empty line which ends the header straddles the 1 KiB blocks
+		// in which the message file is read
+		const vmime::string message =
+			"From: <test@vmime.org>\r\n"
+			"X-Pad: " + vmime::string(968, 'x') + "\r\n"
+			"Subject: Large header\r\n"
+			"\r\n"
+			"Body-Field: not a header\r\n"
+			"\r\n"
+			"Hello, world!";
+
+		VASSERT_EQ("Separator position", 1022, message.find("\r\n\r\n"));
+
+		const vmime::string files[] = {
+			"/Folder/cur/1043236113.351.EmqD" TEST_FLAGS_DELIMITER "S", message,
+			"*"  // end
+		};
+
+		createMaildir(TEST_MAILDIR_KMAIL, files);
+
+		vmime::shared_ptr <vmime::net::store> store = createAndConnectStore();
+		vmime::shared_ptr <vmime::net::folder> folder = store->getFolder(fpath() / "Folder");
+
+		folder->open(vmime::net::folder::MODE_READ_ONLY);
+
+		vmime::shared_ptr <vmime::net::message> msg = folder->getMessage(1);
+
+		folder->fetchMessage(msg, vmime::net::fetchAttributes::FULL_HEADER);
+
+		VASSERT_EQ("Field count", 3, msg->getHeader()->getFieldCount());
+		VASSERT_EQ("Last field", "Subject", msg->getHeader()->getFieldAt(2)->getName());
+
+		vmime::shared_ptr <vmime::message> parsedMsg = msg->getParsedMessage();
+
+		std::ostringstream oss;
+		vmime::utility::outputStreamAdapter os(oss);
+		parsedMsg->getBody()->getContents()->extract(os);
+
+		VASSERT_EQ("Parsed field count", 3, parsedMsg->getHeader()->getFieldCount());
+		VASSERT_EQ("Parsed body", "Body-Field: not a header\r\n\r\nHello, world!", oss.str());
+
+		folder->close(false);
+
+		destroyMaildir();
+	}
+
+
 	void testRenameFolder_KMail() {
 
 		try {

tests/parser/bodyPartTest.cpp

@@ -24,6 +24,39 @@
 #include "tests/testUtils.hpp"
 
 
+// Seekable stream which counts the bytes read from it
+class readCountingInputStream : public vmime::utility::seekableInputStream {
+
+public:
+
+	readCountingInputStream(const vmime::string& str)
+		: m_stream(str), m_readCount(0) {
+
+	}
+
+	bool eof() const { return m_stream.eof(); }
+	void reset() { m_stream.reset(); }
+	size_t skip(const size_t count) { return m_stream.skip(count); }
+	size_t getPosition() const { return m_stream.getPosition(); }
+	void seek(const size_t pos) { m_stream.seek(pos); }
+
+	size_t read(vmime::byte_t* const data, const size_t count) {
+
+		const size_t n = m_stream.read(data, count);
+		m_readCount += n;
+
+		return n;
+	}
+
+	size_t getReadCount() const { return m_readCount; }
+
+private:
+
+	vmime::utility::inputStreamStringAdapter m_stream;
+	size_t m_readCount;
+};
+
+
 VMIME_TEST_SUITE_BEGIN(bodyPartTest)
 
 	VMIME_TEST_LIST_BEGIN
@@ -39,6 +72,7 @@ VMIME_TEST_SUITE_BEGIN(bodyPartTest)
 		VMIME_TEST(testGenerate7bit)
 		VMIME_TEST(testTextUsageForQPEncoding)
 		VMIME_TEST(testParseVeryBigMessage)
+		VMIME_TEST(testParseHeaderDoesNotReadBody)
 		VMIME_TEST(testParseBoundaryPrefix)
 		VMIME_TEST(testParseMissingLastBoundaryAtEnd)
 		VMIME_TEST(testParseMissingLastBoundaryInSubPart)
@@ -335,6 +369,28 @@ VMIME_TEST_SUITE_BEGIN(bodyPartTest)
 		VASSERT_EQ("part2-body", "BODY2", extractContents(p.getBody()->getPartAt(1)->getBody()->getContents()));
 	}
 
+	void testParseHeaderDoesNotReadBody() {
+
+		// When parsing from a stream, the body which follows the header
+		// should not be read (nor copied) to parse the header
+		const vmime::string body(1024 * 1024, 'x');
+		const vmime::string str =
+			"Content-Type: application/octet-stream\r\n"
+			"Content-Transfer-Encoding: binary\r\n"
+			"\r\n"
+			+ body;
+
+		vmime::shared_ptr <readCountingInputStream> is =
+			vmime::make_shared <readCountingInputStream>(str);
+
+		vmime::bodyPart p;
+		p.parse(is, str.length());
+
+		VASSERT_EQ("field-count", 2, p.getHeader()->getFieldCount());
+		VASSERT_EQ("body-length", body.length(), p.getBody()->getContents()->getLength());
+		VASSERT("read-count", is->getReadCount() < 4096);
+	}
+
 	void testParseVeryBigMessage() {
 
 		// When parsing from a seekable input stream, body contents should not