diff --git a/src/vmime/header.cpp b/src/vmime/header.cpp index b2367dfa..905e986e 100644 --- a/src/vmime/header.cpp +++ b/src/vmime/header.cpp @@ -42,6 +42,48 @@ header::~header() { } +void header::parseImpl( + parsingContext& ctx, + const shared_ptr & parser, + const size_t position, + const size_t end, + size_t* newPosition +) { + + // Fields cannot extend beyond the first empty line, which ends the + // header: only extract up to it, not the body which follows + size_t headerEnd = end; + + for (size_t pos = position ; pos < end ; ) { + + parser->seek(pos); + + const byte_t c = parser->peekByte(); + + if (c == '\n') { + + headerEnd = pos + 1; + break; + + } else if (c == '\r' && pos + 1 < end && parser->matchBytes("\r\n", 2)) { + + headerEnd = pos + 2; + break; + } + + const size_t eol = parser->findNext("\n", pos, end); + + if (eol == npos) { + break; + } + + pos = eol + 1; + } + + component::parseImpl(ctx, parser, position, headerEnd, newPosition); +} + + /* RFC #822: diff --git a/src/vmime/header.hpp b/src/vmime/header.hpp index 2d4fa772..d325a3fb 100644 --- a/src/vmime/header.hpp +++ b/src/vmime/header.hpp @@ -339,6 +339,14 @@ class VMIME_EXPORT header : public component { protected: // Component parsing & assembling + void parseImpl( + parsingContext& ctx, + const shared_ptr & parser, + const size_t position, + const size_t end, + size_t* newPosition = NULL + ); + void parseImpl( parsingContext& ctx, const string& buffer, diff --git a/src/vmime/net/maildir/maildirMessage.cpp b/src/vmime/net/maildir/maildirMessage.cpp index ae99c597..ae38088d 100644 --- a/src/vmime/net/maildir/maildirMessage.cpp +++ b/src/vmime/net/maildir/maildirMessage.cpp @@ -39,7 +39,7 @@ #include "vmime/exception.hpp" #include "vmime/platform.hpp" -#include "vmime/utility/outputStreamAdapter.hpp" +#include "vmime/utility/outputStreamStringAdapter.hpp" #include "vmime/utility/stringUtils.hpp" @@ -341,17 +341,28 @@ void maildirMessage::fetch(const shared_ptr & msgFolder, const fe while (!is->eof()) { + // Only search the empty line which ends the header in the new data + // (and the last bytes before, which may be the start of it), not + // again in all the data read so far + const size_t searchStart = (contents.length() >= 2 ? contents.length() - 2 : 0); + const size_t read = is->read(buffer, sizeof(buffer)); vmime::utility::stringUtils::appendBytesToString(contents, buffer, read); - const size_t sep1 = contents.rfind("\r\n\r\n"); - const size_t sep2 = contents.rfind("\n\n"); + size_t headerEnd = contents.find("\n\n", searchStart); - if (sep1 != string::npos) { - contents.erase(contents.begin() + sep1 + 4, contents.end()); - break; - } else if (sep2 != string::npos) { - contents.erase(contents.begin() + sep2 + 2, contents.end()); + if (headerEnd != string::npos) { + headerEnd += 2; + } + + const size_t sep = contents.find("\n\r\n", searchStart); + + if (sep != string::npos && sep + 3 < headerEnd) { + headerEnd = sep + 3; + } + + if (headerEnd != string::npos) { + contents.erase(headerEnd); break; } } @@ -389,13 +400,13 @@ shared_ptr
maildirMessage::getOrCreateHeader() { shared_ptr maildirMessage::getParsedMessage() { - std::ostringstream oss; - utility::outputStreamAdapter os(oss); + string buffer; + utility::outputStreamStringAdapter os(buffer); extract(os); shared_ptr msg = make_shared (); - msg->parse(oss.str()); + msg->parse(buffer); return msg; } diff --git a/src/vmime/net/pop3/POP3Message.cpp b/src/vmime/net/pop3/POP3Message.cpp index 8d6b7f53..221ffa48 100644 --- a/src/vmime/net/pop3/POP3Message.cpp +++ b/src/vmime/net/pop3/POP3Message.cpp @@ -33,11 +33,8 @@ #include "vmime/net/pop3/POP3Folder.hpp" #include "vmime/net/pop3/POP3Store.hpp" -#include "vmime/utility/outputStreamAdapter.hpp" #include "vmime/utility/outputStreamStringAdapter.hpp" -#include - namespace vmime { namespace net { @@ -262,13 +259,13 @@ void POP3Message::setFlags(const int /* flags */, const int /* mode */) { shared_ptr POP3Message::getParsedMessage() { - std::ostringstream oss; - utility::outputStreamAdapter os(oss); + string buffer; + utility::outputStreamStringAdapter os(buffer); extract(os); shared_ptr msg = make_shared (); - msg->parse(oss.str()); + msg->parse(buffer); return msg; } diff --git a/src/vmime/utility/inputStreamStringAdapter.cpp b/src/vmime/utility/inputStreamStringAdapter.cpp index c79744a0..a2f56bf8 100644 --- a/src/vmime/utility/inputStreamStringAdapter.cpp +++ b/src/vmime/utility/inputStreamStringAdapter.cpp @@ -64,18 +64,22 @@ void inputStreamStringAdapter::reset() { size_t inputStreamStringAdapter::read(byte_t* const data, const size_t count) { + // Copy from bytes rather than chars: with the same source and destination + // types, std::copy() is a memmove() instead of a byte-by-byte loop + const byte_t* const buffer = reinterpret_cast (m_buffer.data()); + const size_t remaining = m_end - m_pos; if (remaining <= count) { - std::copy(m_buffer.begin() + m_pos, m_buffer.begin() + m_end, data); + std::copy(buffer + m_pos, buffer + m_end, data); m_pos = m_end; return remaining; } else { - std::copy(m_buffer.begin() + m_pos, m_buffer.begin() + m_pos + count, data); + std::copy(buffer + m_pos, buffer + m_pos + count, data); m_pos += count; return count; diff --git a/src/vmime/utility/parserInputStreamAdapter.cpp b/src/vmime/utility/parserInputStreamAdapter.cpp index 3635c059..211d7209 100644 --- a/src/vmime/utility/parserInputStreamAdapter.cpp +++ b/src/vmime/utility/parserInputStreamAdapter.cpp @@ -62,28 +62,23 @@ const string parserInputStreamAdapter::extract(const size_t begin, const size_t const size_t initialPos = m_stream->getPosition(); - byte_t *buffer = NULL; - try { - buffer = new byte_t[end - begin + 1]; + // Read directly into the string: going through a byte_t buffer costs + // a second copy, which is done byte by byte (types differ) + string str(end - begin, '\0'); m_stream->seek(begin); - const size_t readBytes = m_stream->read(buffer, end - begin); - buffer[readBytes] = '\0'; + const size_t readBytes = m_stream->read(reinterpret_cast (&str[0]), str.length()); + str.resize(readBytes); m_stream->seek(initialPos); - string str(buffer, buffer + readBytes); - delete [] buffer; - return str; } catch (...) { - delete [] buffer; - m_stream->seek(initialPos); throw; } diff --git a/tests/net/maildir/maildirStoreTest.cpp b/tests/net/maildir/maildirStoreTest.cpp index b337492d..5b4b6fbf 100644 --- a/tests/net/maildir/maildirStoreTest.cpp +++ b/tests/net/maildir/maildirStoreTest.cpp @@ -158,6 +158,8 @@ VMIME_TEST_SUITE_BEGIN(maildirStoreTest) VMIME_TEST(testListMessages_KMail) VMIME_TEST(testListMessages_Courier) + VMIME_TEST(testFetchHeader) + VMIME_TEST(testRenameFolder_KMail) VMIME_TEST(testRenameFolder_Courier) @@ -325,6 +327,55 @@ VMIME_TEST_SUITE_BEGIN(maildirStoreTest) } + void testFetchHeader() { + + // The empty line which ends the header straddles the 1 KiB blocks + // in which the message file is read + const vmime::string message = + "From: \r\n" + "X-Pad: " + vmime::string(968, 'x') + "\r\n" + "Subject: Large header\r\n" + "\r\n" + "Body-Field: not a header\r\n" + "\r\n" + "Hello, world!"; + + VASSERT_EQ("Separator position", 1022, message.find("\r\n\r\n")); + + const vmime::string files[] = { + "/Folder/cur/1043236113.351.EmqD" TEST_FLAGS_DELIMITER "S", message, + "*" // end + }; + + createMaildir(TEST_MAILDIR_KMAIL, files); + + vmime::shared_ptr store = createAndConnectStore(); + vmime::shared_ptr folder = store->getFolder(fpath() / "Folder"); + + folder->open(vmime::net::folder::MODE_READ_ONLY); + + vmime::shared_ptr msg = folder->getMessage(1); + + folder->fetchMessage(msg, vmime::net::fetchAttributes::FULL_HEADER); + + VASSERT_EQ("Field count", 3, msg->getHeader()->getFieldCount()); + VASSERT_EQ("Last field", "Subject", msg->getHeader()->getFieldAt(2)->getName()); + + vmime::shared_ptr parsedMsg = msg->getParsedMessage(); + + std::ostringstream oss; + vmime::utility::outputStreamAdapter os(oss); + parsedMsg->getBody()->getContents()->extract(os); + + VASSERT_EQ("Parsed field count", 3, parsedMsg->getHeader()->getFieldCount()); + VASSERT_EQ("Parsed body", "Body-Field: not a header\r\n\r\nHello, world!", oss.str()); + + folder->close(false); + + destroyMaildir(); + } + + void testRenameFolder_KMail() { try { diff --git a/tests/parser/bodyPartTest.cpp b/tests/parser/bodyPartTest.cpp index b82f27b4..86a8f0a2 100644 --- a/tests/parser/bodyPartTest.cpp +++ b/tests/parser/bodyPartTest.cpp @@ -24,6 +24,39 @@ #include "tests/testUtils.hpp" +// Seekable stream which counts the bytes read from it +class readCountingInputStream : public vmime::utility::seekableInputStream { + +public: + + readCountingInputStream(const vmime::string& str) + : m_stream(str), m_readCount(0) { + + } + + bool eof() const { return m_stream.eof(); } + void reset() { m_stream.reset(); } + size_t skip(const size_t count) { return m_stream.skip(count); } + size_t getPosition() const { return m_stream.getPosition(); } + void seek(const size_t pos) { m_stream.seek(pos); } + + size_t read(vmime::byte_t* const data, const size_t count) { + + const size_t n = m_stream.read(data, count); + m_readCount += n; + + return n; + } + + size_t getReadCount() const { return m_readCount; } + +private: + + vmime::utility::inputStreamStringAdapter m_stream; + size_t m_readCount; +}; + + VMIME_TEST_SUITE_BEGIN(bodyPartTest) VMIME_TEST_LIST_BEGIN @@ -39,6 +72,7 @@ VMIME_TEST_SUITE_BEGIN(bodyPartTest) VMIME_TEST(testGenerate7bit) VMIME_TEST(testTextUsageForQPEncoding) VMIME_TEST(testParseVeryBigMessage) + VMIME_TEST(testParseHeaderDoesNotReadBody) VMIME_TEST(testParseBoundaryPrefix) VMIME_TEST(testParseMissingLastBoundaryAtEnd) VMIME_TEST(testParseMissingLastBoundaryInSubPart) @@ -335,6 +369,28 @@ VMIME_TEST_SUITE_BEGIN(bodyPartTest) VASSERT_EQ("part2-body", "BODY2", extractContents(p.getBody()->getPartAt(1)->getBody()->getContents())); } + void testParseHeaderDoesNotReadBody() { + + // When parsing from a stream, the body which follows the header + // should not be read (nor copied) to parse the header + const vmime::string body(1024 * 1024, 'x'); + const vmime::string str = + "Content-Type: application/octet-stream\r\n" + "Content-Transfer-Encoding: binary\r\n" + "\r\n" + + body; + + vmime::shared_ptr is = + vmime::make_shared (str); + + vmime::bodyPart p; + p.parse(is, str.length()); + + VASSERT_EQ("field-count", 2, p.getHeader()->getFieldCount()); + VASSERT_EQ("body-length", body.length(), p.getBody()->getContents()->getLength()); + VASSERT("read-count", is->getReadCount() < 4096); + } + void testParseVeryBigMessage() { // When parsing from a seekable input stream, body contents should not diff --git a/tests/parser/headerTest.cpp b/tests/parser/headerTest.cpp index 839acee5..e2fef9a0 100644 --- a/tests/parser/headerTest.cpp +++ b/tests/parser/headerTest.cpp @@ -61,6 +61,8 @@ VMIME_TEST_SUITE_BEGIN(headerTest) VMIME_TEST(testFindAllFields1) VMIME_TEST(testFindAllFields2) VMIME_TEST(testFindAllFields3) + + VMIME_TEST(testParseFromStream) VMIME_TEST_LIST_END @@ -372,4 +374,64 @@ VMIME_TEST_SUITE_BEGIN(headerTest) VASSERT_EQ("Second value", "C: c2", headerTest::getFieldValue(*res[2])); } + void testParseFromStream() { + + // When parsing from a stream, the header is only extracted up to the + // first empty line: results must be the same as from a string + struct TestCase { + const char* data; + vmime::headerParseRecoveryMethod::headerLineError recovery; + size_t fieldCount; + size_t end; + }; + + static const vmime::headerParseRecoveryMethod::headerLineError SKIP_LINE = + vmime::headerParseRecoveryMethod::SKIP_LINE; + static const vmime::headerParseRecoveryMethod::headerLineError ASSUME_END_OF_HEADERS = + vmime::headerParseRecoveryMethod::ASSUME_END_OF_HEADERS; + + static const TestCase testCases[] = { + { "A: a\r\nB: b\r\n\r\nBODY: x\r\n\r\n", SKIP_LINE, 2, 14 }, + { "A: a\nB: b\n\nBODY: x\n\n", SKIP_LINE, 2, 11 }, + { "A: a\r\nB: b\r\n", SKIP_LINE, 2, 12 }, // no empty line + { "\r\nBODY: x\r\n", SKIP_LINE, 0, 2 }, + { "\nBODY: x\n", SKIP_LINE, 0, 1 }, + { "A: a\r\n \r\nB: b\r\n\r\nBODY: x\r\n", SKIP_LINE, 2, 17 }, // folding + { "A: a\r\n\tfolded\r\n\r\nBODY: x\r\n", SKIP_LINE, 1, 17 }, + { "A: a\r\n\r\r\nB: b\r\n\r\nBODY: x\r\n", SKIP_LINE, 2, 17 }, // not empty + { " lead\r\nA: a\r\n\r\nBODY: x\r\n", SKIP_LINE, 1, 15 }, + { "A: a\r\ngarbage\r\nB: b\r\n\r\nBODY: x\r\n", SKIP_LINE, 2, 23 }, + { "A: a\r\ngarbage\r\nB: b\r\n\r\nBODY: x\r\n", ASSUME_END_OF_HEADERS, 1, 6 }, + }; + + for (size_t i = 0 ; i < sizeof(testCases) / sizeof(testCases[0]) ; ++i) { + + const TestCase& tc = testCases[i]; + const vmime::string data(tc.data); + + std::ostringstream oss; + oss << "Test case #" << i; + const vmime::string msg = oss.str(); + + vmime::parsingContext ctx; + ctx.setHeaderParseErrorRecoveryMethod(tc.recovery); + + vmime::header hdr1; + size_t end1 = 0; + hdr1.parse(ctx, data, 0, data.length(), &end1); + + vmime::header hdr2; + size_t end2 = 0; + hdr2.parse(ctx, vmime::make_shared (data), 0, data.length(), &end2); + + VASSERT_EQ(msg + ": field count", tc.fieldCount, hdr1.getFieldCount()); + VASSERT_EQ(msg + ": end", tc.end, end1); + + VASSERT_EQ(msg + ": stream field count", hdr1.getFieldCount(), hdr2.getFieldCount()); + VASSERT_EQ(msg + ": stream end", end1, end2); + VASSERT_EQ(msg + ": stream length", hdr1.getParsedLength(), hdr2.getParsedLength()); + VASSERT_EQ(msg + ": stream generate", hdr1.generate(), hdr2.generate()); + } + } + VMIME_TEST_SUITE_END