diff --git a/README.md b/README.md index b23f2318..9595fdf0 100644 --- a/README.md +++ b/README.md @@ -85,8 +85,8 @@ sudo apt install pkg-config libssl-dev zlib1g-dev libgtest-dev libgmock-dev Install [dconv](https://github.com/joinframework/dconv) from its release package: ```bash -wget https://github.com/joinframework/dconv/releases/download/v1.1.0/libdconv-dev_1.1.0_all.deb -sudo apt install ./libdconv-dev_1.1.0_all.deb +wget https://github.com/joinframework/dconv/releases/download/v1.2.1/libdconv-dev_1.2.1_all.deb +sudo apt install ./libdconv-dev_1.2.1_all.deb ``` > **Compilers:** GCC 11+ or Clang 16+ (C++20). Clang requires `libclang-rt-dev` for coverage instrumentation (`--coverage`). diff --git a/cmake/joinConfig.cmake.in b/cmake/joinConfig.cmake.in index c65561ac..8f6c8e52 100644 --- a/cmake/joinConfig.cmake.in +++ b/cmake/joinConfig.cmake.in @@ -23,7 +23,7 @@ foreach(comp ${join_FIND_COMPONENTS}) endforeach() if(@JOIN_ENABLE_DATA@ OR @JOIN_ENABLE_SERVICES@) - find_dependency(dconv 1.1.0) + find_dependency(dconv 1.2.1) find_dependency(ZLIB) endif() diff --git a/cmake/packaging.cmake b/cmake/packaging.cmake index 24b2d716..15226ffa 100644 --- a/cmake/packaging.cmake +++ b/cmake/packaging.cmake @@ -32,7 +32,7 @@ if(JOIN_ENABLE_CRYPTO) string(APPEND JOIN_DEV_DEPENDS ", libssl-dev") endif() if(JOIN_ENABLE_DATA) - string(APPEND JOIN_DEV_DEPENDS ", libdconv-dev (>= 1.1.0), zlib1g-dev") + string(APPEND JOIN_DEV_DEPENDS ", libdconv-dev (>= 1.2.1), zlib1g-dev") endif() if(JOIN_ENABLE_IO_URING) string(APPEND JOIN_DEV_DEPENDS ", liburing-dev") diff --git a/data/CMakeLists.txt b/data/CMakeLists.txt index d42d3b11..468ffe82 100644 --- a/data/CMakeLists.txt +++ b/data/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.22.1) -find_package(dconv 1.1.0 REQUIRED) +find_package(dconv 1.2.1 REQUIRED) find_package(ZLIB REQUIRED) set(PUBLIC_HEADERS diff --git a/data/include/join/json.hpp b/data/include/join/json.hpp index 33ece3f6..72cd6936 100644 --- a/data/include/join/json.hpp +++ b/data/include/join/json.hpp @@ -46,6 +46,10 @@ #include #include +// C. +#include +#include + namespace join { namespace details @@ -85,6 +89,40 @@ namespace join constexpr UnescapedTable unescapedLookup{}; + struct alignas (64) EscapedTable + { + uint8_t data[256]; + + constexpr EscapedTable () + : data{} + { + for (unsigned i = 0; i < 0x20; ++i) + { + data[i] = 1; + } + data[static_cast ('"')] = 1; + data[static_cast ('\\')] = 1; + } + }; + + constexpr EscapedTable escapedLookup{}; + + struct alignas (64) WhitespaceTable + { + uint8_t data[256]; + + constexpr WhitespaceTable () + : data{} + { + data[static_cast ('\t')] = 1; + data[static_cast ('\n')] = 1; + data[static_cast ('\r')] = 1; + data[static_cast (' ')] = 1; + } + }; + + constexpr WhitespaceTable whitespaceLookup{}; + constexpr char hexDigits[17] = "0123456789abcdef"; } @@ -1607,9 +1645,9 @@ namespace join { std::string_view chunk; - if constexpr (requires { document.readUntilEscaped (); }) + if constexpr (std::is_same_v) { - chunk = document.readUntilEscaped (); + chunk = readUntilEscaped (document); if (JOIN_UNLIKELY (validateChunk (chunk) == -1)) { @@ -1630,7 +1668,7 @@ namespace join for (;;) { size_t pos = output.size (); - document.readUntilEscaped (output); + readUntilEscaped (document, output); if (JOIN_UNLIKELY (validateChunk (std::string_view (output).substr (pos)) == -1)) { @@ -1707,7 +1745,7 @@ namespace join int ch = document.get (); - if (JOIN_UNLIKELY (dconv::details::whitespaceLookup.data[static_cast (ch)])) + if (JOIN_UNLIKELY (details::whitespaceLookup.data[static_cast (ch)])) { if (JOIN_UNLIKELY (skipWhitespaces (document) == -1)) { @@ -1807,7 +1845,7 @@ namespace join int ch = document.get (); - if (JOIN_UNLIKELY (dconv::details::whitespaceLookup.data[static_cast (ch)])) + if (JOIN_UNLIKELY (details::whitespaceLookup.data[static_cast (ch)])) { if (JOIN_UNLIKELY (skipWhitespaces (document) == -1)) { @@ -1837,22 +1875,258 @@ namespace join return handler.stopObject (); } + /** + * @brief read characters until escaped. + * @param document document to parse. + * @return characters read. + */ + inline std::string_view readUntilEscaped (dconv::StringProbe& document) noexcept + { + const char* beg = document.tell (); + const char* cur = beg; + const char* end = document.end (); + + while (end - cur >= 8) + { + uint64_t word, quote, backslash; + std::memcpy (&word, cur, 8); + quote = word ^ 0x2222222222222222ULL; + backslash = word ^ 0x5C5C5C5C5C5C5C5CULL; + uint64_t mask = + (((quote - 0x0101010101010101ULL) & ~quote) | ((backslash - 0x0101010101010101ULL) & ~backslash) | + ((word - 0x2020202020202020ULL) & ~word)) & + 0x8080808080808080ULL; + if (mask) + { + cur += __builtin_ctzll (mask) >> 3; + break; + } + cur += 8; + } + + while (cur < end && !details::escapedLookup.data[static_cast (*cur)]) + { + ++cur; + } + + document.seek (cur); + return std::string_view (beg, static_cast (cur - beg)); + } + + /** + * @brief read characters until escaped. + * @param document document to parse. + * @param out output buffer. + */ + inline void readUntilEscaped (dconv::StringProbe& document, std::string& out) noexcept + { + out.append (readUntilEscaped (document)); + } + + /** + * @brief read characters until escaped. + * @param document document to parse. + * @param out output buffer. + */ + inline void readUntilEscaped (dconv::StreamProbe& document, std::string& out) noexcept + { + int c; + while ((c = document.peek ()) != std::char_traits::eof () && + !details::escapedLookup.data[static_cast (c)]) + { + out.push_back (static_cast (c)); + document.get (); + } + } + /** * @brief skip whitespaces, and comments when comment parsing is enabled. * @param document document to parse. * @return 0 on success, -1 otherwise. */ - template - inline int skipWhitespaces (ProbeType& document) + inline int skipWhitespaces (dconv::StringProbe& document) noexcept { if constexpr (ReadMode & JsonReadMode::ParseComments) { - return document.skipWhitespacesAndComments (); + return skipWhitespacesAndComments (document); } - else + + const char* cur = document.tell (); + const char* end = document.end (); + + while (cur < end && details::whitespaceLookup.data[static_cast (*cur)]) + { + ++cur; + + while (end - cur >= 8) + { + uint64_t word; + std::memcpy (&word, cur, 8); + word ^= 0x2020202020202020ULL; + if (word != 0) + { + cur += __builtin_ctzll (word) >> 3; + break; + } + cur += 8; + } + } + + document.seek (cur); + + return 0; + } + + /** + * @brief skip whitespaces, and comments when comment parsing is enabled. + * @param document document to parse. + * @return 0 on success, -1 otherwise. + */ + inline int skipWhitespaces (dconv::StreamProbe& document) noexcept + { + if constexpr (ReadMode & JsonReadMode::ParseComments) + { + return skipWhitespacesAndComments (document); + } + + int c; + while ((c = document.peek ()) != std::char_traits::eof () && + details::whitespaceLookup.data[static_cast (c)]) + { + document.get (); + } + + return 0; + } + + /** + * @brief skip whitespaces and comments. + * @param document document to parse. + * @return 0 on success, -1 otherwise. + */ + inline int skipWhitespacesAndComments (dconv::StringProbe& document) noexcept + { + const char* cur = document.tell (); + const char* end = document.end (); + + while (cur < end) { - return document.skipWhitespaces (); + while (cur < end && details::whitespaceLookup.data[static_cast (*cur)]) + { + ++cur; + } + + if (cur >= end || *cur != '/') + { + break; + } + + if (++cur >= end) + { + return -1; + } + + if (*cur == '*') + { + ++cur; + bool closed = false; + while (cur < end) + { + if (*cur == '*' && (cur + 1 < end) && *(cur + 1) == '/') + { + cur += 2; + closed = true; + break; + } + ++cur; + } + if (!closed) + { + return -1; + } + } + else if (*cur == '/') + { + ++cur; + const char* p = static_cast (std::memchr (cur, '\n', end - cur)); + cur = p ? p : end; + } + else + { + return -1; + } } + + document.seek (cur); + + return 0; + } + + /** + * @brief skip whitespaces and comments. + * @param document document to parse. + * @return 0 on success, -1 otherwise. + */ + inline int skipWhitespacesAndComments (dconv::StreamProbe& document) noexcept + { + int c; + + while ((c = document.peek ()) != std::char_traits::eof ()) + { + while ((c = document.peek ()) != std::char_traits::eof () && + details::whitespaceLookup.data[static_cast (c)]) + { + document.get (); + } + + if (c != '/') + { + break; + } + + document.get (); + c = document.peek (); + + if (c == std::char_traits::eof ()) + { + return -1; + } + + if (c == '*') + { + document.get (); + bool closed = false; + + while ((c = document.get ()) != std::char_traits::eof ()) + { + if (c == '*' && document.peek () == '/') + { + document.get (); + closed = true; + break; + } + } + + if (!closed) + { + return -1; + } + } + else if (c == '/') + { + document.get (); + + while ((c = document.get ()) != std::char_traits::eof () && c != '\n') + { + } + } + else + { + return -1; + } + } + + return 0; } /** diff --git a/data/tests/json_reader_test.cpp b/data/tests/json_reader_test.cpp index 8f7ca652..7f004635 100644 --- a/data/tests/json_reader_test.cpp +++ b/data/tests/json_reader_test.cpp @@ -1607,6 +1607,146 @@ TEST (JsonReader, encoding) EXPECT_EQ (join::lastError, JsonErrc::InvalidEncoding); } +/** + * @brief skipWhitespaces test. + */ +TEST (JsonReader, skipWhitespaces) +{ + Value value; + + std::string text (" 1"); + ASSERT_EQ (JsonReader ().deserialize (text, value), 0) << join::lastError.message (); + ASSERT_EQ (value, 1); + + text = "\t\r\n 1"; + ASSERT_EQ (JsonReader ().deserialize (text, value), 0) << join::lastError.message (); + ASSERT_EQ (value, 1); + + text = "\n 1"; + ASSERT_EQ (JsonReader ().deserialize (text, value), 0) << join::lastError.message (); + ASSERT_EQ (value, 1); + + text = "\n 1 "; + ASSERT_EQ (JsonReader ().deserialize (text, value), 0) << join::lastError.message (); + ASSERT_EQ (value, 1); + + text = "[\n 1,\n 2\n]"; + ASSERT_EQ (JsonReader ().deserialize (text, value), 0) << join::lastError.message (); + ASSERT_EQ (value[0], 1); + ASSERT_EQ (value[1], 2); + + std::stringstream stream (" 1"); + ASSERT_EQ (JsonReader ().deserialize (stream, value), 0) << join::lastError.message (); + ASSERT_EQ (value, 1); + + stream.clear (); + stream.str ("\t\r\n 1 "); + ASSERT_EQ (JsonReader ().deserialize (stream, value), 0) << join::lastError.message (); + ASSERT_EQ (value, 1); +} + +/** + * @brief skipWhitespacesAndComments test. + */ +TEST (JsonReader, skipWhitespacesAndComments) +{ + using Reader = BasicJsonReader; + + Value value; + + std::string text (" // comment\n1"); + ASSERT_EQ (Reader ().deserialize (text, value), 0) << join::lastError.message (); + ASSERT_EQ (value, 1); + + text = " /* comment */1"; + ASSERT_EQ (Reader ().deserialize (text, value), 0) << join::lastError.message (); + ASSERT_EQ (value, 1); + + text = "1// comment"; + ASSERT_EQ (Reader ().deserialize (text, value), 0) << join::lastError.message (); + ASSERT_EQ (value, 1); + + text = "1/* comment */"; + ASSERT_EQ (Reader ().deserialize (text, value), 0) << join::lastError.message (); + ASSERT_EQ (value, 1); + + text = "1/"; + ASSERT_EQ (Reader ().deserialize (text, value), -1); + + text = "1/*"; + ASSERT_EQ (Reader ().deserialize (text, value), -1); + + text = "1/!"; + ASSERT_EQ (Reader ().deserialize (text, value), -1); + + std::stringstream stream (" // comment\n1"); + ASSERT_EQ (Reader ().deserialize (stream, value), 0) << join::lastError.message (); + ASSERT_EQ (value, 1); + + stream.clear (); + stream.str (" /* comment */1"); + ASSERT_EQ (Reader ().deserialize (stream, value), 0) << join::lastError.message (); + ASSERT_EQ (value, 1); + + stream.clear (); + stream.str ("1// comment"); + ASSERT_EQ (Reader ().deserialize (stream, value), 0) << join::lastError.message (); + ASSERT_EQ (value, 1); + + stream.clear (); + stream.str ("1/* comment */"); + ASSERT_EQ (Reader ().deserialize (stream, value), 0) << join::lastError.message (); + ASSERT_EQ (value, 1); + + stream.clear (); + stream.str ("1/"); + ASSERT_EQ (Reader ().deserialize (stream, value), -1); + + stream.clear (); + stream.str ("1/*"); + ASSERT_EQ (Reader ().deserialize (stream, value), -1); + + stream.clear (); + stream.str ("1/!"); + ASSERT_EQ (Reader ().deserialize (stream, value), -1); +} + +/** + * @brief readUntilEscaped test. + */ +TEST (JsonReader, readUntilEscaped) +{ + Value value; + + std::string text ("\"abc\""); + ASSERT_EQ (JsonReader ().deserialize (text, value), 0) << join::lastError.message (); + ASSERT_EQ (value, "abc"); + + text = "\"hello wonderful world\""; + ASSERT_EQ (JsonReader ().deserialize (text, value), 0) << join::lastError.message (); + ASSERT_EQ (value, "hello wonderful world"); + + text = "\"hello wonderful\\nworld\""; + ASSERT_EQ (JsonReader ().deserialize (text, value), 0) << join::lastError.message (); + ASSERT_EQ (value, "hello wonderful\nworld"); + + text = "\"abc"; + ASSERT_EQ (JsonReader ().deserialize (text, value), -1); + + std::stringstream stream ("\"hello wonderful world\""); + ASSERT_EQ (JsonReader ().deserialize (stream, value), 0) << join::lastError.message (); + ASSERT_EQ (value, "hello wonderful world"); + + stream.clear (); + stream.str ("\"hello wonderful\\nworld\""); + ASSERT_EQ (JsonReader ().deserialize (stream, value), 0) << join::lastError.message (); + ASSERT_EQ (value, "hello wonderful\nworld"); + + stream.clear (); + stream.str ("\"abc"); + ASSERT_EQ (JsonReader ().deserialize (stream, value), -1); +} + /** * @brief main function. */