Skip to content

Move JSON scanning methods out of the probes #23

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mrabine merged 1 commit into from
Nov 1, 2026
Merged

Move JSON scanning methods out of the probes #23

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion VERSION
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1.1.0
1.2.0
276 changes: 8 additions & 268 deletions dconv/include/dconv/probe.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,7 @@
#define __DCONV_PROBE_HPP__

// C++.
#include <string_view>
#include <istream>
#include <string>
#include <limits>

// C.
Expand All @@ -41,43 +39,6 @@

namespace dconv
{
namespace details
{
struct alignas (64) EscapedTable
{
uint8_t data[256];

constexpr EscapedTable ()
: data{}
{
for (unsigned i = 0; i < 0x20; ++i)
{
data[i] = 1;
}
data[static_cast<unsigned char> ('"')] = 1;
data[static_cast<unsigned char> ('\\')] = 1;
}
};

constexpr EscapedTable escapedLookup{};

struct alignas (64) WhitespaceTable
{
uint8_t data[256];

constexpr WhitespaceTable ()
: data{}
{
data[static_cast<unsigned char> ('\t')] = 1;
data[static_cast<unsigned char> ('\n')] = 1;
data[static_cast<unsigned char> ('\r')] = 1;
data[static_cast<unsigned char> (' ')] = 1;
}
};

constexpr WhitespaceTable whitespaceLookup{};
}

/**
* @brief string probe.
*/
Expand Down Expand Up @@ -262,145 +223,21 @@ namespace dconv
}

/**
* @brief read characters until escaped, without copying them.
* @return characters read.
*/
inline std::string_view readUntilEscaped () noexcept
{
const char* beg = _cur;
const char* cur = _cur;
const char* end = _end;

while (end - cur >= 8)
{
uint64_t word, quote, backslash;
std::memcpy (&word, cur, 8);
quote = word ^ 0x2222222222222222ULL;
backslash = word ^ 0x5C5C5C5C5C5C5C5CULL;
uint64_t mask =
(((quote - 0x0101010101010101ULL) & ~quote) | ((backslash - 0x0101010101010101ULL) & ~backslash) |
((word - 0x2020202020202020ULL) & ~word)) &
0x8080808080808080ULL;
if (mask)
{
cur += __builtin_ctzll (mask) >> 3;
break;
}
cur += 8;
}

while (cur < end && !details::escapedLookup.data[static_cast<unsigned char> (*cur)])
{
++cur;
}

_cur = cur;
return std::string_view (beg, static_cast<size_t> (cur - beg));
}

/**
* @brief read characters until escaped.
* @param out output buffer.
*/
inline void readUntilEscaped (std::string& out) noexcept
{
out.append (readUntilEscaped ());
}

/**
* @brief skip whitespaces.
* @return 0 on success, -1 otherwise.
*/
inline int skipWhitespaces () noexcept
{
const char* cur = _cur;
const char* end = _end;

while (cur < end && details::whitespaceLookup.data[static_cast<unsigned char> (*cur)])
{
++cur;

uint64_t word;
while ((end - cur >= 8) && (std::memcpy (&word, cur, 8), word == 0x2020202020202020ULL))
{
cur += 8;
}
}

_cur = cur;

return 0;
}

/**
* @brief skip whitespaces and comments.
* @return 0 on success, -1 otherwise.
* @brief get input position indicator.
* @return current position.
*/
inline int skipWhitespacesAndComments () noexcept
inline ProbePos tell () const noexcept
{
const char* cur = _cur;
const char* end = _end;

while (cur < end)
{
while (cur < end && details::whitespaceLookup.data[static_cast<unsigned char> (*cur)])
{
++cur;
}

if (cur >= end || *cur != '/')
{
break;
}

if (++cur >= end)
{
return -1;
}

if (*cur == '*')
{
++cur;
bool closed = false;
while (cur < end)
{
if (*cur == '*' && (cur + 1 < end) && *(cur + 1) == '/')
{
cur += 2;
closed = true;
break;
}
++cur;
}
if (!closed)
{
return -1;
}
}
else if (*cur == '/')
{
++cur;
const char* p = static_cast<const char*> (memchr (cur, '\n', end - cur));
cur = p ? p : end;
}
else
{
return -1;
}
}

_cur = cur;

return 0;
return _cur;
}

/**
* @brief get input position indicator.
* @return current position.
* @brief get end position indicator.
* @return end position.
*/
inline ProbePos tell () const noexcept
inline ProbePos end () const noexcept
{
return _cur;
return _end;
}

/**
Expand Down Expand Up @@ -560,103 +397,6 @@ namespace dconv
return _in->sgetn (buf, count);
}

/**
* @brief read characters until escaped.
* @param out output buffer.
*/
inline void readUntilEscaped (std::string& out) noexcept
{
int c;
while ((c = _in->sgetc ()) != std::char_traits<char>::eof () &&
!details::escapedLookup.data[static_cast<unsigned char> (c)])
{
out.push_back (static_cast<char> (c));
_in->sbumpc ();
}
}

/**
* @brief skip whitespaces.
* @return 0 on success, -1 otherwise.
*/
inline int skipWhitespaces () noexcept
{
int c;
while ((c = _in->sgetc ()) != std::char_traits<char>::eof () &&
details::whitespaceLookup.data[static_cast<unsigned char> (c)])
{
_in->sbumpc ();
}

return 0;
}

/**
* @brief skip whitespaces and comments.
* @return 0 on success, -1 otherwise.
*/
inline int skipWhitespacesAndComments () noexcept
{
int c;

while ((c = _in->sgetc ()) != std::char_traits<char>::eof ())
{
while ((c = _in->sgetc ()) != std::char_traits<char>::eof () &&
details::whitespaceLookup.data[static_cast<unsigned char> (c)])
{
_in->sbumpc ();
}

if (c != '/')
{
break;
}

_in->sbumpc ();
c = _in->sgetc ();

if (c == std::char_traits<char>::eof ())
{
return -1;
}

if (c == '*')
{
_in->sbumpc ();
bool closed = false;

while ((c = _in->sbumpc ()) != std::char_traits<char>::eof ())
{
if (c == '*' && _in->sgetc () == '/')
{
_in->sbumpc ();
closed = true;
break;
}
}

if (!closed)
{
return -1;
}
}
else if (c == '/')
{
_in->sbumpc ();

while ((c = _in->sbumpc ()) != std::char_traits<char>::eof () && c != '\n')
{
}
}
else
{
return -1;
}
}

return 0;
}

private:
/// input stream buffer.
std::streambuf* _in;
Expand Down
71 changes: 0 additions & 71 deletions dconv/tests/stream_probe_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -167,77 +167,6 @@ TEST (StreamProbe, read)
ASSERT_EQ (probe.read (buf, 8), 0);
}

/**
* @brief readUntilEscaped test.
*/
TEST (StreamProbe, readUntilEscaped)
{
std::stringstream msg ("hello\"world");
StreamProbe probe (msg);
std::string out;

probe.readUntilEscaped (out);
ASSERT_EQ (out, "hello");
ASSERT_EQ (probe.peek (), '"');
}

/**
* @brief skipWhitespaces test.
*/
TEST (StreamProbe, skipWhitespaces)
{
std::stringstream msg (" hello");
StreamProbe probe (msg);
std::string out;

probe.skipWhitespaces ();
ASSERT_EQ (probe.peek (), 'h');
}

/**
* @brief skipWhitespacesAndComments test.
*/
TEST (StreamProbe, skipWhitespacesAndComments)
{
std::stringstream msg (" // comment\nhello");
StreamProbe probe1 (msg);
ASSERT_EQ (probe1.skipWhitespacesAndComments (), 0);
ASSERT_EQ (probe1.peek (), 'h');

msg.str (" /* comment */hello");
msg.clear ();
StreamProbe probe2 (msg);
ASSERT_EQ (probe2.skipWhitespacesAndComments (), 0);
ASSERT_EQ (probe2.peek (), 'h');

msg.str ("// comment");
msg.clear ();
StreamProbe probe3 (msg);
ASSERT_EQ (probe3.skipWhitespacesAndComments (), 0);
ASSERT_EQ (probe3.peek (), std::char_traits<char>::eof ());

msg.str ("/* comment */");
msg.clear ();
StreamProbe probe4 (msg);
ASSERT_EQ (probe4.skipWhitespacesAndComments (), 0);
ASSERT_EQ (probe4.peek (), std::char_traits<char>::eof ());

msg.str ("/");
msg.clear ();
StreamProbe probe5 (msg);
ASSERT_EQ (probe5.skipWhitespacesAndComments (), -1);

msg.str ("/*");
msg.clear ();
StreamProbe probe6 (msg);
ASSERT_EQ (probe6.skipWhitespacesAndComments (), -1);

msg.str ("/!");
msg.clear ();
StreamProbe probe7 (msg);
ASSERT_EQ (probe7.skipWhitespacesAndComments (), -1);
}

/**
* @brief main function.
*/
Expand Down
Loading
Loading