From 733c0a3685104595870fe3d9a4f3f432030a4931 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 13 Sep 2026 13:29:20 +0200 Subject: [PATCH] Bump simdjson to 4.6.11 --- scripts/update_simdjson.sh | 10 + src/simdjson.cpp | 366 ++-- src/simdjson.h | 3396 ++++++++++++++++++++++++------------ 3 files changed, 2578 insertions(+), 1194 deletions(-) create mode 100755 scripts/update_simdjson.sh diff --git a/scripts/update_simdjson.sh b/scripts/update_simdjson.sh new file mode 100755 index 00000000..7def30a4 --- /dev/null +++ b/scripts/update_simdjson.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -ex +DIRNAME=$(dirname "$0") +LATEST_VERSION=$(curl -s -f https://api.github.com/repos/simdjson/simdjson/releases/latest | grep '"tag_name"' | cut -d '"' -f 4) +TEMP_DIR=$(mktemp -d -t simdjson) +curl -L -f https://github.com/simdjson/simdjson/archive/refs/tags/${LATEST_VERSION}.tar.gz | tar -xzf - --strip-components=1 -C $TEMP_DIR +patch $TEMP_DIR/include/simdjson/generic/numberparsing.h $DIRNAME/numberparsing.h.patch +python3 $TEMP_DIR/singleheader/amalgamate.py +cp $TEMP_DIR/singleheader/simdjson.{cpp,h} $DIRNAME/../src +rm -rf $TEMP_DIR diff --git a/src/simdjson.cpp b/src/simdjson.cpp index ed6257a5..0a2945b9 100644 --- a/src/simdjson.cpp +++ b/src/simdjson.cpp @@ -1,4 +1,4 @@ -/* auto-generated on 2026-04-03 15:26:12 -0400. version 4.6.1 Do not edit! */ +/* auto-generated on 2026-08-23 15:51:58 +0200. version 4.6.11 Do not edit! */ /* including simdjson.cpp: */ /* begin file simdjson.cpp */ #define SIMDJSON_SRC_SIMDJSON_CPP @@ -240,7 +240,7 @@ using std::size_t; #endif #elif defined(__PPC64__) || defined(_M_PPC64) #define SIMDJSON_IS_PPC64 1 -#if defined(__ALTIVEC__) +#if defined(__ALTIVEC__) && defined(__POWER8_VECTOR__) #define SIMDJSON_IS_PPC64_VMX 1 #endif // defined(__ALTIVEC__) #else @@ -10052,9 +10052,14 @@ inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parse // Leaving these here so they can be inlined if so desired inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } + if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } // Stage 1 index output - size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; + size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); + if(rounded_capacity + 9 < rounded_capacity) { + return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity + } + size_t max_structures = rounded_capacity + 9; + if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); if (!structural_indexes) { _capacity = 0; return MEMALLOC; } structural_indexes[0] = 0; @@ -10065,6 +10070,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { + if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -13547,7 +13553,14 @@ namespace stage1 { * complete document, therefore the last json buffer location is the end of the * batch. */ -simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser) { +simdjson_inline bool ends_with_partial_scalar(dom_parser_implementation &parser, size_t len) { + const uint8_t f = parser.buf[parser.structural_indexes[parser.n_structural_indexes - 1]]; + const uint8_t e = parser.buf[len - 1]; + return f != '{' && f != '[' && f != '}' && f != ']' && f != ':' && f != ',' && f != '"' && + e != ' ' && e != '\t' && e != '\n' && e != '\r'; +} + +simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser, bool defer_last = false) { // Variant: do not count separately, just figure out depth if(parser.n_structural_indexes == 0) { return 0; } auto arr_cnt = 0; @@ -13581,7 +13594,7 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par } // Last document is complete, so the next document will appear after! if (!arr_cnt && !obj_cnt) { - return parser.n_structural_indexes; + return defer_last ? i : parser.n_structural_indexes; } // Last document is incomplete; mark the document at i + 1 as the next one return i; @@ -13603,7 +13616,7 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par } if (!arr_cnt && !obj_cnt) { // We have a complete document. - return parser.n_structural_indexes; + return defer_last ? 0 : parser.n_structural_indexes; } return 0; } @@ -13982,11 +13995,6 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa ((error != SUCCESS) && (error != UNCLOSED_STRING)) // when partial we tolerate UNCLOSED_STRING : (error != SUCCESS); // if partial is false, we must have SUCCESS const bool have_unclosed_string = (error == UNCLOSED_STRING); - if (simdjson_unlikely(should_we_exit)) { return error; } - - if (unescaped_chars_error) { - return UNESCAPED_CHARS; - } parser.n_structural_indexes = uint32_t(indexer.tail - parser.structural_indexes.get()); /*** * The On-Demand API requires special padding. @@ -14011,6 +14019,12 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa parser.structural_indexes[parser.n_structural_indexes + 1] = uint32_t(len); parser.structural_indexes[parser.n_structural_indexes + 2] = 0; parser.next_structural_index = 0; + + // Bail out only once the count and sentinels above are set: + // document_stream::truncated_bytes() reads them even on error. + if (simdjson_unlikely(should_we_exit)) { return error; } + if (unescaped_chars_error) { return UNESCAPED_CHARS; } + // a valid JSON file cannot have zero structural indexes - we should have found something if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; @@ -14027,7 +14041,7 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } } // We truncate the input to the end of the last complete document (or zero). - auto new_structural_indexes = find_next_document_index(parser); + auto new_structural_indexes = find_next_document_index(parser, !have_unclosed_string && ends_with_partial_scalar(parser, len)); if (new_structural_indexes == 0 && parser.n_structural_indexes > 0) { if(parser.structural_indexes[0] == 0) { // If the buffer is partial and we started at index 0 but the document is @@ -15234,6 +15248,9 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_ const uint8_t *p = value; if (*p == '-') p++; while (numberparsing::is_digit(*p)) p++; + // The digit run must be terminated by a structural or whitespace character; otherwise the + // token is malformed (e.g. "123456789123456789123x"). + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } size_t len = size_t(p - value); tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); @@ -16591,9 +16608,14 @@ inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parse // Leaving these here so they can be inlined if so desired inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } + if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } // Stage 1 index output - size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; + size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); + if(rounded_capacity + 9 < rounded_capacity) { + return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity + } + size_t max_structures = rounded_capacity + 9; + if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); if (!structural_indexes) { _capacity = 0; return MEMALLOC; } structural_indexes[0] = 0; @@ -16604,6 +16626,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { + if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -19945,7 +19968,14 @@ namespace stage1 { * complete document, therefore the last json buffer location is the end of the * batch. */ -simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser) { +simdjson_inline bool ends_with_partial_scalar(dom_parser_implementation &parser, size_t len) { + const uint8_t f = parser.buf[parser.structural_indexes[parser.n_structural_indexes - 1]]; + const uint8_t e = parser.buf[len - 1]; + return f != '{' && f != '[' && f != '}' && f != ']' && f != ':' && f != ',' && f != '"' && + e != ' ' && e != '\t' && e != '\n' && e != '\r'; +} + +simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser, bool defer_last = false) { // Variant: do not count separately, just figure out depth if(parser.n_structural_indexes == 0) { return 0; } auto arr_cnt = 0; @@ -19979,7 +20009,7 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par } // Last document is complete, so the next document will appear after! if (!arr_cnt && !obj_cnt) { - return parser.n_structural_indexes; + return defer_last ? i : parser.n_structural_indexes; } // Last document is incomplete; mark the document at i + 1 as the next one return i; @@ -20001,7 +20031,7 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par } if (!arr_cnt && !obj_cnt) { // We have a complete document. - return parser.n_structural_indexes; + return defer_last ? 0 : parser.n_structural_indexes; } return 0; } @@ -20380,11 +20410,6 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa ((error != SUCCESS) && (error != UNCLOSED_STRING)) // when partial we tolerate UNCLOSED_STRING : (error != SUCCESS); // if partial is false, we must have SUCCESS const bool have_unclosed_string = (error == UNCLOSED_STRING); - if (simdjson_unlikely(should_we_exit)) { return error; } - - if (unescaped_chars_error) { - return UNESCAPED_CHARS; - } parser.n_structural_indexes = uint32_t(indexer.tail - parser.structural_indexes.get()); /*** * The On-Demand API requires special padding. @@ -20409,6 +20434,12 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa parser.structural_indexes[parser.n_structural_indexes + 1] = uint32_t(len); parser.structural_indexes[parser.n_structural_indexes + 2] = 0; parser.next_structural_index = 0; + + // Bail out only once the count and sentinels above are set: + // document_stream::truncated_bytes() reads them even on error. + if (simdjson_unlikely(should_we_exit)) { return error; } + if (unescaped_chars_error) { return UNESCAPED_CHARS; } + // a valid JSON file cannot have zero structural indexes - we should have found something if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; @@ -20425,7 +20456,7 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } } // We truncate the input to the end of the last complete document (or zero). - auto new_structural_indexes = find_next_document_index(parser); + auto new_structural_indexes = find_next_document_index(parser, !have_unclosed_string && ends_with_partial_scalar(parser, len)); if (new_structural_indexes == 0 && parser.n_structural_indexes > 0) { if(parser.structural_indexes[0] == 0) { // If the buffer is partial and we started at index 0 but the document is @@ -21632,6 +21663,9 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_ const uint8_t *p = value; if (*p == '-') p++; while (numberparsing::is_digit(*p)) p++; + // The digit run must be terminated by a structural or whitespace character; otherwise the + // token is malformed (e.g. "123456789123456789123x"). + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } size_t len = size_t(p - value); tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); @@ -22985,9 +23019,14 @@ inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parse // Leaving these here so they can be inlined if so desired inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } + if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } // Stage 1 index output - size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; + size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); + if(rounded_capacity + 9 < rounded_capacity) { + return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity + } + size_t max_structures = rounded_capacity + 9; + if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); if (!structural_indexes) { _capacity = 0; return MEMALLOC; } structural_indexes[0] = 0; @@ -22998,6 +23037,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { + if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -26338,7 +26378,14 @@ namespace stage1 { * complete document, therefore the last json buffer location is the end of the * batch. */ -simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser) { +simdjson_inline bool ends_with_partial_scalar(dom_parser_implementation &parser, size_t len) { + const uint8_t f = parser.buf[parser.structural_indexes[parser.n_structural_indexes - 1]]; + const uint8_t e = parser.buf[len - 1]; + return f != '{' && f != '[' && f != '}' && f != ']' && f != ':' && f != ',' && f != '"' && + e != ' ' && e != '\t' && e != '\n' && e != '\r'; +} + +simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser, bool defer_last = false) { // Variant: do not count separately, just figure out depth if(parser.n_structural_indexes == 0) { return 0; } auto arr_cnt = 0; @@ -26372,7 +26419,7 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par } // Last document is complete, so the next document will appear after! if (!arr_cnt && !obj_cnt) { - return parser.n_structural_indexes; + return defer_last ? i : parser.n_structural_indexes; } // Last document is incomplete; mark the document at i + 1 as the next one return i; @@ -26394,7 +26441,7 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par } if (!arr_cnt && !obj_cnt) { // We have a complete document. - return parser.n_structural_indexes; + return defer_last ? 0 : parser.n_structural_indexes; } return 0; } @@ -26773,11 +26820,6 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa ((error != SUCCESS) && (error != UNCLOSED_STRING)) // when partial we tolerate UNCLOSED_STRING : (error != SUCCESS); // if partial is false, we must have SUCCESS const bool have_unclosed_string = (error == UNCLOSED_STRING); - if (simdjson_unlikely(should_we_exit)) { return error; } - - if (unescaped_chars_error) { - return UNESCAPED_CHARS; - } parser.n_structural_indexes = uint32_t(indexer.tail - parser.structural_indexes.get()); /*** * The On-Demand API requires special padding. @@ -26802,6 +26844,12 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa parser.structural_indexes[parser.n_structural_indexes + 1] = uint32_t(len); parser.structural_indexes[parser.n_structural_indexes + 2] = 0; parser.next_structural_index = 0; + + // Bail out only once the count and sentinels above are set: + // document_stream::truncated_bytes() reads them even on error. + if (simdjson_unlikely(should_we_exit)) { return error; } + if (unescaped_chars_error) { return UNESCAPED_CHARS; } + // a valid JSON file cannot have zero structural indexes - we should have found something if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; @@ -26818,7 +26866,7 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } } // We truncate the input to the end of the last complete document (or zero). - auto new_structural_indexes = find_next_document_index(parser); + auto new_structural_indexes = find_next_document_index(parser, !have_unclosed_string && ends_with_partial_scalar(parser, len)); if (new_structural_indexes == 0 && parser.n_structural_indexes > 0) { if(parser.structural_indexes[0] == 0) { // If the buffer is partial and we started at index 0 but the document is @@ -28025,6 +28073,9 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_ const uint8_t *p = value; if (*p == '-') p++; while (numberparsing::is_digit(*p)) p++; + // The digit run must be terminated by a structural or whitespace character; otherwise the + // token is malformed (e.g. "123456789123456789123x"). + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } size_t len = size_t(p - value); tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); @@ -29536,9 +29587,14 @@ inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parse // Leaving these here so they can be inlined if so desired inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } + if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } // Stage 1 index output - size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; + size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); + if(rounded_capacity + 9 < rounded_capacity) { + return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity + } + size_t max_structures = rounded_capacity + 9; + if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); if (!structural_indexes) { _capacity = 0; return MEMALLOC; } structural_indexes[0] = 0; @@ -29549,6 +29605,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { + if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -33002,7 +33059,14 @@ namespace stage1 { * complete document, therefore the last json buffer location is the end of the * batch. */ -simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser) { +simdjson_inline bool ends_with_partial_scalar(dom_parser_implementation &parser, size_t len) { + const uint8_t f = parser.buf[parser.structural_indexes[parser.n_structural_indexes - 1]]; + const uint8_t e = parser.buf[len - 1]; + return f != '{' && f != '[' && f != '}' && f != ']' && f != ':' && f != ',' && f != '"' && + e != ' ' && e != '\t' && e != '\n' && e != '\r'; +} + +simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser, bool defer_last = false) { // Variant: do not count separately, just figure out depth if(parser.n_structural_indexes == 0) { return 0; } auto arr_cnt = 0; @@ -33036,7 +33100,7 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par } // Last document is complete, so the next document will appear after! if (!arr_cnt && !obj_cnt) { - return parser.n_structural_indexes; + return defer_last ? i : parser.n_structural_indexes; } // Last document is incomplete; mark the document at i + 1 as the next one return i; @@ -33058,7 +33122,7 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par } if (!arr_cnt && !obj_cnt) { // We have a complete document. - return parser.n_structural_indexes; + return defer_last ? 0 : parser.n_structural_indexes; } return 0; } @@ -33437,11 +33501,6 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa ((error != SUCCESS) && (error != UNCLOSED_STRING)) // when partial we tolerate UNCLOSED_STRING : (error != SUCCESS); // if partial is false, we must have SUCCESS const bool have_unclosed_string = (error == UNCLOSED_STRING); - if (simdjson_unlikely(should_we_exit)) { return error; } - - if (unescaped_chars_error) { - return UNESCAPED_CHARS; - } parser.n_structural_indexes = uint32_t(indexer.tail - parser.structural_indexes.get()); /*** * The On-Demand API requires special padding. @@ -33466,6 +33525,12 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa parser.structural_indexes[parser.n_structural_indexes + 1] = uint32_t(len); parser.structural_indexes[parser.n_structural_indexes + 2] = 0; parser.next_structural_index = 0; + + // Bail out only once the count and sentinels above are set: + // document_stream::truncated_bytes() reads them even on error. + if (simdjson_unlikely(should_we_exit)) { return error; } + if (unescaped_chars_error) { return UNESCAPED_CHARS; } + // a valid JSON file cannot have zero structural indexes - we should have found something if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; @@ -33482,7 +33547,7 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } } // We truncate the input to the end of the last complete document (or zero). - auto new_structural_indexes = find_next_document_index(parser); + auto new_structural_indexes = find_next_document_index(parser, !have_unclosed_string && ends_with_partial_scalar(parser, len)); if (new_structural_indexes == 0 && parser.n_structural_indexes > 0) { if(parser.structural_indexes[0] == 0) { // If the buffer is partial and we started at index 0 but the document is @@ -34689,6 +34754,9 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_ const uint8_t *p = value; if (*p == '-') p++; while (numberparsing::is_digit(*p)) p++; + // The digit run must be terminated by a structural or whitespace character; otherwise the + // token is malformed (e.g. "123456789123456789123x"). + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } size_t len = size_t(p - value); tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); @@ -36447,9 +36515,14 @@ inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parse // Leaving these here so they can be inlined if so desired inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } + if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } // Stage 1 index output - size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; + size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); + if(rounded_capacity + 9 < rounded_capacity) { + return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity + } + size_t max_structures = rounded_capacity + 9; + if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); if (!structural_indexes) { _capacity = 0; return MEMALLOC; } structural_indexes[0] = 0; @@ -36460,6 +36533,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { + if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -40228,7 +40302,14 @@ namespace stage1 { * complete document, therefore the last json buffer location is the end of the * batch. */ -simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser) { +simdjson_inline bool ends_with_partial_scalar(dom_parser_implementation &parser, size_t len) { + const uint8_t f = parser.buf[parser.structural_indexes[parser.n_structural_indexes - 1]]; + const uint8_t e = parser.buf[len - 1]; + return f != '{' && f != '[' && f != '}' && f != ']' && f != ':' && f != ',' && f != '"' && + e != ' ' && e != '\t' && e != '\n' && e != '\r'; +} + +simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser, bool defer_last = false) { // Variant: do not count separately, just figure out depth if(parser.n_structural_indexes == 0) { return 0; } auto arr_cnt = 0; @@ -40262,7 +40343,7 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par } // Last document is complete, so the next document will appear after! if (!arr_cnt && !obj_cnt) { - return parser.n_structural_indexes; + return defer_last ? i : parser.n_structural_indexes; } // Last document is incomplete; mark the document at i + 1 as the next one return i; @@ -40284,7 +40365,7 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par } if (!arr_cnt && !obj_cnt) { // We have a complete document. - return parser.n_structural_indexes; + return defer_last ? 0 : parser.n_structural_indexes; } return 0; } @@ -40663,11 +40744,6 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa ((error != SUCCESS) && (error != UNCLOSED_STRING)) // when partial we tolerate UNCLOSED_STRING : (error != SUCCESS); // if partial is false, we must have SUCCESS const bool have_unclosed_string = (error == UNCLOSED_STRING); - if (simdjson_unlikely(should_we_exit)) { return error; } - - if (unescaped_chars_error) { - return UNESCAPED_CHARS; - } parser.n_structural_indexes = uint32_t(indexer.tail - parser.structural_indexes.get()); /*** * The On-Demand API requires special padding. @@ -40692,6 +40768,12 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa parser.structural_indexes[parser.n_structural_indexes + 1] = uint32_t(len); parser.structural_indexes[parser.n_structural_indexes + 2] = 0; parser.next_structural_index = 0; + + // Bail out only once the count and sentinels above are set: + // document_stream::truncated_bytes() reads them even on error. + if (simdjson_unlikely(should_we_exit)) { return error; } + if (unescaped_chars_error) { return UNESCAPED_CHARS; } + // a valid JSON file cannot have zero structural indexes - we should have found something if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; @@ -40708,7 +40790,7 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } } // We truncate the input to the end of the last complete document (or zero). - auto new_structural_indexes = find_next_document_index(parser); + auto new_structural_indexes = find_next_document_index(parser, !have_unclosed_string && ends_with_partial_scalar(parser, len)); if (new_structural_indexes == 0 && parser.n_structural_indexes > 0) { if(parser.structural_indexes[0] == 0) { // If the buffer is partial and we started at index 0 but the document is @@ -41915,6 +41997,9 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_ const uint8_t *p = value; if (*p == '-') p++; while (numberparsing::is_digit(*p)) p++; + // The digit run must be terminated by a structural or whitespace character; otherwise the + // token is malformed (e.g. "123456789123456789123x"). + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } size_t len = size_t(p - value); tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); @@ -43204,9 +43289,14 @@ inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parse // Leaving these here so they can be inlined if so desired inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } + if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } // Stage 1 index output - size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; + size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); + if(rounded_capacity + 9 < rounded_capacity) { + return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity + } + size_t max_structures = rounded_capacity + 9; + if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); if (!structural_indexes) { _capacity = 0; return MEMALLOC; } structural_indexes[0] = 0; @@ -43217,6 +43307,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { + if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -46485,7 +46576,14 @@ namespace stage1 { * complete document, therefore the last json buffer location is the end of the * batch. */ -simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser) { +simdjson_inline bool ends_with_partial_scalar(dom_parser_implementation &parser, size_t len) { + const uint8_t f = parser.buf[parser.structural_indexes[parser.n_structural_indexes - 1]]; + const uint8_t e = parser.buf[len - 1]; + return f != '{' && f != '[' && f != '}' && f != ']' && f != ':' && f != ',' && f != '"' && + e != ' ' && e != '\t' && e != '\n' && e != '\r'; +} + +simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser, bool defer_last = false) { // Variant: do not count separately, just figure out depth if(parser.n_structural_indexes == 0) { return 0; } auto arr_cnt = 0; @@ -46519,7 +46617,7 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par } // Last document is complete, so the next document will appear after! if (!arr_cnt && !obj_cnt) { - return parser.n_structural_indexes; + return defer_last ? i : parser.n_structural_indexes; } // Last document is incomplete; mark the document at i + 1 as the next one return i; @@ -46541,7 +46639,7 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par } if (!arr_cnt && !obj_cnt) { // We have a complete document. - return parser.n_structural_indexes; + return defer_last ? 0 : parser.n_structural_indexes; } return 0; } @@ -46920,11 +47018,6 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa ((error != SUCCESS) && (error != UNCLOSED_STRING)) // when partial we tolerate UNCLOSED_STRING : (error != SUCCESS); // if partial is false, we must have SUCCESS const bool have_unclosed_string = (error == UNCLOSED_STRING); - if (simdjson_unlikely(should_we_exit)) { return error; } - - if (unescaped_chars_error) { - return UNESCAPED_CHARS; - } parser.n_structural_indexes = uint32_t(indexer.tail - parser.structural_indexes.get()); /*** * The On-Demand API requires special padding. @@ -46949,6 +47042,12 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa parser.structural_indexes[parser.n_structural_indexes + 1] = uint32_t(len); parser.structural_indexes[parser.n_structural_indexes + 2] = 0; parser.next_structural_index = 0; + + // Bail out only once the count and sentinels above are set: + // document_stream::truncated_bytes() reads them even on error. + if (simdjson_unlikely(should_we_exit)) { return error; } + if (unescaped_chars_error) { return UNESCAPED_CHARS; } + // a valid JSON file cannot have zero structural indexes - we should have found something if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; @@ -46965,7 +47064,7 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } } // We truncate the input to the end of the last complete document (or zero). - auto new_structural_indexes = find_next_document_index(parser); + auto new_structural_indexes = find_next_document_index(parser, !have_unclosed_string && ends_with_partial_scalar(parser, len)); if (new_structural_indexes == 0 && parser.n_structural_indexes > 0) { if(parser.structural_indexes[0] == 0) { // If the buffer is partial and we started at index 0 but the document is @@ -48172,6 +48271,9 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_ const uint8_t *p = value; if (*p == '-') p++; while (numberparsing::is_digit(*p)) p++; + // The digit run must be terminated by a structural or whitespace character; otherwise the + // token is malformed (e.g. "123456789123456789123x"). + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } size_t len = size_t(p - value); tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); @@ -49398,9 +49500,14 @@ inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parse // Leaving these here so they can be inlined if so desired inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } + if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } // Stage 1 index output - size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; + size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); + if(rounded_capacity + 9 < rounded_capacity) { + return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity + } + size_t max_structures = rounded_capacity + 9; + if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); if (!structural_indexes) { _capacity = 0; return MEMALLOC; } structural_indexes[0] = 0; @@ -49411,6 +49518,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { + if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -52646,7 +52754,14 @@ namespace stage1 { * complete document, therefore the last json buffer location is the end of the * batch. */ -simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser) { +simdjson_inline bool ends_with_partial_scalar(dom_parser_implementation &parser, size_t len) { + const uint8_t f = parser.buf[parser.structural_indexes[parser.n_structural_indexes - 1]]; + const uint8_t e = parser.buf[len - 1]; + return f != '{' && f != '[' && f != '}' && f != ']' && f != ':' && f != ',' && f != '"' && + e != ' ' && e != '\t' && e != '\n' && e != '\r'; +} + +simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser, bool defer_last = false) { // Variant: do not count separately, just figure out depth if(parser.n_structural_indexes == 0) { return 0; } auto arr_cnt = 0; @@ -52680,7 +52795,7 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par } // Last document is complete, so the next document will appear after! if (!arr_cnt && !obj_cnt) { - return parser.n_structural_indexes; + return defer_last ? i : parser.n_structural_indexes; } // Last document is incomplete; mark the document at i + 1 as the next one return i; @@ -52702,7 +52817,7 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par } if (!arr_cnt && !obj_cnt) { // We have a complete document. - return parser.n_structural_indexes; + return defer_last ? 0 : parser.n_structural_indexes; } return 0; } @@ -53081,11 +53196,6 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa ((error != SUCCESS) && (error != UNCLOSED_STRING)) // when partial we tolerate UNCLOSED_STRING : (error != SUCCESS); // if partial is false, we must have SUCCESS const bool have_unclosed_string = (error == UNCLOSED_STRING); - if (simdjson_unlikely(should_we_exit)) { return error; } - - if (unescaped_chars_error) { - return UNESCAPED_CHARS; - } parser.n_structural_indexes = uint32_t(indexer.tail - parser.structural_indexes.get()); /*** * The On-Demand API requires special padding. @@ -53110,6 +53220,12 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa parser.structural_indexes[parser.n_structural_indexes + 1] = uint32_t(len); parser.structural_indexes[parser.n_structural_indexes + 2] = 0; parser.next_structural_index = 0; + + // Bail out only once the count and sentinels above are set: + // document_stream::truncated_bytes() reads them even on error. + if (simdjson_unlikely(should_we_exit)) { return error; } + if (unescaped_chars_error) { return UNESCAPED_CHARS; } + // a valid JSON file cannot have zero structural indexes - we should have found something if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; @@ -53126,7 +53242,7 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } } // We truncate the input to the end of the last complete document (or zero). - auto new_structural_indexes = find_next_document_index(parser); + auto new_structural_indexes = find_next_document_index(parser, !have_unclosed_string && ends_with_partial_scalar(parser, len)); if (new_structural_indexes == 0 && parser.n_structural_indexes > 0) { if(parser.structural_indexes[0] == 0) { // If the buffer is partial and we started at index 0 but the document is @@ -54333,6 +54449,9 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_ const uint8_t *p = value; if (*p == '-') p++; while (numberparsing::is_digit(*p)) p++; + // The digit run must be terminated by a structural or whitespace character; otherwise the + // token is malformed (e.g. "123456789123456789123x"). + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } size_t len = size_t(p - value); tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); @@ -55580,9 +55699,14 @@ inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parse // Leaving these here so they can be inlined if so desired inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } + if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } // Stage 1 index output - size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; + size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); + if(rounded_capacity + 9 < rounded_capacity) { + return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity + } + size_t max_structures = rounded_capacity + 9; + if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); if (!structural_indexes) { _capacity = 0; return MEMALLOC; } structural_indexes[0] = 0; @@ -55593,6 +55717,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { + if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -59226,7 +59351,14 @@ namespace stage1 { * complete document, therefore the last json buffer location is the end of the * batch. */ -simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser) { +simdjson_inline bool ends_with_partial_scalar(dom_parser_implementation &parser, size_t len) { + const uint8_t f = parser.buf[parser.structural_indexes[parser.n_structural_indexes - 1]]; + const uint8_t e = parser.buf[len - 1]; + return f != '{' && f != '[' && f != '}' && f != ']' && f != ':' && f != ',' && f != '"' && + e != ' ' && e != '\t' && e != '\n' && e != '\r'; +} + +simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser, bool defer_last = false) { // Variant: do not count separately, just figure out depth if(parser.n_structural_indexes == 0) { return 0; } auto arr_cnt = 0; @@ -59260,7 +59392,7 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par } // Last document is complete, so the next document will appear after! if (!arr_cnt && !obj_cnt) { - return parser.n_structural_indexes; + return defer_last ? i : parser.n_structural_indexes; } // Last document is incomplete; mark the document at i + 1 as the next one return i; @@ -59282,7 +59414,7 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par } if (!arr_cnt && !obj_cnt) { // We have a complete document. - return parser.n_structural_indexes; + return defer_last ? 0 : parser.n_structural_indexes; } return 0; } @@ -59661,11 +59793,6 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa ((error != SUCCESS) && (error != UNCLOSED_STRING)) // when partial we tolerate UNCLOSED_STRING : (error != SUCCESS); // if partial is false, we must have SUCCESS const bool have_unclosed_string = (error == UNCLOSED_STRING); - if (simdjson_unlikely(should_we_exit)) { return error; } - - if (unescaped_chars_error) { - return UNESCAPED_CHARS; - } parser.n_structural_indexes = uint32_t(indexer.tail - parser.structural_indexes.get()); /*** * The On-Demand API requires special padding. @@ -59690,6 +59817,12 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa parser.structural_indexes[parser.n_structural_indexes + 1] = uint32_t(len); parser.structural_indexes[parser.n_structural_indexes + 2] = 0; parser.next_structural_index = 0; + + // Bail out only once the count and sentinels above are set: + // document_stream::truncated_bytes() reads them even on error. + if (simdjson_unlikely(should_we_exit)) { return error; } + if (unescaped_chars_error) { return UNESCAPED_CHARS; } + // a valid JSON file cannot have zero structural indexes - we should have found something if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; @@ -59706,7 +59839,7 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } } // We truncate the input to the end of the last complete document (or zero). - auto new_structural_indexes = find_next_document_index(parser); + auto new_structural_indexes = find_next_document_index(parser, !have_unclosed_string && ends_with_partial_scalar(parser, len)); if (new_structural_indexes == 0 && parser.n_structural_indexes > 0) { if(parser.structural_indexes[0] == 0) { // If the buffer is partial and we started at index 0 but the document is @@ -60913,6 +61046,9 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_ const uint8_t *p = value; if (*p == '-') p++; while (numberparsing::is_digit(*p)) p++; + // The digit run must be terminated by a structural or whitespace character; otherwise the + // token is malformed (e.g. "123456789123456789123x"). + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } size_t len = size_t(p - value); tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); @@ -61739,9 +61875,14 @@ inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parse // Leaving these here so they can be inlined if so desired inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } + if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } // Stage 1 index output - size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; + size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); + if(rounded_capacity + 9 < rounded_capacity) { + return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity + } + size_t max_structures = rounded_capacity + 9; + if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); if (!structural_indexes) { _capacity = 0; return MEMALLOC; } structural_indexes[0] = 0; @@ -61752,6 +61893,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { + if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -63715,7 +63857,14 @@ namespace stage1 { * complete document, therefore the last json buffer location is the end of the * batch. */ -simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser) { +simdjson_inline bool ends_with_partial_scalar(dom_parser_implementation &parser, size_t len) { + const uint8_t f = parser.buf[parser.structural_indexes[parser.n_structural_indexes - 1]]; + const uint8_t e = parser.buf[len - 1]; + return f != '{' && f != '[' && f != '}' && f != ']' && f != ':' && f != ',' && f != '"' && + e != ' ' && e != '\t' && e != '\n' && e != '\r'; +} + +simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser, bool defer_last = false) { // Variant: do not count separately, just figure out depth if(parser.n_structural_indexes == 0) { return 0; } auto arr_cnt = 0; @@ -63749,7 +63898,7 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par } // Last document is complete, so the next document will appear after! if (!arr_cnt && !obj_cnt) { - return parser.n_structural_indexes; + return defer_last ? i : parser.n_structural_indexes; } // Last document is incomplete; mark the document at i + 1 as the next one return i; @@ -63771,7 +63920,7 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par } if (!arr_cnt && !obj_cnt) { // We have a complete document. - return parser.n_structural_indexes; + return defer_last ? 0 : parser.n_structural_indexes; } return 0; } @@ -64787,6 +64936,9 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_ const uint8_t *p = value; if (*p == '-') p++; while (numberparsing::is_digit(*p)) p++; + // The digit run must be terminated by a structural or whitespace character; otherwise the + // token is malformed (e.g. "123456789123456789123x"). + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } size_t len = size_t(p - value); tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); @@ -64976,8 +65128,8 @@ simdjson_inline void validate_utf8_character() { // 2-byte if ((buf[idx] & 0x20) == 0) { // missing continuation - if (simdjson_unlikely(idx+1 > len || !is_continuation(buf[idx+1]))) { - if (idx+1 > len && is_streaming(partial)) { idx = len; return; } + if (simdjson_unlikely(idx+1 >= len || !is_continuation(buf[idx+1]))) { + if (idx+1 >= len && is_streaming(partial)) { idx = len; return; } error = UTF8_ERROR; idx++; return; @@ -64991,8 +65143,8 @@ simdjson_inline void validate_utf8_character() { // 3-byte if ((buf[idx] & 0x10) == 0) { // missing continuation - if (simdjson_unlikely(idx+2 > len || !is_continuation(buf[idx+1]) || !is_continuation(buf[idx+2]))) { - if (idx+2 > len && is_streaming(partial)) { idx = len; return; } + if (simdjson_unlikely(idx+2 >= len || !is_continuation(buf[idx+1]) || !is_continuation(buf[idx+2]))) { + if (idx+2 >= len && is_streaming(partial)) { idx = len; return; } error = UTF8_ERROR; idx++; return; @@ -65007,8 +65159,8 @@ simdjson_inline void validate_utf8_character() { // 4-byte // missing continuation - if (simdjson_unlikely(idx+3 > len || !is_continuation(buf[idx+1]) || !is_continuation(buf[idx+2]) || !is_continuation(buf[idx+3]))) { - if (idx+2 > len && is_streaming(partial)) { idx = len; return; } + if (simdjson_unlikely(idx+3 >= len || !is_continuation(buf[idx+1]) || !is_continuation(buf[idx+2]) || !is_continuation(buf[idx+3]))) { + if (idx+3 >= len && is_streaming(partial)) { idx = len; return; } error = UTF8_ERROR; idx++; return; @@ -65130,9 +65282,15 @@ simdjson_warn_unused simdjson_inline error_code scan() { add_structural(); // Primitive or invalid character (invalid characters will be checked in stage 2) } else { - // Anything else, add the structural and go until we find the next one + // Anything else, add the structural and go until we find the next one. + // We also stop on '"' so that an unclosed string still reaches + // validate_string(); a quote swallowed by the run would hide it. A + // quote cannot occur inside a valid primitive. We deliberately do not + // stop on every ESC_ASCII character: that also covers a backslash and the + // control characters, and ending the run there makes the fallback + // disagree with the SIMD kernels. add_structural(); - while (idx+1 0) { if(parser.structural_indexes[0] == 0) { // If the buffer is partial and we started at index 0 but the document is diff --git a/src/simdjson.h b/src/simdjson.h index 9fb7ad31..9b1cf625 100644 --- a/src/simdjson.h +++ b/src/simdjson.h @@ -1,4 +1,4 @@ -/* auto-generated on 2026-04-03 15:26:12 -0400. version 4.6.1 Do not edit! */ +/* auto-generated on 2026-08-23 15:51:58 +0200. version 4.6.11 Do not edit! */ /* including simdjson.h: */ /* begin file simdjson.h */ #ifndef SIMDJSON_H @@ -260,7 +260,7 @@ using std::size_t; #endif #elif defined(__PPC64__) || defined(_M_PPC64) #define SIMDJSON_IS_PPC64 1 -#if defined(__ALTIVEC__) +#if defined(__ALTIVEC__) && defined(__POWER8_VECTOR__) #define SIMDJSON_IS_PPC64_VMX 1 #endif // defined(__ALTIVEC__) #else @@ -2538,7 +2538,7 @@ namespace std { #define SIMDJSON_SIMDJSON_VERSION_H /** The version of simdjson being used (major.minor.revision) */ -#define SIMDJSON_VERSION "4.6.1" +#define SIMDJSON_VERSION "4.6.11" namespace simdjson { enum { @@ -2553,7 +2553,7 @@ enum { /** * The revision (major.minor.REVISION) of simdjson being used. */ - SIMDJSON_VERSION_REVISION = 1 + SIMDJSON_VERSION_REVISION = 11 }; } // namespace simdjson @@ -4734,6 +4734,7 @@ inline char *allocate_padded_buffer(size_t length) noexcept { inline padded_string::padded_string() noexcept = default; inline padded_string::padded_string(size_t length) noexcept : viable_size(length), data_ptr(internal::allocate_padded_buffer(length)) { + if (data_ptr == nullptr) { viable_size = 0; } } inline padded_string::padded_string(const char *data, size_t length) noexcept : viable_size(length), data_ptr(internal::allocate_padded_buffer(length)) { @@ -6351,6 +6352,18 @@ class document_stream { * } * size_t truncated = stream.truncated_bytes(); * + * IMPORTANT: this value is only meaningful under the conditions below. It is + * computed from stage-1 bookkeeping, and outside these conditions it is not + * merely imprecise, it is arbitrary -- it can exceed size_in_bytes() or wrap + * around to a huge value. Check it only when both of the following hold: + * + * - you iterated all the way to the end of the stream; + * - no document reported an error. Iteration stops at the first failed + * document, which can leave the bookkeeping from a mid-stream batch. + * + * If you need to know about a truncated tail outside those conditions, track + * it yourself from the last successful document (see iterator::current_index() + * and iterator::source()). */ inline size_t truncated_bytes() const noexcept; /** @@ -8067,16 +8080,28 @@ inline std::pair get_next_key_and_json_path( } key = json_path.substr(key_start, i - key_start); - } else if ((i+1 < json_path.size()) && json_path[i] == '[' && (json_path[i+1] == '\'' || json_path[i+1] == '"')) { + } else if ((i + 1 < json_path.size()) && json_path[i] == '[' && + (json_path[i + 1] == '\'' || json_path[i + 1] == '"')) { + // Bracket-quoted key: ['key'] or ["key"]. + // Require a matching closing quote and a following ']'. If either is + // missing, return an empty key and the original path so callers can treat + // this as a parse failure (e.g. INVALID_JSON_POINTER) without advancing. + // Without this check, i += 2 can make i > size() and substr throws + // std::out_of_range, which aborts noexcept callers such as + // at_path_with_wildcard / for_each_at_path_with_wildcard. + const char quote = json_path[i + 1]; i += 2; - size_t key_start = i; - while (i < json_path.length() && json_path[i] != '\'' && json_path[i] != '"') { + const size_t key_start = i; + while (i < json_path.length() && json_path[i] != quote) { ++i; } - + if (i >= json_path.length() || // missing closing quote + i + 1 >= json_path.length() || // missing ] + json_path[i + 1] != ']') { + return {key, json_path}; + } key = json_path.substr(key_start, i - key_start); - - i += 2; + i += 2; // past quote and ] } else if ((i+2 < json_path.size()) && json_path[i] == '[' && json_path[i+1] == '*' && json_path[i+2] == ']') { // i.e [*].additional_keys or [*]["additional_keys"] key = "*"; i += 3; @@ -9933,6 +9958,9 @@ simdjson_inline size_t document_stream::iterator::current_index() const noexcept simdjson_inline std::string_view document_stream::iterator::source() const noexcept { const char* start = reinterpret_cast(stream->buf) + current_index(); + if (stream->error) { + return std::string_view(start, stream->len - current_index()); + } bool object_or_array = ((*start == '[') || (*start == '{')); if(object_or_array) { size_t next_doc_index = stream->batch_start + stream->parser->implementation->structural_indexes[stream->parser->implementation->next_structural_index - 1]; @@ -14981,9 +15009,14 @@ inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parse // Leaving these here so they can be inlined if so desired inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } + if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } // Stage 1 index output - size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; + size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); + if(rounded_capacity + 9 < rounded_capacity) { + return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity + } + size_t max_structures = rounded_capacity + 9; + if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); if (!structural_indexes) { _capacity = 0; return MEMALLOC; } structural_indexes[0] = 0; @@ -14994,6 +15027,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { + if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -17200,9 +17234,14 @@ inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parse // Leaving these here so they can be inlined if so desired inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } + if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } // Stage 1 index output - size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; + size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); + if(rounded_capacity + 9 < rounded_capacity) { + return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity + } + size_t max_structures = rounded_capacity + 9; + if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); if (!structural_indexes) { _capacity = 0; return MEMALLOC; } structural_indexes[0] = 0; @@ -17213,6 +17252,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { + if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -19906,9 +19946,14 @@ inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parse // Leaving these here so they can be inlined if so desired inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } + if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } // Stage 1 index output - size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; + size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); + if(rounded_capacity + 9 < rounded_capacity) { + return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity + } + size_t max_structures = rounded_capacity + 9; + if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); if (!structural_indexes) { _capacity = 0; return MEMALLOC; } structural_indexes[0] = 0; @@ -19919,6 +19964,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { + if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -22612,9 +22658,14 @@ inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parse // Leaving these here so they can be inlined if so desired inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } + if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } // Stage 1 index output - size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; + size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); + if(rounded_capacity + 9 < rounded_capacity) { + return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity + } + size_t max_structures = rounded_capacity + 9; + if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); if (!structural_indexes) { _capacity = 0; return MEMALLOC; } structural_indexes[0] = 0; @@ -22625,6 +22676,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { + if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -25433,9 +25485,14 @@ inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parse // Leaving these here so they can be inlined if so desired inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } + if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } // Stage 1 index output - size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; + size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); + if(rounded_capacity + 9 < rounded_capacity) { + return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity + } + size_t max_structures = rounded_capacity + 9; + if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); if (!structural_indexes) { _capacity = 0; return MEMALLOC; } structural_indexes[0] = 0; @@ -25446,6 +25503,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { + if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -28571,9 +28629,14 @@ inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parse // Leaving these here so they can be inlined if so desired inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } + if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } // Stage 1 index output - size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; + size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); + if(rounded_capacity + 9 < rounded_capacity) { + return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity + } + size_t max_structures = rounded_capacity + 9; + if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); if (!structural_indexes) { _capacity = 0; return MEMALLOC; } structural_indexes[0] = 0; @@ -28584,6 +28647,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { + if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -31209,9 +31273,14 @@ inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parse // Leaving these here so they can be inlined if so desired inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } + if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } // Stage 1 index output - size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; + size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); + if(rounded_capacity + 9 < rounded_capacity) { + return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity + } + size_t max_structures = rounded_capacity + 9; + if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); if (!structural_indexes) { _capacity = 0; return MEMALLOC; } structural_indexes[0] = 0; @@ -31222,6 +31291,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { + if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -33825,9 +33895,14 @@ inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parse // Leaving these here so they can be inlined if so desired inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } + if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } // Stage 1 index output - size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; + size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); + if(rounded_capacity + 9 < rounded_capacity) { + return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity + } + size_t max_structures = rounded_capacity + 9; + if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); if (!structural_indexes) { _capacity = 0; return MEMALLOC; } structural_indexes[0] = 0; @@ -33838,6 +33913,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { + if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -36458,9 +36534,14 @@ inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parse // Leaving these here so they can be inlined if so desired inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } + if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } // Stage 1 index output - size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; + size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); + if(rounded_capacity + 9 < rounded_capacity) { + return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity + } + size_t max_structures = rounded_capacity + 9; + if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); if (!structural_indexes) { _capacity = 0; return MEMALLOC; } structural_indexes[0] = 0; @@ -36471,6 +36552,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { + if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -39848,6 +39930,7 @@ simdjson_warn_unused simdjson_result extract_fractured_json( /* begin file simdjson/generic/builder/json_string_builder-inl.h for arm64 */ #include #include +#include #include #ifndef SIMDJSON_GENERIC_STRING_BUILDER_INL_H @@ -39886,7 +39969,7 @@ simdjson_warn_unused simdjson_result extract_fractured_json( #define SIMDJSON_EXPERIMENTAL_HAS_RVV 1 #endif #endif -#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) +#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) && defined(__POWER8_VECTOR__) #ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64 #define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1 #endif @@ -40603,6 +40686,11 @@ simdjson_inline void string_builder::append(number_type v) noexcept { simdjson_inline void string_builder::escape_and_append(std::string_view input) noexcept { // escaping might turn a control character into \x00xx so 6 characters. + // Guard against size_t overflow in the multiplication below. + if (input.size() > (std::numeric_limits::max)() / 6) { + set_valid(false); + return; + } if (capacity_check(6 * input.size())) { position += write_string_escaped(input, buffer.get() + position); } @@ -40611,6 +40699,11 @@ string_builder::escape_and_append(std::string_view input) noexcept { simdjson_inline void string_builder::escape_and_append_with_quotes(std::string_view input) noexcept { // escaping might turn a control character into \x00xx so 6 characters. + // Guard against size_t overflow in the arithmetic below. + if (input.size() > ((std::numeric_limits::max)() - 2) / 6) { + set_valid(false); + return; + } if (capacity_check(2 + 6 * input.size())) { buffer.get()[position++] = '"'; position += write_string_escaped(input, buffer.get() + position); @@ -41932,6 +42025,7 @@ simdjson_warn_unused simdjson_result extract_fractured_json( /* begin file simdjson/generic/builder/json_string_builder-inl.h for fallback */ #include #include +#include #include #ifndef SIMDJSON_GENERIC_STRING_BUILDER_INL_H @@ -41970,7 +42064,7 @@ simdjson_warn_unused simdjson_result extract_fractured_json( #define SIMDJSON_EXPERIMENTAL_HAS_RVV 1 #endif #endif -#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) +#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) && defined(__POWER8_VECTOR__) #ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64 #define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1 #endif @@ -42687,6 +42781,11 @@ simdjson_inline void string_builder::append(number_type v) noexcept { simdjson_inline void string_builder::escape_and_append(std::string_view input) noexcept { // escaping might turn a control character into \x00xx so 6 characters. + // Guard against size_t overflow in the multiplication below. + if (input.size() > (std::numeric_limits::max)() / 6) { + set_valid(false); + return; + } if (capacity_check(6 * input.size())) { position += write_string_escaped(input, buffer.get() + position); } @@ -42695,6 +42794,11 @@ string_builder::escape_and_append(std::string_view input) noexcept { simdjson_inline void string_builder::escape_and_append_with_quotes(std::string_view input) noexcept { // escaping might turn a control character into \x00xx so 6 characters. + // Guard against size_t overflow in the arithmetic below. + if (input.size() > ((std::numeric_limits::max)() - 2) / 6) { + set_valid(false); + return; + } if (capacity_check(2 + 6 * input.size())) { buffer.get()[position++] = '"'; position += write_string_escaped(input, buffer.get() + position); @@ -44503,6 +44607,7 @@ simdjson_warn_unused simdjson_result extract_fractured_json( /* begin file simdjson/generic/builder/json_string_builder-inl.h for haswell */ #include #include +#include #include #ifndef SIMDJSON_GENERIC_STRING_BUILDER_INL_H @@ -44541,7 +44646,7 @@ simdjson_warn_unused simdjson_result extract_fractured_json( #define SIMDJSON_EXPERIMENTAL_HAS_RVV 1 #endif #endif -#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) +#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) && defined(__POWER8_VECTOR__) #ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64 #define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1 #endif @@ -45258,6 +45363,11 @@ simdjson_inline void string_builder::append(number_type v) noexcept { simdjson_inline void string_builder::escape_and_append(std::string_view input) noexcept { // escaping might turn a control character into \x00xx so 6 characters. + // Guard against size_t overflow in the multiplication below. + if (input.size() > (std::numeric_limits::max)() / 6) { + set_valid(false); + return; + } if (capacity_check(6 * input.size())) { position += write_string_escaped(input, buffer.get() + position); } @@ -45266,6 +45376,11 @@ string_builder::escape_and_append(std::string_view input) noexcept { simdjson_inline void string_builder::escape_and_append_with_quotes(std::string_view input) noexcept { // escaping might turn a control character into \x00xx so 6 characters. + // Guard against size_t overflow in the arithmetic below. + if (input.size() > ((std::numeric_limits::max)() - 2) / 6) { + set_valid(false); + return; + } if (capacity_check(2 + 6 * input.size())) { buffer.get()[position++] = '"'; position += write_string_escaped(input, buffer.get() + position); @@ -47074,6 +47189,7 @@ simdjson_warn_unused simdjson_result extract_fractured_json( /* begin file simdjson/generic/builder/json_string_builder-inl.h for icelake */ #include #include +#include #include #ifndef SIMDJSON_GENERIC_STRING_BUILDER_INL_H @@ -47112,7 +47228,7 @@ simdjson_warn_unused simdjson_result extract_fractured_json( #define SIMDJSON_EXPERIMENTAL_HAS_RVV 1 #endif #endif -#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) +#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) && defined(__POWER8_VECTOR__) #ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64 #define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1 #endif @@ -47829,6 +47945,11 @@ simdjson_inline void string_builder::append(number_type v) noexcept { simdjson_inline void string_builder::escape_and_append(std::string_view input) noexcept { // escaping might turn a control character into \x00xx so 6 characters. + // Guard against size_t overflow in the multiplication below. + if (input.size() > (std::numeric_limits::max)() / 6) { + set_valid(false); + return; + } if (capacity_check(6 * input.size())) { position += write_string_escaped(input, buffer.get() + position); } @@ -47837,6 +47958,11 @@ string_builder::escape_and_append(std::string_view input) noexcept { simdjson_inline void string_builder::escape_and_append_with_quotes(std::string_view input) noexcept { // escaping might turn a control character into \x00xx so 6 characters. + // Guard against size_t overflow in the arithmetic below. + if (input.size() > ((std::numeric_limits::max)() - 2) / 6) { + set_valid(false); + return; + } if (capacity_check(2 + 6 * input.size())) { buffer.get()[position++] = '"'; position += write_string_escaped(input, buffer.get() + position); @@ -49760,6 +49886,7 @@ simdjson_warn_unused simdjson_result extract_fractured_json( /* begin file simdjson/generic/builder/json_string_builder-inl.h for ppc64 */ #include #include +#include #include #ifndef SIMDJSON_GENERIC_STRING_BUILDER_INL_H @@ -49798,7 +49925,7 @@ simdjson_warn_unused simdjson_result extract_fractured_json( #define SIMDJSON_EXPERIMENTAL_HAS_RVV 1 #endif #endif -#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) +#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) && defined(__POWER8_VECTOR__) #ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64 #define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1 #endif @@ -50515,6 +50642,11 @@ simdjson_inline void string_builder::append(number_type v) noexcept { simdjson_inline void string_builder::escape_and_append(std::string_view input) noexcept { // escaping might turn a control character into \x00xx so 6 characters. + // Guard against size_t overflow in the multiplication below. + if (input.size() > (std::numeric_limits::max)() / 6) { + set_valid(false); + return; + } if (capacity_check(6 * input.size())) { position += write_string_escaped(input, buffer.get() + position); } @@ -50523,6 +50655,11 @@ string_builder::escape_and_append(std::string_view input) noexcept { simdjson_inline void string_builder::escape_and_append_with_quotes(std::string_view input) noexcept { // escaping might turn a control character into \x00xx so 6 characters. + // Guard against size_t overflow in the arithmetic below. + if (input.size() > ((std::numeric_limits::max)() - 2) / 6) { + set_valid(false); + return; + } if (capacity_check(2 + 6 * input.size())) { buffer.get()[position++] = '"'; position += write_string_escaped(input, buffer.get() + position); @@ -52763,6 +52900,7 @@ simdjson_warn_unused simdjson_result extract_fractured_json( /* begin file simdjson/generic/builder/json_string_builder-inl.h for westmere */ #include #include +#include #include #ifndef SIMDJSON_GENERIC_STRING_BUILDER_INL_H @@ -52801,7 +52939,7 @@ simdjson_warn_unused simdjson_result extract_fractured_json( #define SIMDJSON_EXPERIMENTAL_HAS_RVV 1 #endif #endif -#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) +#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) && defined(__POWER8_VECTOR__) #ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64 #define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1 #endif @@ -53518,6 +53656,11 @@ simdjson_inline void string_builder::append(number_type v) noexcept { simdjson_inline void string_builder::escape_and_append(std::string_view input) noexcept { // escaping might turn a control character into \x00xx so 6 characters. + // Guard against size_t overflow in the multiplication below. + if (input.size() > (std::numeric_limits::max)() / 6) { + set_valid(false); + return; + } if (capacity_check(6 * input.size())) { position += write_string_escaped(input, buffer.get() + position); } @@ -53526,6 +53669,11 @@ string_builder::escape_and_append(std::string_view input) noexcept { simdjson_inline void string_builder::escape_and_append_with_quotes(std::string_view input) noexcept { // escaping might turn a control character into \x00xx so 6 characters. + // Guard against size_t overflow in the arithmetic below. + if (input.size() > ((std::numeric_limits::max)() - 2) / 6) { + set_valid(false); + return; + } if (capacity_check(2 + 6 * input.size())) { buffer.get()[position++] = '"'; position += write_string_escaped(input, buffer.get() + position); @@ -55240,6 +55388,7 @@ simdjson_warn_unused simdjson_result extract_fractured_json( /* begin file simdjson/generic/builder/json_string_builder-inl.h for lsx */ #include #include +#include #include #ifndef SIMDJSON_GENERIC_STRING_BUILDER_INL_H @@ -55278,7 +55427,7 @@ simdjson_warn_unused simdjson_result extract_fractured_json( #define SIMDJSON_EXPERIMENTAL_HAS_RVV 1 #endif #endif -#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) +#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) && defined(__POWER8_VECTOR__) #ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64 #define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1 #endif @@ -55995,6 +56144,11 @@ simdjson_inline void string_builder::append(number_type v) noexcept { simdjson_inline void string_builder::escape_and_append(std::string_view input) noexcept { // escaping might turn a control character into \x00xx so 6 characters. + // Guard against size_t overflow in the multiplication below. + if (input.size() > (std::numeric_limits::max)() / 6) { + set_valid(false); + return; + } if (capacity_check(6 * input.size())) { position += write_string_escaped(input, buffer.get() + position); } @@ -56003,6 +56157,11 @@ string_builder::escape_and_append(std::string_view input) noexcept { simdjson_inline void string_builder::escape_and_append_with_quotes(std::string_view input) noexcept { // escaping might turn a control character into \x00xx so 6 characters. + // Guard against size_t overflow in the arithmetic below. + if (input.size() > ((std::numeric_limits::max)() - 2) / 6) { + set_valid(false); + return; + } if (capacity_check(2 + 6 * input.size())) { buffer.get()[position++] = '"'; position += write_string_escaped(input, buffer.get() + position); @@ -57740,6 +57899,7 @@ simdjson_warn_unused simdjson_result extract_fractured_json( /* begin file simdjson/generic/builder/json_string_builder-inl.h for lasx */ #include #include +#include #include #ifndef SIMDJSON_GENERIC_STRING_BUILDER_INL_H @@ -57778,7 +57938,7 @@ simdjson_warn_unused simdjson_result extract_fractured_json( #define SIMDJSON_EXPERIMENTAL_HAS_RVV 1 #endif #endif -#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) +#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) && defined(__POWER8_VECTOR__) #ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64 #define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1 #endif @@ -58495,6 +58655,11 @@ simdjson_inline void string_builder::append(number_type v) noexcept { simdjson_inline void string_builder::escape_and_append(std::string_view input) noexcept { // escaping might turn a control character into \x00xx so 6 characters. + // Guard against size_t overflow in the multiplication below. + if (input.size() > (std::numeric_limits::max)() / 6) { + set_valid(false); + return; + } if (capacity_check(6 * input.size())) { position += write_string_escaped(input, buffer.get() + position); } @@ -58503,6 +58668,11 @@ string_builder::escape_and_append(std::string_view input) noexcept { simdjson_inline void string_builder::escape_and_append_with_quotes(std::string_view input) noexcept { // escaping might turn a control character into \x00xx so 6 characters. + // Guard against size_t overflow in the arithmetic below. + if (input.size() > ((std::numeric_limits::max)() - 2) / 6) { + set_valid(false); + return; + } if (capacity_check(2 + 6 * input.size())) { buffer.get()[position++] = '"'; position += write_string_escaped(input, buffer.get() + position); @@ -60244,6 +60414,7 @@ simdjson_warn_unused simdjson_result extract_fractured_json( /* begin file simdjson/generic/builder/json_string_builder-inl.h for rvv_vls */ #include #include +#include #include #ifndef SIMDJSON_GENERIC_STRING_BUILDER_INL_H @@ -60282,7 +60453,7 @@ simdjson_warn_unused simdjson_result extract_fractured_json( #define SIMDJSON_EXPERIMENTAL_HAS_RVV 1 #endif #endif -#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) +#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) && defined(__POWER8_VECTOR__) #ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64 #define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1 #endif @@ -60999,6 +61170,11 @@ simdjson_inline void string_builder::append(number_type v) noexcept { simdjson_inline void string_builder::escape_and_append(std::string_view input) noexcept { // escaping might turn a control character into \x00xx so 6 characters. + // Guard against size_t overflow in the multiplication below. + if (input.size() > (std::numeric_limits::max)() / 6) { + set_valid(false); + return; + } if (capacity_check(6 * input.size())) { position += write_string_escaped(input, buffer.get() + position); } @@ -61007,6 +61183,11 @@ string_builder::escape_and_append(std::string_view input) noexcept { simdjson_inline void string_builder::escape_and_append_with_quotes(std::string_view input) noexcept { // escaping might turn a control character into \x00xx so 6 characters. + // Guard against size_t overflow in the arithmetic below. + if (input.size() > ((std::numeric_limits::max)() - 2) / 6) { + set_valid(false); + return; + } if (capacity_check(2 + 6 * input.size())) { buffer.get()[position++] = '"'; position += write_string_escaped(input, buffer.get() + position); @@ -63566,13 +63747,21 @@ class value { simdjson_inline simdjson_result at_path(std::string_view at_path) noexcept; /** - * Get all values matching the given JSONPath expression with wildcard support. + * Call the provided callback for each value matching the given JSONPath + * expression with wildcard support. * Supports wildcard character (*) for arrays or ".*" for objects. * * @param json_path JSONPath expression with wildcards - * @return Vector of values matching the wildcard pattern + * @param callback Function called for each matching value + * @return error_code indicating success or failure */ - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; protected: /** @@ -63765,9 +63954,23 @@ struct simdjson_result : public arm64::implementation_si simdjson_inline simdjson_result current_depth() const noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; }; +// Forward-declare explicit specializations so MSVC /permissive- sees them before +// any template instantiation that would resolve element.get(val) to the primary. +template<> simdjson_inline error_code +simdjson_result::get( + arm64::ondemand::value &out) noexcept; +template<> simdjson_inline simdjson_result +simdjson_result::get() noexcept; + } // namespace simdjson #endif // SIMDJSON_GENERIC_ONDEMAND_VALUE_H @@ -65106,8 +65309,6 @@ class parser { static simdjson_inline bool release_parser(); private: - friend bool release_parser(); - friend ondemand::parser& get_parser(); /** Get the thread-local parser instance, allocates it if needed */ static simdjson_inline simdjson_warn_unused std::unique_ptr& get_parser_instance(); /** Get the thread-local parser instance, it might be null */ @@ -65270,13 +65471,21 @@ class array { inline simdjson_result at_path(std::string_view json_path) noexcept; /** - * Get all values matching the given JSONPath expression with wildcard support. + * Call the provided callback for each value matching the given JSONPath + * expression with wildcard support. * Supports wildcard patterns like "[*]" to match all array elements. * * @param json_path JSONPath expression with wildcards - * @return Vector of values matching the wildcard pattern + * @param callback Function called for each matching value + * @return error_code indicating success or failure */ - inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; /** * Consumes the array and returns a string_view instance corresponding to the @@ -65400,7 +65609,13 @@ struct simdjson_result : public arm64::implementation_si simdjson_inline simdjson_result at(size_t index) noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; simdjson_inline simdjson_result raw_json() noexcept; #if SIMDJSON_SUPPORTS_CONCEPTS // TODO: move this code into object-inl.h @@ -66300,21 +66515,24 @@ class document { simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; /** - * Get all values matching the given JSONPath expression with wildcard support. + * Call the provided callback for each value matching the given JSONPath + * expression with wildcard support. * * Supports wildcard patterns like "$.array[*]" or "$.object.*" to match multiple elements. * - * This method materializes all matching values into a vector. * The document will be consumed after this call. * * @param json_path JSONPath expression with wildcards - * @return Vector of values matching the wildcard pattern, or: - * - INVALID_JSON_POINTER if the JSONPath cannot be parsed - * - NO_SUCH_FIELD if a field does not exist - * - INDEX_OUT_OF_BOUNDS if an array index is out of bounds - * - INCORRECT_TYPE if path traversal encounters wrong type + * @param callback Function called for each matching value + * @return error_code indicating success or failure */ - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; /** * Consumes the document and returns a string_view instance corresponding to the @@ -66535,7 +66753,13 @@ class document_reference { simdjson_inline simdjson_result raw_json_token() noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; private: document *doc{nullptr}; @@ -66621,7 +66845,13 @@ struct simdjson_result : public arm64::implementation simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; #if SIMDJSON_STATIC_REFLECTION template requires(std::is_class_v && (sizeof...(FieldNames) > 0)) @@ -66706,7 +66936,13 @@ struct simdjson_result : public arm64::impl simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; #if SIMDJSON_STATIC_REFLECTION template requires(std::is_class_v && (sizeof...(FieldNames) > 0)) @@ -66839,6 +67075,18 @@ class document_stream { * } * size_t truncated = stream.truncated_bytes(); * + * IMPORTANT: this value is only meaningful under the conditions below. It is + * computed from stage-1 bookkeeping, and outside these conditions it is not + * merely imprecise, it is arbitrary -- it can exceed size_in_bytes() or wrap + * around to a huge value. Check it only when both of the following hold: + * + * - you iterated all the way to the end of the stream; + * - no document reported an error. Iteration stops at the first failed + * document, which can leave the bookkeeping from a mid-stream batch. + * + * If you need to know about a truncated tail outside those conditions, track + * it yourself from the last successful document (see iterator::current_index() + * and iterator::source()). */ inline size_t truncated_bytes() const noexcept; @@ -67359,13 +67607,21 @@ class object { inline simdjson_result at_path(std::string_view json_path) noexcept; /** - * Get all values matching the given JSONPath expression with wildcard support. + * Call the provided callback for each value matching the given JSONPath + * expression with wildcard support. * Supports wildcard patterns like ".*" to match all object fields. * * @param json_path JSONPath expression with wildcards - * @return Vector of values matching the wildcard pattern + * @param callback Function called for each matching value + * @return error_code indicating success or failure */ - inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; /** * Reset the iterator so that we are pointing back at the @@ -67515,7 +67771,13 @@ struct simdjson_result : public arm64::implementation_s simdjson_inline simdjson_result operator[](std::string_view key) && noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; inline simdjson_result reset() noexcept; inline simdjson_result is_empty() noexcept; inline simdjson_result count_fields() & noexcept; @@ -68529,46 +68791,40 @@ inline simdjson_result array::at_path(std::string_view json_path) noexcep return at_pointer(json_pointer); } -inline simdjson_result> array::at_path_with_wildcard(std::string_view json_path) noexcept { - std::vector result; - +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +inline error_code array::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { auto result_pair = get_next_key_and_json_path(json_path); std::string_view key = result_pair.first; std::string_view remaining_path = result_pair.second; + // Empty key means the path segment could not be parsed (including malformed + // bracket-quoted keys). Without this check, an empty key is treated as index + // 0 and remaining_path may be unchanged, causing infinite recursion. + if (key.empty()) { + return INVALID_JSON_POINTER; + } // Wildcard case - if(key=="*"){ - for(auto element: *this){ - - if(element.error()){ - return element.error(); - } - - if(remaining_path.empty()){ - // Use value_unsafe() because we've already checked for errors above. - // The 'element' is a simdjson_result wrapper, and we need to extract - // the underlying value. value_unsafe() is safe here because error() returned false. - result.push_back(std::move(element).value_unsafe()); - - }else{ - auto nested_result = element.at_path_with_wildcard(remaining_path); - - if(nested_result.error()){ - return nested_result.error(); - } - // Same logic as above. - std::vector nested_matches = std::move(nested_result).value_unsafe(); - - result.insert(result.end(), - std::make_move_iterator(nested_matches.begin()), - std::make_move_iterator(nested_matches.end())); + if (key=="*"){ + for(auto element: *this) { + value val; + SIMDJSON_TRY(element.get(val)); + if (remaining_path.empty()) { + callback(val); + } else { + error_code err = element.for_each_at_path_with_wildcard(remaining_path, callback); + if(err) { return err; } } } - return result; - }else{ + return SUCCESS; + } else { // Specific index case in which we access the element at the given index - size_t idx=0; + size_t idx = 0; - for(char c:key){ + for (char c : key) { if(c < '0' || c > '9'){ return INVALID_JSON_POINTER; } @@ -68576,16 +68832,13 @@ inline simdjson_result> array::at_path_with_wildcard(std::str } auto element = at(idx); - - if(element.error()){ - return element.error(); - } - - if(remaining_path.empty()){ - result.push_back(std::move(element).value_unsafe()); - return result; - }else{ - return element.at_path_with_wildcard(remaining_path); + value val; + SIMDJSON_TRY(element.get(val)); + if (remaining_path.empty()){ + callback(val); + return SUCCESS; + } else { + return element.for_each_at_path_with_wildcard(remaining_path, callback); } } } @@ -68648,9 +68901,15 @@ simdjson_inline simdjson_result simdjson_result> simdjson_result::at_path_with_wildcard(std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code simdjson_result::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); + return first.for_each_at_path_with_wildcard(json_path, std::forward(callback)); } simdjson_inline simdjson_result simdjson_result::raw_json() noexcept { if (error()) { return error(); } @@ -69071,14 +69330,21 @@ simdjson_inline simdjson_result value::at_path(std::string_view json_path } } -inline simdjson_result> value::at_path_with_wildcard(std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +inline error_code value::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { + if (size_t(iter.depth()) >= iter.json_iter().parser->max_depth()) { return DEPTH_ERROR; } json_type t; SIMDJSON_TRY(type().get(t)); switch (t) { case json_type::array: - return (*this).get_array().at_path_with_wildcard(json_path); + return (*this).get_array().for_each_at_path_with_wildcard(json_path, std::forward(callback)); case json_type::object: - return (*this).get_object().at_path_with_wildcard(json_path); + return (*this).get_object().for_each_at_path_with_wildcard(json_path, std::forward(callback)); default: return INVALID_JSON_POINTER; } @@ -69342,12 +69608,18 @@ simdjson_inline simdjson_result simdjson_result> simdjson_result::at_path_with_wildcard( - std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +inline error_code simdjson_result::for_each_at_path_with_wildcard( + std::string_view json_path, Func&& callback) noexcept { if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); + return first.for_each_at_path_with_wildcard(json_path, std::forward(callback)); } } // namespace simdjson @@ -69723,8 +69995,14 @@ simdjson_inline simdjson_result document::at_path(std::string_view json_p } } -simdjson_inline simdjson_result> document::at_path_with_wildcard(std::string_view json_path) noexcept { - rewind(); // Rewind the document each time at_path_with_wildcard is called +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code document::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { + rewind(); // Rewind the document each time for_each_at_path_with_wildcard is called if (json_path.empty()) { return INVALID_JSON_POINTER; } @@ -69732,9 +70010,9 @@ simdjson_inline simdjson_result> document::at_path_with_wildc SIMDJSON_TRY(type().get(t)); switch (t) { case json_type::array: - return (*this).get_array().at_path_with_wildcard(json_path); + return (*this).get_array().for_each_at_path_with_wildcard(json_path, std::forward(callback)); case json_type::object: - return (*this).get_object().at_path_with_wildcard(json_path); + return (*this).get_object().for_each_at_path_with_wildcard(json_path, std::forward(callback)); default: return INVALID_JSON_POINTER; } @@ -70071,9 +70349,15 @@ simdjson_inline simdjson_result simdjson_result> simdjson_result::at_path_with_wildcard(std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code simdjson_result::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); + return first.for_each_at_path_with_wildcard(json_path, std::forward(callback)); } #if SIMDJSON_STATIC_REFLECTION @@ -70184,7 +70468,13 @@ simdjson_inline simdjson_result document_reference::get_number() noexcep simdjson_inline simdjson_result document_reference::raw_json_token() noexcept { return doc->raw_json_token(); } simdjson_inline simdjson_result document_reference::at_pointer(std::string_view json_pointer) noexcept { return doc->at_pointer(json_pointer); } simdjson_inline simdjson_result document_reference::at_path(std::string_view json_path) noexcept { return doc->at_path(json_path); } -simdjson_inline simdjson_result> document_reference::at_path_with_wildcard(std::string_view json_path) noexcept { return doc->at_path_with_wildcard(json_path); } +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code document_reference::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { return doc->for_each_at_path_with_wildcard(json_path, std::forward(callback)); } simdjson_inline simdjson_result document_reference::raw_json() noexcept { return doc->raw_json();} simdjson_inline document_reference::operator document&() const noexcept { return *doc; } #if SIMDJSON_SUPPORTS_CONCEPTS && SIMDJSON_STATIC_REFLECTION @@ -70449,11 +70739,17 @@ simdjson_inline simdjson_result simdjson_result> simdjson_result::at_path_with_wildcard(std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code simdjson_result::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); + return first.for_each_at_path_with_wildcard(json_path, std::forward(callback)); } #if SIMDJSON_STATIC_REFLECTION template @@ -70827,7 +71123,7 @@ simdjson_inline std::string_view document_stream::iterator::source() const noexc // TODO: We could remove trailing whitespaces // This returns a string spanning from start of value to the beginning of the next document (excluded) { - auto next_index = stream->parser->implementation->structural_indexes[++cur_struct_index]; + auto next_index = stream->batch_start + stream->parser->implementation->structural_indexes[++cur_struct_index]; // normally the length would be next_index - current_index() - 1, except for the last document size_t svlen = next_index - current_index(); const char *start = reinterpret_cast(stream->buf) + current_index(); @@ -72031,46 +72327,44 @@ inline simdjson_result object::at_path(std::string_view json_path) noexce return at_pointer(json_pointer); } -inline simdjson_result> object::at_path_with_wildcard(std::string_view json_path) noexcept { - std::vector result; - +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +inline error_code object::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { auto result_pair = get_next_key_and_json_path(json_path); std::string_view key = result_pair.first; std::string_view remaining_path = result_pair.second; + // Empty key means the path segment could not be parsed (including malformed + // bracket-quoted keys). Match DOM at_path_with_wildcard and avoid treating + // "no progress" as a recursive lookup of "". + if (key.empty()) { + return INVALID_JSON_POINTER; + } // Handle when its the case for wildcard if (key == "*") { // Loop through each field in the object for (auto field : *this) { value val; SIMDJSON_TRY(field.value().get(val)); - if (remaining_path.empty()) { - result.push_back(std::move(val)); + callback(val); } else { - auto nested_result = val.at_path_with_wildcard(remaining_path); - - if (nested_result.error()) { - return nested_result.error(); - } - // Extract and append all nested matches to our result - std::vector nested_vec; - SIMDJSON_TRY(std::move(nested_result).get(nested_vec)); - - result.insert(result.end(), - std::make_move_iterator(nested_vec.begin()), - std::make_move_iterator(nested_vec.end())); + SIMDJSON_TRY(val.for_each_at_path_with_wildcard(remaining_path, callback)); } } - return result; + return SUCCESS; } else { value val; SIMDJSON_TRY(find_field(key).get(val)); if (remaining_path.empty()) { - result.push_back(std::move(val)); - return result; + callback(val); + return SUCCESS; } else { - return val.at_path_with_wildcard(remaining_path); + return val.for_each_at_path_with_wildcard(remaining_path, callback); } } } @@ -72201,9 +72495,15 @@ simdjson_inline simdjson_result simdjson_result> simdjson_result::at_path_with_wildcard(std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code simdjson_result::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); + return first.for_each_at_path_with_wildcard(json_path, std::forward(callback)); } inline simdjson_result simdjson_result::reset() noexcept { @@ -72408,7 +72708,7 @@ simdjson_inline parser::parser(size_t max_capacity) noexcept simdjson_warn_unused simdjson_inline error_code parser::allocate(size_t new_capacity, size_t new_max_depth) noexcept { if (new_capacity > max_capacity()) { return CAPACITY; } - if (string_buf && new_capacity == capacity() && new_max_depth == max_depth()) { return SUCCESS; } + if (string_buf && new_capacity <= capacity() && new_max_depth <= max_depth()) { return SUCCESS; } // string_capacity copied from document::allocate _capacity = 0; @@ -72584,7 +72884,7 @@ simdjson_inline simdjson_warn_unused ondemand::parser& parser::get_parser() { return *parser::get_parser_instance(); } -simdjson_inline bool release_parser() { +simdjson_inline bool parser::release_parser() { auto &parser_instance = parser::get_threadlocal_parser_if_exists(); if (parser_instance) { parser_instance.reset(); @@ -72701,13 +73001,9 @@ simdjson_inline bool raw_json_string::unsafe_is_equal(std::string_view target) c if(target.size() <= SIMDJSON_PADDING) { return (raw()[target.size()] == '"') && !memcmp(raw(), target.data(), target.size()); } - const char * r{raw()}; - size_t pos{0}; - for(;pos < target.size();pos++) { - if(r[pos] != target[pos]) { return false; } - } - if(r[pos] != '"') { return false; } - return true; + // Past SIMDJSON_PADDING we can no longer rely on the padding to keep the + // comparison in bounds, so we must stop at the quote terminating the key. + return is_equal(target); } simdjson_inline bool raw_json_string::is_equal(std::string_view target) const noexcept { @@ -76842,13 +77138,21 @@ class value { simdjson_inline simdjson_result at_path(std::string_view at_path) noexcept; /** - * Get all values matching the given JSONPath expression with wildcard support. + * Call the provided callback for each value matching the given JSONPath + * expression with wildcard support. * Supports wildcard character (*) for arrays or ".*" for objects. * * @param json_path JSONPath expression with wildcards - * @return Vector of values matching the wildcard pattern + * @param callback Function called for each matching value + * @return error_code indicating success or failure */ - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; protected: /** @@ -77041,9 +77345,23 @@ struct simdjson_result : public fallback::implementat simdjson_inline simdjson_result current_depth() const noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; }; +// Forward-declare explicit specializations so MSVC /permissive- sees them before +// any template instantiation that would resolve element.get(val) to the primary. +template<> simdjson_inline error_code +simdjson_result::get( + fallback::ondemand::value &out) noexcept; +template<> simdjson_inline simdjson_result +simdjson_result::get() noexcept; + } // namespace simdjson #endif // SIMDJSON_GENERIC_ONDEMAND_VALUE_H @@ -78382,8 +78700,6 @@ class parser { static simdjson_inline bool release_parser(); private: - friend bool release_parser(); - friend ondemand::parser& get_parser(); /** Get the thread-local parser instance, allocates it if needed */ static simdjson_inline simdjson_warn_unused std::unique_ptr& get_parser_instance(); /** Get the thread-local parser instance, it might be null */ @@ -78546,13 +78862,21 @@ class array { inline simdjson_result at_path(std::string_view json_path) noexcept; /** - * Get all values matching the given JSONPath expression with wildcard support. + * Call the provided callback for each value matching the given JSONPath + * expression with wildcard support. * Supports wildcard patterns like "[*]" to match all array elements. * * @param json_path JSONPath expression with wildcards - * @return Vector of values matching the wildcard pattern + * @param callback Function called for each matching value + * @return error_code indicating success or failure */ - inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; /** * Consumes the array and returns a string_view instance corresponding to the @@ -78676,7 +79000,13 @@ struct simdjson_result : public fallback::implementat simdjson_inline simdjson_result at(size_t index) noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; simdjson_inline simdjson_result raw_json() noexcept; #if SIMDJSON_SUPPORTS_CONCEPTS // TODO: move this code into object-inl.h @@ -79576,21 +79906,24 @@ class document { simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; /** - * Get all values matching the given JSONPath expression with wildcard support. + * Call the provided callback for each value matching the given JSONPath + * expression with wildcard support. * * Supports wildcard patterns like "$.array[*]" or "$.object.*" to match multiple elements. * - * This method materializes all matching values into a vector. * The document will be consumed after this call. * * @param json_path JSONPath expression with wildcards - * @return Vector of values matching the wildcard pattern, or: - * - INVALID_JSON_POINTER if the JSONPath cannot be parsed - * - NO_SUCH_FIELD if a field does not exist - * - INDEX_OUT_OF_BOUNDS if an array index is out of bounds - * - INCORRECT_TYPE if path traversal encounters wrong type + * @param callback Function called for each matching value + * @return error_code indicating success or failure */ - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; /** * Consumes the document and returns a string_view instance corresponding to the @@ -79811,7 +80144,13 @@ class document_reference { simdjson_inline simdjson_result raw_json_token() noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; private: document *doc{nullptr}; @@ -79897,7 +80236,13 @@ struct simdjson_result : public fallback::implemen simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; #if SIMDJSON_STATIC_REFLECTION template requires(std::is_class_v && (sizeof...(FieldNames) > 0)) @@ -79982,7 +80327,13 @@ struct simdjson_result : public fallback simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; #if SIMDJSON_STATIC_REFLECTION template requires(std::is_class_v && (sizeof...(FieldNames) > 0)) @@ -80115,6 +80466,18 @@ class document_stream { * } * size_t truncated = stream.truncated_bytes(); * + * IMPORTANT: this value is only meaningful under the conditions below. It is + * computed from stage-1 bookkeeping, and outside these conditions it is not + * merely imprecise, it is arbitrary -- it can exceed size_in_bytes() or wrap + * around to a huge value. Check it only when both of the following hold: + * + * - you iterated all the way to the end of the stream; + * - no document reported an error. Iteration stops at the first failed + * document, which can leave the bookkeeping from a mid-stream batch. + * + * If you need to know about a truncated tail outside those conditions, track + * it yourself from the last successful document (see iterator::current_index() + * and iterator::source()). */ inline size_t truncated_bytes() const noexcept; @@ -80635,13 +80998,21 @@ class object { inline simdjson_result at_path(std::string_view json_path) noexcept; /** - * Get all values matching the given JSONPath expression with wildcard support. + * Call the provided callback for each value matching the given JSONPath + * expression with wildcard support. * Supports wildcard patterns like ".*" to match all object fields. * * @param json_path JSONPath expression with wildcards - * @return Vector of values matching the wildcard pattern + * @param callback Function called for each matching value + * @return error_code indicating success or failure */ - inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; /** * Reset the iterator so that we are pointing back at the @@ -80791,7 +81162,13 @@ struct simdjson_result : public fallback::implementa simdjson_inline simdjson_result operator[](std::string_view key) && noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; inline simdjson_result reset() noexcept; inline simdjson_result is_empty() noexcept; inline simdjson_result count_fields() & noexcept; @@ -81805,46 +82182,40 @@ inline simdjson_result array::at_path(std::string_view json_path) noexcep return at_pointer(json_pointer); } -inline simdjson_result> array::at_path_with_wildcard(std::string_view json_path) noexcept { - std::vector result; - +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +inline error_code array::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { auto result_pair = get_next_key_and_json_path(json_path); std::string_view key = result_pair.first; std::string_view remaining_path = result_pair.second; + // Empty key means the path segment could not be parsed (including malformed + // bracket-quoted keys). Without this check, an empty key is treated as index + // 0 and remaining_path may be unchanged, causing infinite recursion. + if (key.empty()) { + return INVALID_JSON_POINTER; + } // Wildcard case - if(key=="*"){ - for(auto element: *this){ - - if(element.error()){ - return element.error(); - } - - if(remaining_path.empty()){ - // Use value_unsafe() because we've already checked for errors above. - // The 'element' is a simdjson_result wrapper, and we need to extract - // the underlying value. value_unsafe() is safe here because error() returned false. - result.push_back(std::move(element).value_unsafe()); - - }else{ - auto nested_result = element.at_path_with_wildcard(remaining_path); - - if(nested_result.error()){ - return nested_result.error(); - } - // Same logic as above. - std::vector nested_matches = std::move(nested_result).value_unsafe(); - - result.insert(result.end(), - std::make_move_iterator(nested_matches.begin()), - std::make_move_iterator(nested_matches.end())); + if (key=="*"){ + for(auto element: *this) { + value val; + SIMDJSON_TRY(element.get(val)); + if (remaining_path.empty()) { + callback(val); + } else { + error_code err = element.for_each_at_path_with_wildcard(remaining_path, callback); + if(err) { return err; } } } - return result; - }else{ + return SUCCESS; + } else { // Specific index case in which we access the element at the given index - size_t idx=0; + size_t idx = 0; - for(char c:key){ + for (char c : key) { if(c < '0' || c > '9'){ return INVALID_JSON_POINTER; } @@ -81852,16 +82223,13 @@ inline simdjson_result> array::at_path_with_wildcard(std::str } auto element = at(idx); - - if(element.error()){ - return element.error(); - } - - if(remaining_path.empty()){ - result.push_back(std::move(element).value_unsafe()); - return result; - }else{ - return element.at_path_with_wildcard(remaining_path); + value val; + SIMDJSON_TRY(element.get(val)); + if (remaining_path.empty()){ + callback(val); + return SUCCESS; + } else { + return element.for_each_at_path_with_wildcard(remaining_path, callback); } } } @@ -81924,9 +82292,15 @@ simdjson_inline simdjson_result simdjson_result> simdjson_result::at_path_with_wildcard(std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code simdjson_result::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); + return first.for_each_at_path_with_wildcard(json_path, std::forward(callback)); } simdjson_inline simdjson_result simdjson_result::raw_json() noexcept { if (error()) { return error(); } @@ -82347,14 +82721,21 @@ simdjson_inline simdjson_result value::at_path(std::string_view json_path } } -inline simdjson_result> value::at_path_with_wildcard(std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +inline error_code value::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { + if (size_t(iter.depth()) >= iter.json_iter().parser->max_depth()) { return DEPTH_ERROR; } json_type t; SIMDJSON_TRY(type().get(t)); switch (t) { case json_type::array: - return (*this).get_array().at_path_with_wildcard(json_path); + return (*this).get_array().for_each_at_path_with_wildcard(json_path, std::forward(callback)); case json_type::object: - return (*this).get_object().at_path_with_wildcard(json_path); + return (*this).get_object().for_each_at_path_with_wildcard(json_path, std::forward(callback)); default: return INVALID_JSON_POINTER; } @@ -82618,12 +82999,18 @@ simdjson_inline simdjson_result simdjson_result> simdjson_result::at_path_with_wildcard( - std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +inline error_code simdjson_result::for_each_at_path_with_wildcard( + std::string_view json_path, Func&& callback) noexcept { if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); + return first.for_each_at_path_with_wildcard(json_path, std::forward(callback)); } } // namespace simdjson @@ -82999,8 +83386,14 @@ simdjson_inline simdjson_result document::at_path(std::string_view json_p } } -simdjson_inline simdjson_result> document::at_path_with_wildcard(std::string_view json_path) noexcept { - rewind(); // Rewind the document each time at_path_with_wildcard is called +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code document::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { + rewind(); // Rewind the document each time for_each_at_path_with_wildcard is called if (json_path.empty()) { return INVALID_JSON_POINTER; } @@ -83008,9 +83401,9 @@ simdjson_inline simdjson_result> document::at_path_with_wildc SIMDJSON_TRY(type().get(t)); switch (t) { case json_type::array: - return (*this).get_array().at_path_with_wildcard(json_path); + return (*this).get_array().for_each_at_path_with_wildcard(json_path, std::forward(callback)); case json_type::object: - return (*this).get_object().at_path_with_wildcard(json_path); + return (*this).get_object().for_each_at_path_with_wildcard(json_path, std::forward(callback)); default: return INVALID_JSON_POINTER; } @@ -83347,9 +83740,15 @@ simdjson_inline simdjson_result simdjson_result> simdjson_result::at_path_with_wildcard(std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code simdjson_result::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); + return first.for_each_at_path_with_wildcard(json_path, std::forward(callback)); } #if SIMDJSON_STATIC_REFLECTION @@ -83460,7 +83859,13 @@ simdjson_inline simdjson_result document_reference::get_number() noexcep simdjson_inline simdjson_result document_reference::raw_json_token() noexcept { return doc->raw_json_token(); } simdjson_inline simdjson_result document_reference::at_pointer(std::string_view json_pointer) noexcept { return doc->at_pointer(json_pointer); } simdjson_inline simdjson_result document_reference::at_path(std::string_view json_path) noexcept { return doc->at_path(json_path); } -simdjson_inline simdjson_result> document_reference::at_path_with_wildcard(std::string_view json_path) noexcept { return doc->at_path_with_wildcard(json_path); } +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code document_reference::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { return doc->for_each_at_path_with_wildcard(json_path, std::forward(callback)); } simdjson_inline simdjson_result document_reference::raw_json() noexcept { return doc->raw_json();} simdjson_inline document_reference::operator document&() const noexcept { return *doc; } #if SIMDJSON_SUPPORTS_CONCEPTS && SIMDJSON_STATIC_REFLECTION @@ -83725,11 +84130,17 @@ simdjson_inline simdjson_result simdjson_result> simdjson_result::at_path_with_wildcard(std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code simdjson_result::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); + return first.for_each_at_path_with_wildcard(json_path, std::forward(callback)); } #if SIMDJSON_STATIC_REFLECTION template @@ -84103,7 +84514,7 @@ simdjson_inline std::string_view document_stream::iterator::source() const noexc // TODO: We could remove trailing whitespaces // This returns a string spanning from start of value to the beginning of the next document (excluded) { - auto next_index = stream->parser->implementation->structural_indexes[++cur_struct_index]; + auto next_index = stream->batch_start + stream->parser->implementation->structural_indexes[++cur_struct_index]; // normally the length would be next_index - current_index() - 1, except for the last document size_t svlen = next_index - current_index(); const char *start = reinterpret_cast(stream->buf) + current_index(); @@ -85307,46 +85718,44 @@ inline simdjson_result object::at_path(std::string_view json_path) noexce return at_pointer(json_pointer); } -inline simdjson_result> object::at_path_with_wildcard(std::string_view json_path) noexcept { - std::vector result; - +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +inline error_code object::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { auto result_pair = get_next_key_and_json_path(json_path); std::string_view key = result_pair.first; std::string_view remaining_path = result_pair.second; + // Empty key means the path segment could not be parsed (including malformed + // bracket-quoted keys). Match DOM at_path_with_wildcard and avoid treating + // "no progress" as a recursive lookup of "". + if (key.empty()) { + return INVALID_JSON_POINTER; + } // Handle when its the case for wildcard if (key == "*") { // Loop through each field in the object for (auto field : *this) { value val; SIMDJSON_TRY(field.value().get(val)); - if (remaining_path.empty()) { - result.push_back(std::move(val)); + callback(val); } else { - auto nested_result = val.at_path_with_wildcard(remaining_path); - - if (nested_result.error()) { - return nested_result.error(); - } - // Extract and append all nested matches to our result - std::vector nested_vec; - SIMDJSON_TRY(std::move(nested_result).get(nested_vec)); - - result.insert(result.end(), - std::make_move_iterator(nested_vec.begin()), - std::make_move_iterator(nested_vec.end())); + SIMDJSON_TRY(val.for_each_at_path_with_wildcard(remaining_path, callback)); } } - return result; + return SUCCESS; } else { value val; SIMDJSON_TRY(find_field(key).get(val)); if (remaining_path.empty()) { - result.push_back(std::move(val)); - return result; + callback(val); + return SUCCESS; } else { - return val.at_path_with_wildcard(remaining_path); + return val.for_each_at_path_with_wildcard(remaining_path, callback); } } } @@ -85477,9 +85886,15 @@ simdjson_inline simdjson_result simdjson_result> simdjson_result::at_path_with_wildcard(std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code simdjson_result::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); + return first.for_each_at_path_with_wildcard(json_path, std::forward(callback)); } inline simdjson_result simdjson_result::reset() noexcept { @@ -85684,7 +86099,7 @@ simdjson_inline parser::parser(size_t max_capacity) noexcept simdjson_warn_unused simdjson_inline error_code parser::allocate(size_t new_capacity, size_t new_max_depth) noexcept { if (new_capacity > max_capacity()) { return CAPACITY; } - if (string_buf && new_capacity == capacity() && new_max_depth == max_depth()) { return SUCCESS; } + if (string_buf && new_capacity <= capacity() && new_max_depth <= max_depth()) { return SUCCESS; } // string_capacity copied from document::allocate _capacity = 0; @@ -85860,7 +86275,7 @@ simdjson_inline simdjson_warn_unused ondemand::parser& parser::get_parser() { return *parser::get_parser_instance(); } -simdjson_inline bool release_parser() { +simdjson_inline bool parser::release_parser() { auto &parser_instance = parser::get_threadlocal_parser_if_exists(); if (parser_instance) { parser_instance.reset(); @@ -85977,13 +86392,9 @@ simdjson_inline bool raw_json_string::unsafe_is_equal(std::string_view target) c if(target.size() <= SIMDJSON_PADDING) { return (raw()[target.size()] == '"') && !memcmp(raw(), target.data(), target.size()); } - const char * r{raw()}; - size_t pos{0}; - for(;pos < target.size();pos++) { - if(r[pos] != target[pos]) { return false; } - } - if(r[pos] != '"') { return false; } - return true; + // Past SIMDJSON_PADDING we can no longer rely on the padding to keep the + // comparison in bounds, so we must stop at the quote terminating the key. + return is_equal(target); } simdjson_inline bool raw_json_string::is_equal(std::string_view target) const noexcept { @@ -90605,13 +91016,21 @@ class value { simdjson_inline simdjson_result at_path(std::string_view at_path) noexcept; /** - * Get all values matching the given JSONPath expression with wildcard support. + * Call the provided callback for each value matching the given JSONPath + * expression with wildcard support. * Supports wildcard character (*) for arrays or ".*" for objects. * * @param json_path JSONPath expression with wildcards - * @return Vector of values matching the wildcard pattern + * @param callback Function called for each matching value + * @return error_code indicating success or failure */ - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; protected: /** @@ -90804,9 +91223,23 @@ struct simdjson_result : public haswell::implementatio simdjson_inline simdjson_result current_depth() const noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; }; +// Forward-declare explicit specializations so MSVC /permissive- sees them before +// any template instantiation that would resolve element.get(val) to the primary. +template<> simdjson_inline error_code +simdjson_result::get( + haswell::ondemand::value &out) noexcept; +template<> simdjson_inline simdjson_result +simdjson_result::get() noexcept; + } // namespace simdjson #endif // SIMDJSON_GENERIC_ONDEMAND_VALUE_H @@ -92145,8 +92578,6 @@ class parser { static simdjson_inline bool release_parser(); private: - friend bool release_parser(); - friend ondemand::parser& get_parser(); /** Get the thread-local parser instance, allocates it if needed */ static simdjson_inline simdjson_warn_unused std::unique_ptr& get_parser_instance(); /** Get the thread-local parser instance, it might be null */ @@ -92309,13 +92740,21 @@ class array { inline simdjson_result at_path(std::string_view json_path) noexcept; /** - * Get all values matching the given JSONPath expression with wildcard support. + * Call the provided callback for each value matching the given JSONPath + * expression with wildcard support. * Supports wildcard patterns like "[*]" to match all array elements. * * @param json_path JSONPath expression with wildcards - * @return Vector of values matching the wildcard pattern + * @param callback Function called for each matching value + * @return error_code indicating success or failure */ - inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; /** * Consumes the array and returns a string_view instance corresponding to the @@ -92439,7 +92878,13 @@ struct simdjson_result : public haswell::implementatio simdjson_inline simdjson_result at(size_t index) noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; simdjson_inline simdjson_result raw_json() noexcept; #if SIMDJSON_SUPPORTS_CONCEPTS // TODO: move this code into object-inl.h @@ -93339,21 +93784,24 @@ class document { simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; /** - * Get all values matching the given JSONPath expression with wildcard support. + * Call the provided callback for each value matching the given JSONPath + * expression with wildcard support. * * Supports wildcard patterns like "$.array[*]" or "$.object.*" to match multiple elements. * - * This method materializes all matching values into a vector. * The document will be consumed after this call. * * @param json_path JSONPath expression with wildcards - * @return Vector of values matching the wildcard pattern, or: - * - INVALID_JSON_POINTER if the JSONPath cannot be parsed - * - NO_SUCH_FIELD if a field does not exist - * - INDEX_OUT_OF_BOUNDS if an array index is out of bounds - * - INCORRECT_TYPE if path traversal encounters wrong type + * @param callback Function called for each matching value + * @return error_code indicating success or failure */ - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; /** * Consumes the document and returns a string_view instance corresponding to the @@ -93574,7 +94022,13 @@ class document_reference { simdjson_inline simdjson_result raw_json_token() noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; private: document *doc{nullptr}; @@ -93660,7 +94114,13 @@ struct simdjson_result : public haswell::implementa simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; #if SIMDJSON_STATIC_REFLECTION template requires(std::is_class_v && (sizeof...(FieldNames) > 0)) @@ -93745,7 +94205,13 @@ struct simdjson_result : public haswell:: simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; #if SIMDJSON_STATIC_REFLECTION template requires(std::is_class_v && (sizeof...(FieldNames) > 0)) @@ -93878,6 +94344,18 @@ class document_stream { * } * size_t truncated = stream.truncated_bytes(); * + * IMPORTANT: this value is only meaningful under the conditions below. It is + * computed from stage-1 bookkeeping, and outside these conditions it is not + * merely imprecise, it is arbitrary -- it can exceed size_in_bytes() or wrap + * around to a huge value. Check it only when both of the following hold: + * + * - you iterated all the way to the end of the stream; + * - no document reported an error. Iteration stops at the first failed + * document, which can leave the bookkeeping from a mid-stream batch. + * + * If you need to know about a truncated tail outside those conditions, track + * it yourself from the last successful document (see iterator::current_index() + * and iterator::source()). */ inline size_t truncated_bytes() const noexcept; @@ -94398,13 +94876,21 @@ class object { inline simdjson_result at_path(std::string_view json_path) noexcept; /** - * Get all values matching the given JSONPath expression with wildcard support. + * Call the provided callback for each value matching the given JSONPath + * expression with wildcard support. * Supports wildcard patterns like ".*" to match all object fields. * * @param json_path JSONPath expression with wildcards - * @return Vector of values matching the wildcard pattern + * @param callback Function called for each matching value + * @return error_code indicating success or failure */ - inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; /** * Reset the iterator so that we are pointing back at the @@ -94554,7 +95040,13 @@ struct simdjson_result : public haswell::implementati simdjson_inline simdjson_result operator[](std::string_view key) && noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; inline simdjson_result reset() noexcept; inline simdjson_result is_empty() noexcept; inline simdjson_result count_fields() & noexcept; @@ -95568,46 +96060,40 @@ inline simdjson_result array::at_path(std::string_view json_path) noexcep return at_pointer(json_pointer); } -inline simdjson_result> array::at_path_with_wildcard(std::string_view json_path) noexcept { - std::vector result; - +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +inline error_code array::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { auto result_pair = get_next_key_and_json_path(json_path); std::string_view key = result_pair.first; std::string_view remaining_path = result_pair.second; + // Empty key means the path segment could not be parsed (including malformed + // bracket-quoted keys). Without this check, an empty key is treated as index + // 0 and remaining_path may be unchanged, causing infinite recursion. + if (key.empty()) { + return INVALID_JSON_POINTER; + } // Wildcard case - if(key=="*"){ - for(auto element: *this){ - - if(element.error()){ - return element.error(); - } - - if(remaining_path.empty()){ - // Use value_unsafe() because we've already checked for errors above. - // The 'element' is a simdjson_result wrapper, and we need to extract - // the underlying value. value_unsafe() is safe here because error() returned false. - result.push_back(std::move(element).value_unsafe()); - - }else{ - auto nested_result = element.at_path_with_wildcard(remaining_path); - - if(nested_result.error()){ - return nested_result.error(); - } - // Same logic as above. - std::vector nested_matches = std::move(nested_result).value_unsafe(); - - result.insert(result.end(), - std::make_move_iterator(nested_matches.begin()), - std::make_move_iterator(nested_matches.end())); + if (key=="*"){ + for(auto element: *this) { + value val; + SIMDJSON_TRY(element.get(val)); + if (remaining_path.empty()) { + callback(val); + } else { + error_code err = element.for_each_at_path_with_wildcard(remaining_path, callback); + if(err) { return err; } } } - return result; - }else{ + return SUCCESS; + } else { // Specific index case in which we access the element at the given index - size_t idx=0; + size_t idx = 0; - for(char c:key){ + for (char c : key) { if(c < '0' || c > '9'){ return INVALID_JSON_POINTER; } @@ -95615,16 +96101,13 @@ inline simdjson_result> array::at_path_with_wildcard(std::str } auto element = at(idx); - - if(element.error()){ - return element.error(); - } - - if(remaining_path.empty()){ - result.push_back(std::move(element).value_unsafe()); - return result; - }else{ - return element.at_path_with_wildcard(remaining_path); + value val; + SIMDJSON_TRY(element.get(val)); + if (remaining_path.empty()){ + callback(val); + return SUCCESS; + } else { + return element.for_each_at_path_with_wildcard(remaining_path, callback); } } } @@ -95687,9 +96170,15 @@ simdjson_inline simdjson_result simdjson_result> simdjson_result::at_path_with_wildcard(std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code simdjson_result::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); + return first.for_each_at_path_with_wildcard(json_path, std::forward(callback)); } simdjson_inline simdjson_result simdjson_result::raw_json() noexcept { if (error()) { return error(); } @@ -96110,14 +96599,21 @@ simdjson_inline simdjson_result value::at_path(std::string_view json_path } } -inline simdjson_result> value::at_path_with_wildcard(std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +inline error_code value::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { + if (size_t(iter.depth()) >= iter.json_iter().parser->max_depth()) { return DEPTH_ERROR; } json_type t; SIMDJSON_TRY(type().get(t)); switch (t) { case json_type::array: - return (*this).get_array().at_path_with_wildcard(json_path); + return (*this).get_array().for_each_at_path_with_wildcard(json_path, std::forward(callback)); case json_type::object: - return (*this).get_object().at_path_with_wildcard(json_path); + return (*this).get_object().for_each_at_path_with_wildcard(json_path, std::forward(callback)); default: return INVALID_JSON_POINTER; } @@ -96381,12 +96877,18 @@ simdjson_inline simdjson_result simdjson_result> simdjson_result::at_path_with_wildcard( - std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +inline error_code simdjson_result::for_each_at_path_with_wildcard( + std::string_view json_path, Func&& callback) noexcept { if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); + return first.for_each_at_path_with_wildcard(json_path, std::forward(callback)); } } // namespace simdjson @@ -96762,8 +97264,14 @@ simdjson_inline simdjson_result document::at_path(std::string_view json_p } } -simdjson_inline simdjson_result> document::at_path_with_wildcard(std::string_view json_path) noexcept { - rewind(); // Rewind the document each time at_path_with_wildcard is called +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code document::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { + rewind(); // Rewind the document each time for_each_at_path_with_wildcard is called if (json_path.empty()) { return INVALID_JSON_POINTER; } @@ -96771,9 +97279,9 @@ simdjson_inline simdjson_result> document::at_path_with_wildc SIMDJSON_TRY(type().get(t)); switch (t) { case json_type::array: - return (*this).get_array().at_path_with_wildcard(json_path); + return (*this).get_array().for_each_at_path_with_wildcard(json_path, std::forward(callback)); case json_type::object: - return (*this).get_object().at_path_with_wildcard(json_path); + return (*this).get_object().for_each_at_path_with_wildcard(json_path, std::forward(callback)); default: return INVALID_JSON_POINTER; } @@ -97110,9 +97618,15 @@ simdjson_inline simdjson_result simdjson_result> simdjson_result::at_path_with_wildcard(std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code simdjson_result::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); + return first.for_each_at_path_with_wildcard(json_path, std::forward(callback)); } #if SIMDJSON_STATIC_REFLECTION @@ -97223,7 +97737,13 @@ simdjson_inline simdjson_result document_reference::get_number() noexcep simdjson_inline simdjson_result document_reference::raw_json_token() noexcept { return doc->raw_json_token(); } simdjson_inline simdjson_result document_reference::at_pointer(std::string_view json_pointer) noexcept { return doc->at_pointer(json_pointer); } simdjson_inline simdjson_result document_reference::at_path(std::string_view json_path) noexcept { return doc->at_path(json_path); } -simdjson_inline simdjson_result> document_reference::at_path_with_wildcard(std::string_view json_path) noexcept { return doc->at_path_with_wildcard(json_path); } +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code document_reference::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { return doc->for_each_at_path_with_wildcard(json_path, std::forward(callback)); } simdjson_inline simdjson_result document_reference::raw_json() noexcept { return doc->raw_json();} simdjson_inline document_reference::operator document&() const noexcept { return *doc; } #if SIMDJSON_SUPPORTS_CONCEPTS && SIMDJSON_STATIC_REFLECTION @@ -97488,11 +98008,17 @@ simdjson_inline simdjson_result simdjson_result> simdjson_result::at_path_with_wildcard(std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code simdjson_result::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); + return first.for_each_at_path_with_wildcard(json_path, std::forward(callback)); } #if SIMDJSON_STATIC_REFLECTION template @@ -97866,7 +98392,7 @@ simdjson_inline std::string_view document_stream::iterator::source() const noexc // TODO: We could remove trailing whitespaces // This returns a string spanning from start of value to the beginning of the next document (excluded) { - auto next_index = stream->parser->implementation->structural_indexes[++cur_struct_index]; + auto next_index = stream->batch_start + stream->parser->implementation->structural_indexes[++cur_struct_index]; // normally the length would be next_index - current_index() - 1, except for the last document size_t svlen = next_index - current_index(); const char *start = reinterpret_cast(stream->buf) + current_index(); @@ -99070,46 +99596,44 @@ inline simdjson_result object::at_path(std::string_view json_path) noexce return at_pointer(json_pointer); } -inline simdjson_result> object::at_path_with_wildcard(std::string_view json_path) noexcept { - std::vector result; - +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +inline error_code object::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { auto result_pair = get_next_key_and_json_path(json_path); std::string_view key = result_pair.first; std::string_view remaining_path = result_pair.second; + // Empty key means the path segment could not be parsed (including malformed + // bracket-quoted keys). Match DOM at_path_with_wildcard and avoid treating + // "no progress" as a recursive lookup of "". + if (key.empty()) { + return INVALID_JSON_POINTER; + } // Handle when its the case for wildcard if (key == "*") { // Loop through each field in the object for (auto field : *this) { value val; SIMDJSON_TRY(field.value().get(val)); - if (remaining_path.empty()) { - result.push_back(std::move(val)); + callback(val); } else { - auto nested_result = val.at_path_with_wildcard(remaining_path); - - if (nested_result.error()) { - return nested_result.error(); - } - // Extract and append all nested matches to our result - std::vector nested_vec; - SIMDJSON_TRY(std::move(nested_result).get(nested_vec)); - - result.insert(result.end(), - std::make_move_iterator(nested_vec.begin()), - std::make_move_iterator(nested_vec.end())); + SIMDJSON_TRY(val.for_each_at_path_with_wildcard(remaining_path, callback)); } } - return result; + return SUCCESS; } else { value val; SIMDJSON_TRY(find_field(key).get(val)); if (remaining_path.empty()) { - result.push_back(std::move(val)); - return result; + callback(val); + return SUCCESS; } else { - return val.at_path_with_wildcard(remaining_path); + return val.for_each_at_path_with_wildcard(remaining_path, callback); } } } @@ -99240,9 +99764,15 @@ simdjson_inline simdjson_result simdjson_result> simdjson_result::at_path_with_wildcard(std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code simdjson_result::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); + return first.for_each_at_path_with_wildcard(json_path, std::forward(callback)); } inline simdjson_result simdjson_result::reset() noexcept { @@ -99447,7 +99977,7 @@ simdjson_inline parser::parser(size_t max_capacity) noexcept simdjson_warn_unused simdjson_inline error_code parser::allocate(size_t new_capacity, size_t new_max_depth) noexcept { if (new_capacity > max_capacity()) { return CAPACITY; } - if (string_buf && new_capacity == capacity() && new_max_depth == max_depth()) { return SUCCESS; } + if (string_buf && new_capacity <= capacity() && new_max_depth <= max_depth()) { return SUCCESS; } // string_capacity copied from document::allocate _capacity = 0; @@ -99623,7 +100153,7 @@ simdjson_inline simdjson_warn_unused ondemand::parser& parser::get_parser() { return *parser::get_parser_instance(); } -simdjson_inline bool release_parser() { +simdjson_inline bool parser::release_parser() { auto &parser_instance = parser::get_threadlocal_parser_if_exists(); if (parser_instance) { parser_instance.reset(); @@ -99740,13 +100270,9 @@ simdjson_inline bool raw_json_string::unsafe_is_equal(std::string_view target) c if(target.size() <= SIMDJSON_PADDING) { return (raw()[target.size()] == '"') && !memcmp(raw(), target.data(), target.size()); } - const char * r{raw()}; - size_t pos{0}; - for(;pos < target.size();pos++) { - if(r[pos] != target[pos]) { return false; } - } - if(r[pos] != '"') { return false; } - return true; + // Past SIMDJSON_PADDING we can no longer rely on the padding to keep the + // comparison in bounds, so we must stop at the quote terminating the key. + return is_equal(target); } simdjson_inline bool raw_json_string::is_equal(std::string_view target) const noexcept { @@ -104368,13 +104894,21 @@ class value { simdjson_inline simdjson_result at_path(std::string_view at_path) noexcept; /** - * Get all values matching the given JSONPath expression with wildcard support. + * Call the provided callback for each value matching the given JSONPath + * expression with wildcard support. * Supports wildcard character (*) for arrays or ".*" for objects. * * @param json_path JSONPath expression with wildcards - * @return Vector of values matching the wildcard pattern + * @param callback Function called for each matching value + * @return error_code indicating success or failure */ - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; protected: /** @@ -104567,9 +105101,23 @@ struct simdjson_result : public icelake::implementatio simdjson_inline simdjson_result current_depth() const noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; }; +// Forward-declare explicit specializations so MSVC /permissive- sees them before +// any template instantiation that would resolve element.get(val) to the primary. +template<> simdjson_inline error_code +simdjson_result::get( + icelake::ondemand::value &out) noexcept; +template<> simdjson_inline simdjson_result +simdjson_result::get() noexcept; + } // namespace simdjson #endif // SIMDJSON_GENERIC_ONDEMAND_VALUE_H @@ -105908,8 +106456,6 @@ class parser { static simdjson_inline bool release_parser(); private: - friend bool release_parser(); - friend ondemand::parser& get_parser(); /** Get the thread-local parser instance, allocates it if needed */ static simdjson_inline simdjson_warn_unused std::unique_ptr& get_parser_instance(); /** Get the thread-local parser instance, it might be null */ @@ -106072,13 +106618,21 @@ class array { inline simdjson_result at_path(std::string_view json_path) noexcept; /** - * Get all values matching the given JSONPath expression with wildcard support. + * Call the provided callback for each value matching the given JSONPath + * expression with wildcard support. * Supports wildcard patterns like "[*]" to match all array elements. * * @param json_path JSONPath expression with wildcards - * @return Vector of values matching the wildcard pattern + * @param callback Function called for each matching value + * @return error_code indicating success or failure */ - inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; /** * Consumes the array and returns a string_view instance corresponding to the @@ -106202,7 +106756,13 @@ struct simdjson_result : public icelake::implementatio simdjson_inline simdjson_result at(size_t index) noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; simdjson_inline simdjson_result raw_json() noexcept; #if SIMDJSON_SUPPORTS_CONCEPTS // TODO: move this code into object-inl.h @@ -107102,21 +107662,24 @@ class document { simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; /** - * Get all values matching the given JSONPath expression with wildcard support. + * Call the provided callback for each value matching the given JSONPath + * expression with wildcard support. * * Supports wildcard patterns like "$.array[*]" or "$.object.*" to match multiple elements. * - * This method materializes all matching values into a vector. * The document will be consumed after this call. * * @param json_path JSONPath expression with wildcards - * @return Vector of values matching the wildcard pattern, or: - * - INVALID_JSON_POINTER if the JSONPath cannot be parsed - * - NO_SUCH_FIELD if a field does not exist - * - INDEX_OUT_OF_BOUNDS if an array index is out of bounds - * - INCORRECT_TYPE if path traversal encounters wrong type + * @param callback Function called for each matching value + * @return error_code indicating success or failure */ - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; /** * Consumes the document and returns a string_view instance corresponding to the @@ -107337,7 +107900,13 @@ class document_reference { simdjson_inline simdjson_result raw_json_token() noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; private: document *doc{nullptr}; @@ -107423,7 +107992,13 @@ struct simdjson_result : public icelake::implementa simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; #if SIMDJSON_STATIC_REFLECTION template requires(std::is_class_v && (sizeof...(FieldNames) > 0)) @@ -107508,7 +108083,13 @@ struct simdjson_result : public icelake:: simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; #if SIMDJSON_STATIC_REFLECTION template requires(std::is_class_v && (sizeof...(FieldNames) > 0)) @@ -107641,6 +108222,18 @@ class document_stream { * } * size_t truncated = stream.truncated_bytes(); * + * IMPORTANT: this value is only meaningful under the conditions below. It is + * computed from stage-1 bookkeeping, and outside these conditions it is not + * merely imprecise, it is arbitrary -- it can exceed size_in_bytes() or wrap + * around to a huge value. Check it only when both of the following hold: + * + * - you iterated all the way to the end of the stream; + * - no document reported an error. Iteration stops at the first failed + * document, which can leave the bookkeeping from a mid-stream batch. + * + * If you need to know about a truncated tail outside those conditions, track + * it yourself from the last successful document (see iterator::current_index() + * and iterator::source()). */ inline size_t truncated_bytes() const noexcept; @@ -108161,13 +108754,21 @@ class object { inline simdjson_result at_path(std::string_view json_path) noexcept; /** - * Get all values matching the given JSONPath expression with wildcard support. + * Call the provided callback for each value matching the given JSONPath + * expression with wildcard support. * Supports wildcard patterns like ".*" to match all object fields. * * @param json_path JSONPath expression with wildcards - * @return Vector of values matching the wildcard pattern + * @param callback Function called for each matching value + * @return error_code indicating success or failure */ - inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; /** * Reset the iterator so that we are pointing back at the @@ -108317,7 +108918,13 @@ struct simdjson_result : public icelake::implementati simdjson_inline simdjson_result operator[](std::string_view key) && noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; inline simdjson_result reset() noexcept; inline simdjson_result is_empty() noexcept; inline simdjson_result count_fields() & noexcept; @@ -109331,46 +109938,40 @@ inline simdjson_result array::at_path(std::string_view json_path) noexcep return at_pointer(json_pointer); } -inline simdjson_result> array::at_path_with_wildcard(std::string_view json_path) noexcept { - std::vector result; - +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +inline error_code array::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { auto result_pair = get_next_key_and_json_path(json_path); std::string_view key = result_pair.first; std::string_view remaining_path = result_pair.second; + // Empty key means the path segment could not be parsed (including malformed + // bracket-quoted keys). Without this check, an empty key is treated as index + // 0 and remaining_path may be unchanged, causing infinite recursion. + if (key.empty()) { + return INVALID_JSON_POINTER; + } // Wildcard case - if(key=="*"){ - for(auto element: *this){ - - if(element.error()){ - return element.error(); - } - - if(remaining_path.empty()){ - // Use value_unsafe() because we've already checked for errors above. - // The 'element' is a simdjson_result wrapper, and we need to extract - // the underlying value. value_unsafe() is safe here because error() returned false. - result.push_back(std::move(element).value_unsafe()); - - }else{ - auto nested_result = element.at_path_with_wildcard(remaining_path); - - if(nested_result.error()){ - return nested_result.error(); - } - // Same logic as above. - std::vector nested_matches = std::move(nested_result).value_unsafe(); - - result.insert(result.end(), - std::make_move_iterator(nested_matches.begin()), - std::make_move_iterator(nested_matches.end())); + if (key=="*"){ + for(auto element: *this) { + value val; + SIMDJSON_TRY(element.get(val)); + if (remaining_path.empty()) { + callback(val); + } else { + error_code err = element.for_each_at_path_with_wildcard(remaining_path, callback); + if(err) { return err; } } } - return result; - }else{ + return SUCCESS; + } else { // Specific index case in which we access the element at the given index - size_t idx=0; + size_t idx = 0; - for(char c:key){ + for (char c : key) { if(c < '0' || c > '9'){ return INVALID_JSON_POINTER; } @@ -109378,16 +109979,13 @@ inline simdjson_result> array::at_path_with_wildcard(std::str } auto element = at(idx); - - if(element.error()){ - return element.error(); - } - - if(remaining_path.empty()){ - result.push_back(std::move(element).value_unsafe()); - return result; - }else{ - return element.at_path_with_wildcard(remaining_path); + value val; + SIMDJSON_TRY(element.get(val)); + if (remaining_path.empty()){ + callback(val); + return SUCCESS; + } else { + return element.for_each_at_path_with_wildcard(remaining_path, callback); } } } @@ -109450,9 +110048,15 @@ simdjson_inline simdjson_result simdjson_result> simdjson_result::at_path_with_wildcard(std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code simdjson_result::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); + return first.for_each_at_path_with_wildcard(json_path, std::forward(callback)); } simdjson_inline simdjson_result simdjson_result::raw_json() noexcept { if (error()) { return error(); } @@ -109873,14 +110477,21 @@ simdjson_inline simdjson_result value::at_path(std::string_view json_path } } -inline simdjson_result> value::at_path_with_wildcard(std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +inline error_code value::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { + if (size_t(iter.depth()) >= iter.json_iter().parser->max_depth()) { return DEPTH_ERROR; } json_type t; SIMDJSON_TRY(type().get(t)); switch (t) { case json_type::array: - return (*this).get_array().at_path_with_wildcard(json_path); + return (*this).get_array().for_each_at_path_with_wildcard(json_path, std::forward(callback)); case json_type::object: - return (*this).get_object().at_path_with_wildcard(json_path); + return (*this).get_object().for_each_at_path_with_wildcard(json_path, std::forward(callback)); default: return INVALID_JSON_POINTER; } @@ -110144,12 +110755,18 @@ simdjson_inline simdjson_result simdjson_result> simdjson_result::at_path_with_wildcard( - std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +inline error_code simdjson_result::for_each_at_path_with_wildcard( + std::string_view json_path, Func&& callback) noexcept { if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); + return first.for_each_at_path_with_wildcard(json_path, std::forward(callback)); } } // namespace simdjson @@ -110525,8 +111142,14 @@ simdjson_inline simdjson_result document::at_path(std::string_view json_p } } -simdjson_inline simdjson_result> document::at_path_with_wildcard(std::string_view json_path) noexcept { - rewind(); // Rewind the document each time at_path_with_wildcard is called +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code document::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { + rewind(); // Rewind the document each time for_each_at_path_with_wildcard is called if (json_path.empty()) { return INVALID_JSON_POINTER; } @@ -110534,9 +111157,9 @@ simdjson_inline simdjson_result> document::at_path_with_wildc SIMDJSON_TRY(type().get(t)); switch (t) { case json_type::array: - return (*this).get_array().at_path_with_wildcard(json_path); + return (*this).get_array().for_each_at_path_with_wildcard(json_path, std::forward(callback)); case json_type::object: - return (*this).get_object().at_path_with_wildcard(json_path); + return (*this).get_object().for_each_at_path_with_wildcard(json_path, std::forward(callback)); default: return INVALID_JSON_POINTER; } @@ -110873,9 +111496,15 @@ simdjson_inline simdjson_result simdjson_result> simdjson_result::at_path_with_wildcard(std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code simdjson_result::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); + return first.for_each_at_path_with_wildcard(json_path, std::forward(callback)); } #if SIMDJSON_STATIC_REFLECTION @@ -110986,7 +111615,13 @@ simdjson_inline simdjson_result document_reference::get_number() noexcep simdjson_inline simdjson_result document_reference::raw_json_token() noexcept { return doc->raw_json_token(); } simdjson_inline simdjson_result document_reference::at_pointer(std::string_view json_pointer) noexcept { return doc->at_pointer(json_pointer); } simdjson_inline simdjson_result document_reference::at_path(std::string_view json_path) noexcept { return doc->at_path(json_path); } -simdjson_inline simdjson_result> document_reference::at_path_with_wildcard(std::string_view json_path) noexcept { return doc->at_path_with_wildcard(json_path); } +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code document_reference::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { return doc->for_each_at_path_with_wildcard(json_path, std::forward(callback)); } simdjson_inline simdjson_result document_reference::raw_json() noexcept { return doc->raw_json();} simdjson_inline document_reference::operator document&() const noexcept { return *doc; } #if SIMDJSON_SUPPORTS_CONCEPTS && SIMDJSON_STATIC_REFLECTION @@ -111251,11 +111886,17 @@ simdjson_inline simdjson_result simdjson_result> simdjson_result::at_path_with_wildcard(std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code simdjson_result::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); + return first.for_each_at_path_with_wildcard(json_path, std::forward(callback)); } #if SIMDJSON_STATIC_REFLECTION template @@ -111629,7 +112270,7 @@ simdjson_inline std::string_view document_stream::iterator::source() const noexc // TODO: We could remove trailing whitespaces // This returns a string spanning from start of value to the beginning of the next document (excluded) { - auto next_index = stream->parser->implementation->structural_indexes[++cur_struct_index]; + auto next_index = stream->batch_start + stream->parser->implementation->structural_indexes[++cur_struct_index]; // normally the length would be next_index - current_index() - 1, except for the last document size_t svlen = next_index - current_index(); const char *start = reinterpret_cast(stream->buf) + current_index(); @@ -112833,46 +113474,44 @@ inline simdjson_result object::at_path(std::string_view json_path) noexce return at_pointer(json_pointer); } -inline simdjson_result> object::at_path_with_wildcard(std::string_view json_path) noexcept { - std::vector result; - +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +inline error_code object::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { auto result_pair = get_next_key_and_json_path(json_path); std::string_view key = result_pair.first; std::string_view remaining_path = result_pair.second; + // Empty key means the path segment could not be parsed (including malformed + // bracket-quoted keys). Match DOM at_path_with_wildcard and avoid treating + // "no progress" as a recursive lookup of "". + if (key.empty()) { + return INVALID_JSON_POINTER; + } // Handle when its the case for wildcard if (key == "*") { // Loop through each field in the object for (auto field : *this) { value val; SIMDJSON_TRY(field.value().get(val)); - if (remaining_path.empty()) { - result.push_back(std::move(val)); + callback(val); } else { - auto nested_result = val.at_path_with_wildcard(remaining_path); - - if (nested_result.error()) { - return nested_result.error(); - } - // Extract and append all nested matches to our result - std::vector nested_vec; - SIMDJSON_TRY(std::move(nested_result).get(nested_vec)); - - result.insert(result.end(), - std::make_move_iterator(nested_vec.begin()), - std::make_move_iterator(nested_vec.end())); + SIMDJSON_TRY(val.for_each_at_path_with_wildcard(remaining_path, callback)); } } - return result; + return SUCCESS; } else { value val; SIMDJSON_TRY(find_field(key).get(val)); if (remaining_path.empty()) { - result.push_back(std::move(val)); - return result; + callback(val); + return SUCCESS; } else { - return val.at_path_with_wildcard(remaining_path); + return val.for_each_at_path_with_wildcard(remaining_path, callback); } } } @@ -113003,9 +113642,15 @@ simdjson_inline simdjson_result simdjson_result> simdjson_result::at_path_with_wildcard(std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code simdjson_result::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); + return first.for_each_at_path_with_wildcard(json_path, std::forward(callback)); } inline simdjson_result simdjson_result::reset() noexcept { @@ -113210,7 +113855,7 @@ simdjson_inline parser::parser(size_t max_capacity) noexcept simdjson_warn_unused simdjson_inline error_code parser::allocate(size_t new_capacity, size_t new_max_depth) noexcept { if (new_capacity > max_capacity()) { return CAPACITY; } - if (string_buf && new_capacity == capacity() && new_max_depth == max_depth()) { return SUCCESS; } + if (string_buf && new_capacity <= capacity() && new_max_depth <= max_depth()) { return SUCCESS; } // string_capacity copied from document::allocate _capacity = 0; @@ -113386,7 +114031,7 @@ simdjson_inline simdjson_warn_unused ondemand::parser& parser::get_parser() { return *parser::get_parser_instance(); } -simdjson_inline bool release_parser() { +simdjson_inline bool parser::release_parser() { auto &parser_instance = parser::get_threadlocal_parser_if_exists(); if (parser_instance) { parser_instance.reset(); @@ -113503,13 +114148,9 @@ simdjson_inline bool raw_json_string::unsafe_is_equal(std::string_view target) c if(target.size() <= SIMDJSON_PADDING) { return (raw()[target.size()] == '"') && !memcmp(raw(), target.data(), target.size()); } - const char * r{raw()}; - size_t pos{0}; - for(;pos < target.size();pos++) { - if(r[pos] != target[pos]) { return false; } - } - if(r[pos] != '"') { return false; } - return true; + // Past SIMDJSON_PADDING we can no longer rely on the padding to keep the + // comparison in bounds, so we must stop at the quote terminating the key. + return is_equal(target); } simdjson_inline bool raw_json_string::is_equal(std::string_view target) const noexcept { @@ -118246,13 +118887,21 @@ class value { simdjson_inline simdjson_result at_path(std::string_view at_path) noexcept; /** - * Get all values matching the given JSONPath expression with wildcard support. + * Call the provided callback for each value matching the given JSONPath + * expression with wildcard support. * Supports wildcard character (*) for arrays or ".*" for objects. * * @param json_path JSONPath expression with wildcards - * @return Vector of values matching the wildcard pattern + * @param callback Function called for each matching value + * @return error_code indicating success or failure */ - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; protected: /** @@ -118445,9 +119094,23 @@ struct simdjson_result : public ppc64::implementation_si simdjson_inline simdjson_result current_depth() const noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; }; +// Forward-declare explicit specializations so MSVC /permissive- sees them before +// any template instantiation that would resolve element.get(val) to the primary. +template<> simdjson_inline error_code +simdjson_result::get( + ppc64::ondemand::value &out) noexcept; +template<> simdjson_inline simdjson_result +simdjson_result::get() noexcept; + } // namespace simdjson #endif // SIMDJSON_GENERIC_ONDEMAND_VALUE_H @@ -119786,8 +120449,6 @@ class parser { static simdjson_inline bool release_parser(); private: - friend bool release_parser(); - friend ondemand::parser& get_parser(); /** Get the thread-local parser instance, allocates it if needed */ static simdjson_inline simdjson_warn_unused std::unique_ptr& get_parser_instance(); /** Get the thread-local parser instance, it might be null */ @@ -119950,13 +120611,21 @@ class array { inline simdjson_result at_path(std::string_view json_path) noexcept; /** - * Get all values matching the given JSONPath expression with wildcard support. + * Call the provided callback for each value matching the given JSONPath + * expression with wildcard support. * Supports wildcard patterns like "[*]" to match all array elements. * * @param json_path JSONPath expression with wildcards - * @return Vector of values matching the wildcard pattern + * @param callback Function called for each matching value + * @return error_code indicating success or failure */ - inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; /** * Consumes the array and returns a string_view instance corresponding to the @@ -120080,7 +120749,13 @@ struct simdjson_result : public ppc64::implementation_si simdjson_inline simdjson_result at(size_t index) noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; simdjson_inline simdjson_result raw_json() noexcept; #if SIMDJSON_SUPPORTS_CONCEPTS // TODO: move this code into object-inl.h @@ -120980,21 +121655,24 @@ class document { simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; /** - * Get all values matching the given JSONPath expression with wildcard support. + * Call the provided callback for each value matching the given JSONPath + * expression with wildcard support. * * Supports wildcard patterns like "$.array[*]" or "$.object.*" to match multiple elements. * - * This method materializes all matching values into a vector. * The document will be consumed after this call. * * @param json_path JSONPath expression with wildcards - * @return Vector of values matching the wildcard pattern, or: - * - INVALID_JSON_POINTER if the JSONPath cannot be parsed - * - NO_SUCH_FIELD if a field does not exist - * - INDEX_OUT_OF_BOUNDS if an array index is out of bounds - * - INCORRECT_TYPE if path traversal encounters wrong type + * @param callback Function called for each matching value + * @return error_code indicating success or failure */ - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; /** * Consumes the document and returns a string_view instance corresponding to the @@ -121215,7 +121893,13 @@ class document_reference { simdjson_inline simdjson_result raw_json_token() noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; private: document *doc{nullptr}; @@ -121301,7 +121985,13 @@ struct simdjson_result : public ppc64::implementation simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; #if SIMDJSON_STATIC_REFLECTION template requires(std::is_class_v && (sizeof...(FieldNames) > 0)) @@ -121386,7 +122076,13 @@ struct simdjson_result : public ppc64::impl simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; #if SIMDJSON_STATIC_REFLECTION template requires(std::is_class_v && (sizeof...(FieldNames) > 0)) @@ -121519,6 +122215,18 @@ class document_stream { * } * size_t truncated = stream.truncated_bytes(); * + * IMPORTANT: this value is only meaningful under the conditions below. It is + * computed from stage-1 bookkeeping, and outside these conditions it is not + * merely imprecise, it is arbitrary -- it can exceed size_in_bytes() or wrap + * around to a huge value. Check it only when both of the following hold: + * + * - you iterated all the way to the end of the stream; + * - no document reported an error. Iteration stops at the first failed + * document, which can leave the bookkeeping from a mid-stream batch. + * + * If you need to know about a truncated tail outside those conditions, track + * it yourself from the last successful document (see iterator::current_index() + * and iterator::source()). */ inline size_t truncated_bytes() const noexcept; @@ -122039,13 +122747,21 @@ class object { inline simdjson_result at_path(std::string_view json_path) noexcept; /** - * Get all values matching the given JSONPath expression with wildcard support. + * Call the provided callback for each value matching the given JSONPath + * expression with wildcard support. * Supports wildcard patterns like ".*" to match all object fields. * * @param json_path JSONPath expression with wildcards - * @return Vector of values matching the wildcard pattern + * @param callback Function called for each matching value + * @return error_code indicating success or failure */ - inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; /** * Reset the iterator so that we are pointing back at the @@ -122195,7 +122911,13 @@ struct simdjson_result : public ppc64::implementation_s simdjson_inline simdjson_result operator[](std::string_view key) && noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; inline simdjson_result reset() noexcept; inline simdjson_result is_empty() noexcept; inline simdjson_result count_fields() & noexcept; @@ -123209,46 +123931,40 @@ inline simdjson_result array::at_path(std::string_view json_path) noexcep return at_pointer(json_pointer); } -inline simdjson_result> array::at_path_with_wildcard(std::string_view json_path) noexcept { - std::vector result; - +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +inline error_code array::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { auto result_pair = get_next_key_and_json_path(json_path); std::string_view key = result_pair.first; std::string_view remaining_path = result_pair.second; + // Empty key means the path segment could not be parsed (including malformed + // bracket-quoted keys). Without this check, an empty key is treated as index + // 0 and remaining_path may be unchanged, causing infinite recursion. + if (key.empty()) { + return INVALID_JSON_POINTER; + } // Wildcard case - if(key=="*"){ - for(auto element: *this){ - - if(element.error()){ - return element.error(); - } - - if(remaining_path.empty()){ - // Use value_unsafe() because we've already checked for errors above. - // The 'element' is a simdjson_result wrapper, and we need to extract - // the underlying value. value_unsafe() is safe here because error() returned false. - result.push_back(std::move(element).value_unsafe()); - - }else{ - auto nested_result = element.at_path_with_wildcard(remaining_path); - - if(nested_result.error()){ - return nested_result.error(); - } - // Same logic as above. - std::vector nested_matches = std::move(nested_result).value_unsafe(); - - result.insert(result.end(), - std::make_move_iterator(nested_matches.begin()), - std::make_move_iterator(nested_matches.end())); + if (key=="*"){ + for(auto element: *this) { + value val; + SIMDJSON_TRY(element.get(val)); + if (remaining_path.empty()) { + callback(val); + } else { + error_code err = element.for_each_at_path_with_wildcard(remaining_path, callback); + if(err) { return err; } } } - return result; - }else{ + return SUCCESS; + } else { // Specific index case in which we access the element at the given index - size_t idx=0; + size_t idx = 0; - for(char c:key){ + for (char c : key) { if(c < '0' || c > '9'){ return INVALID_JSON_POINTER; } @@ -123256,16 +123972,13 @@ inline simdjson_result> array::at_path_with_wildcard(std::str } auto element = at(idx); - - if(element.error()){ - return element.error(); - } - - if(remaining_path.empty()){ - result.push_back(std::move(element).value_unsafe()); - return result; - }else{ - return element.at_path_with_wildcard(remaining_path); + value val; + SIMDJSON_TRY(element.get(val)); + if (remaining_path.empty()){ + callback(val); + return SUCCESS; + } else { + return element.for_each_at_path_with_wildcard(remaining_path, callback); } } } @@ -123328,9 +124041,15 @@ simdjson_inline simdjson_result simdjson_result> simdjson_result::at_path_with_wildcard(std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code simdjson_result::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); + return first.for_each_at_path_with_wildcard(json_path, std::forward(callback)); } simdjson_inline simdjson_result simdjson_result::raw_json() noexcept { if (error()) { return error(); } @@ -123751,14 +124470,21 @@ simdjson_inline simdjson_result value::at_path(std::string_view json_path } } -inline simdjson_result> value::at_path_with_wildcard(std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +inline error_code value::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { + if (size_t(iter.depth()) >= iter.json_iter().parser->max_depth()) { return DEPTH_ERROR; } json_type t; SIMDJSON_TRY(type().get(t)); switch (t) { case json_type::array: - return (*this).get_array().at_path_with_wildcard(json_path); + return (*this).get_array().for_each_at_path_with_wildcard(json_path, std::forward(callback)); case json_type::object: - return (*this).get_object().at_path_with_wildcard(json_path); + return (*this).get_object().for_each_at_path_with_wildcard(json_path, std::forward(callback)); default: return INVALID_JSON_POINTER; } @@ -124022,12 +124748,18 @@ simdjson_inline simdjson_result simdjson_result> simdjson_result::at_path_with_wildcard( - std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +inline error_code simdjson_result::for_each_at_path_with_wildcard( + std::string_view json_path, Func&& callback) noexcept { if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); + return first.for_each_at_path_with_wildcard(json_path, std::forward(callback)); } } // namespace simdjson @@ -124403,8 +125135,14 @@ simdjson_inline simdjson_result document::at_path(std::string_view json_p } } -simdjson_inline simdjson_result> document::at_path_with_wildcard(std::string_view json_path) noexcept { - rewind(); // Rewind the document each time at_path_with_wildcard is called +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code document::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { + rewind(); // Rewind the document each time for_each_at_path_with_wildcard is called if (json_path.empty()) { return INVALID_JSON_POINTER; } @@ -124412,9 +125150,9 @@ simdjson_inline simdjson_result> document::at_path_with_wildc SIMDJSON_TRY(type().get(t)); switch (t) { case json_type::array: - return (*this).get_array().at_path_with_wildcard(json_path); + return (*this).get_array().for_each_at_path_with_wildcard(json_path, std::forward(callback)); case json_type::object: - return (*this).get_object().at_path_with_wildcard(json_path); + return (*this).get_object().for_each_at_path_with_wildcard(json_path, std::forward(callback)); default: return INVALID_JSON_POINTER; } @@ -124751,9 +125489,15 @@ simdjson_inline simdjson_result simdjson_result> simdjson_result::at_path_with_wildcard(std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code simdjson_result::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); + return first.for_each_at_path_with_wildcard(json_path, std::forward(callback)); } #if SIMDJSON_STATIC_REFLECTION @@ -124864,7 +125608,13 @@ simdjson_inline simdjson_result document_reference::get_number() noexcep simdjson_inline simdjson_result document_reference::raw_json_token() noexcept { return doc->raw_json_token(); } simdjson_inline simdjson_result document_reference::at_pointer(std::string_view json_pointer) noexcept { return doc->at_pointer(json_pointer); } simdjson_inline simdjson_result document_reference::at_path(std::string_view json_path) noexcept { return doc->at_path(json_path); } -simdjson_inline simdjson_result> document_reference::at_path_with_wildcard(std::string_view json_path) noexcept { return doc->at_path_with_wildcard(json_path); } +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code document_reference::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { return doc->for_each_at_path_with_wildcard(json_path, std::forward(callback)); } simdjson_inline simdjson_result document_reference::raw_json() noexcept { return doc->raw_json();} simdjson_inline document_reference::operator document&() const noexcept { return *doc; } #if SIMDJSON_SUPPORTS_CONCEPTS && SIMDJSON_STATIC_REFLECTION @@ -125129,11 +125879,17 @@ simdjson_inline simdjson_result simdjson_result> simdjson_result::at_path_with_wildcard(std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code simdjson_result::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); + return first.for_each_at_path_with_wildcard(json_path, std::forward(callback)); } #if SIMDJSON_STATIC_REFLECTION template @@ -125507,7 +126263,7 @@ simdjson_inline std::string_view document_stream::iterator::source() const noexc // TODO: We could remove trailing whitespaces // This returns a string spanning from start of value to the beginning of the next document (excluded) { - auto next_index = stream->parser->implementation->structural_indexes[++cur_struct_index]; + auto next_index = stream->batch_start + stream->parser->implementation->structural_indexes[++cur_struct_index]; // normally the length would be next_index - current_index() - 1, except for the last document size_t svlen = next_index - current_index(); const char *start = reinterpret_cast(stream->buf) + current_index(); @@ -126711,46 +127467,44 @@ inline simdjson_result object::at_path(std::string_view json_path) noexce return at_pointer(json_pointer); } -inline simdjson_result> object::at_path_with_wildcard(std::string_view json_path) noexcept { - std::vector result; - +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +inline error_code object::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { auto result_pair = get_next_key_and_json_path(json_path); std::string_view key = result_pair.first; std::string_view remaining_path = result_pair.second; + // Empty key means the path segment could not be parsed (including malformed + // bracket-quoted keys). Match DOM at_path_with_wildcard and avoid treating + // "no progress" as a recursive lookup of "". + if (key.empty()) { + return INVALID_JSON_POINTER; + } // Handle when its the case for wildcard if (key == "*") { // Loop through each field in the object for (auto field : *this) { value val; SIMDJSON_TRY(field.value().get(val)); - if (remaining_path.empty()) { - result.push_back(std::move(val)); + callback(val); } else { - auto nested_result = val.at_path_with_wildcard(remaining_path); - - if (nested_result.error()) { - return nested_result.error(); - } - // Extract and append all nested matches to our result - std::vector nested_vec; - SIMDJSON_TRY(std::move(nested_result).get(nested_vec)); - - result.insert(result.end(), - std::make_move_iterator(nested_vec.begin()), - std::make_move_iterator(nested_vec.end())); + SIMDJSON_TRY(val.for_each_at_path_with_wildcard(remaining_path, callback)); } } - return result; + return SUCCESS; } else { value val; SIMDJSON_TRY(find_field(key).get(val)); if (remaining_path.empty()) { - result.push_back(std::move(val)); - return result; + callback(val); + return SUCCESS; } else { - return val.at_path_with_wildcard(remaining_path); + return val.for_each_at_path_with_wildcard(remaining_path, callback); } } } @@ -126881,9 +127635,15 @@ simdjson_inline simdjson_result simdjson_result> simdjson_result::at_path_with_wildcard(std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code simdjson_result::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); + return first.for_each_at_path_with_wildcard(json_path, std::forward(callback)); } inline simdjson_result simdjson_result::reset() noexcept { @@ -127088,7 +127848,7 @@ simdjson_inline parser::parser(size_t max_capacity) noexcept simdjson_warn_unused simdjson_inline error_code parser::allocate(size_t new_capacity, size_t new_max_depth) noexcept { if (new_capacity > max_capacity()) { return CAPACITY; } - if (string_buf && new_capacity == capacity() && new_max_depth == max_depth()) { return SUCCESS; } + if (string_buf && new_capacity <= capacity() && new_max_depth <= max_depth()) { return SUCCESS; } // string_capacity copied from document::allocate _capacity = 0; @@ -127264,7 +128024,7 @@ simdjson_inline simdjson_warn_unused ondemand::parser& parser::get_parser() { return *parser::get_parser_instance(); } -simdjson_inline bool release_parser() { +simdjson_inline bool parser::release_parser() { auto &parser_instance = parser::get_threadlocal_parser_if_exists(); if (parser_instance) { parser_instance.reset(); @@ -127381,13 +128141,9 @@ simdjson_inline bool raw_json_string::unsafe_is_equal(std::string_view target) c if(target.size() <= SIMDJSON_PADDING) { return (raw()[target.size()] == '"') && !memcmp(raw(), target.data(), target.size()); } - const char * r{raw()}; - size_t pos{0}; - for(;pos < target.size();pos++) { - if(r[pos] != target[pos]) { return false; } - } - if(r[pos] != '"') { return false; } - return true; + // Past SIMDJSON_PADDING we can no longer rely on the padding to keep the + // comparison in bounds, so we must stop at the quote terminating the key. + return is_equal(target); } simdjson_inline bool raw_json_string::is_equal(std::string_view target) const noexcept { @@ -132441,13 +133197,21 @@ class value { simdjson_inline simdjson_result at_path(std::string_view at_path) noexcept; /** - * Get all values matching the given JSONPath expression with wildcard support. + * Call the provided callback for each value matching the given JSONPath + * expression with wildcard support. * Supports wildcard character (*) for arrays or ".*" for objects. * * @param json_path JSONPath expression with wildcards - * @return Vector of values matching the wildcard pattern + * @param callback Function called for each matching value + * @return error_code indicating success or failure */ - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; protected: /** @@ -132640,9 +133404,23 @@ struct simdjson_result : public westmere::implementat simdjson_inline simdjson_result current_depth() const noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; }; +// Forward-declare explicit specializations so MSVC /permissive- sees them before +// any template instantiation that would resolve element.get(val) to the primary. +template<> simdjson_inline error_code +simdjson_result::get( + westmere::ondemand::value &out) noexcept; +template<> simdjson_inline simdjson_result +simdjson_result::get() noexcept; + } // namespace simdjson #endif // SIMDJSON_GENERIC_ONDEMAND_VALUE_H @@ -133981,8 +134759,6 @@ class parser { static simdjson_inline bool release_parser(); private: - friend bool release_parser(); - friend ondemand::parser& get_parser(); /** Get the thread-local parser instance, allocates it if needed */ static simdjson_inline simdjson_warn_unused std::unique_ptr& get_parser_instance(); /** Get the thread-local parser instance, it might be null */ @@ -134145,13 +134921,21 @@ class array { inline simdjson_result at_path(std::string_view json_path) noexcept; /** - * Get all values matching the given JSONPath expression with wildcard support. + * Call the provided callback for each value matching the given JSONPath + * expression with wildcard support. * Supports wildcard patterns like "[*]" to match all array elements. * * @param json_path JSONPath expression with wildcards - * @return Vector of values matching the wildcard pattern + * @param callback Function called for each matching value + * @return error_code indicating success or failure */ - inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; /** * Consumes the array and returns a string_view instance corresponding to the @@ -134275,7 +135059,13 @@ struct simdjson_result : public westmere::implementat simdjson_inline simdjson_result at(size_t index) noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; simdjson_inline simdjson_result raw_json() noexcept; #if SIMDJSON_SUPPORTS_CONCEPTS // TODO: move this code into object-inl.h @@ -135175,21 +135965,24 @@ class document { simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; /** - * Get all values matching the given JSONPath expression with wildcard support. + * Call the provided callback for each value matching the given JSONPath + * expression with wildcard support. * * Supports wildcard patterns like "$.array[*]" or "$.object.*" to match multiple elements. * - * This method materializes all matching values into a vector. * The document will be consumed after this call. * * @param json_path JSONPath expression with wildcards - * @return Vector of values matching the wildcard pattern, or: - * - INVALID_JSON_POINTER if the JSONPath cannot be parsed - * - NO_SUCH_FIELD if a field does not exist - * - INDEX_OUT_OF_BOUNDS if an array index is out of bounds - * - INCORRECT_TYPE if path traversal encounters wrong type + * @param callback Function called for each matching value + * @return error_code indicating success or failure */ - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; /** * Consumes the document and returns a string_view instance corresponding to the @@ -135410,7 +136203,13 @@ class document_reference { simdjson_inline simdjson_result raw_json_token() noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; private: document *doc{nullptr}; @@ -135496,7 +136295,13 @@ struct simdjson_result : public westmere::implemen simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; #if SIMDJSON_STATIC_REFLECTION template requires(std::is_class_v && (sizeof...(FieldNames) > 0)) @@ -135581,7 +136386,13 @@ struct simdjson_result : public westmere simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; #if SIMDJSON_STATIC_REFLECTION template requires(std::is_class_v && (sizeof...(FieldNames) > 0)) @@ -135714,6 +136525,18 @@ class document_stream { * } * size_t truncated = stream.truncated_bytes(); * + * IMPORTANT: this value is only meaningful under the conditions below. It is + * computed from stage-1 bookkeeping, and outside these conditions it is not + * merely imprecise, it is arbitrary -- it can exceed size_in_bytes() or wrap + * around to a huge value. Check it only when both of the following hold: + * + * - you iterated all the way to the end of the stream; + * - no document reported an error. Iteration stops at the first failed + * document, which can leave the bookkeeping from a mid-stream batch. + * + * If you need to know about a truncated tail outside those conditions, track + * it yourself from the last successful document (see iterator::current_index() + * and iterator::source()). */ inline size_t truncated_bytes() const noexcept; @@ -136234,13 +137057,21 @@ class object { inline simdjson_result at_path(std::string_view json_path) noexcept; /** - * Get all values matching the given JSONPath expression with wildcard support. + * Call the provided callback for each value matching the given JSONPath + * expression with wildcard support. * Supports wildcard patterns like ".*" to match all object fields. * * @param json_path JSONPath expression with wildcards - * @return Vector of values matching the wildcard pattern + * @param callback Function called for each matching value + * @return error_code indicating success or failure */ - inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; /** * Reset the iterator so that we are pointing back at the @@ -136390,7 +137221,13 @@ struct simdjson_result : public westmere::implementa simdjson_inline simdjson_result operator[](std::string_view key) && noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; inline simdjson_result reset() noexcept; inline simdjson_result is_empty() noexcept; inline simdjson_result count_fields() & noexcept; @@ -137404,46 +138241,40 @@ inline simdjson_result array::at_path(std::string_view json_path) noexcep return at_pointer(json_pointer); } -inline simdjson_result> array::at_path_with_wildcard(std::string_view json_path) noexcept { - std::vector result; - +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +inline error_code array::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { auto result_pair = get_next_key_and_json_path(json_path); std::string_view key = result_pair.first; std::string_view remaining_path = result_pair.second; + // Empty key means the path segment could not be parsed (including malformed + // bracket-quoted keys). Without this check, an empty key is treated as index + // 0 and remaining_path may be unchanged, causing infinite recursion. + if (key.empty()) { + return INVALID_JSON_POINTER; + } // Wildcard case - if(key=="*"){ - for(auto element: *this){ - - if(element.error()){ - return element.error(); - } - - if(remaining_path.empty()){ - // Use value_unsafe() because we've already checked for errors above. - // The 'element' is a simdjson_result wrapper, and we need to extract - // the underlying value. value_unsafe() is safe here because error() returned false. - result.push_back(std::move(element).value_unsafe()); - - }else{ - auto nested_result = element.at_path_with_wildcard(remaining_path); - - if(nested_result.error()){ - return nested_result.error(); - } - // Same logic as above. - std::vector nested_matches = std::move(nested_result).value_unsafe(); - - result.insert(result.end(), - std::make_move_iterator(nested_matches.begin()), - std::make_move_iterator(nested_matches.end())); + if (key=="*"){ + for(auto element: *this) { + value val; + SIMDJSON_TRY(element.get(val)); + if (remaining_path.empty()) { + callback(val); + } else { + error_code err = element.for_each_at_path_with_wildcard(remaining_path, callback); + if(err) { return err; } } } - return result; - }else{ + return SUCCESS; + } else { // Specific index case in which we access the element at the given index - size_t idx=0; + size_t idx = 0; - for(char c:key){ + for (char c : key) { if(c < '0' || c > '9'){ return INVALID_JSON_POINTER; } @@ -137451,16 +138282,13 @@ inline simdjson_result> array::at_path_with_wildcard(std::str } auto element = at(idx); - - if(element.error()){ - return element.error(); - } - - if(remaining_path.empty()){ - result.push_back(std::move(element).value_unsafe()); - return result; - }else{ - return element.at_path_with_wildcard(remaining_path); + value val; + SIMDJSON_TRY(element.get(val)); + if (remaining_path.empty()){ + callback(val); + return SUCCESS; + } else { + return element.for_each_at_path_with_wildcard(remaining_path, callback); } } } @@ -137523,9 +138351,15 @@ simdjson_inline simdjson_result simdjson_result> simdjson_result::at_path_with_wildcard(std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code simdjson_result::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); + return first.for_each_at_path_with_wildcard(json_path, std::forward(callback)); } simdjson_inline simdjson_result simdjson_result::raw_json() noexcept { if (error()) { return error(); } @@ -137946,14 +138780,21 @@ simdjson_inline simdjson_result value::at_path(std::string_view json_path } } -inline simdjson_result> value::at_path_with_wildcard(std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +inline error_code value::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { + if (size_t(iter.depth()) >= iter.json_iter().parser->max_depth()) { return DEPTH_ERROR; } json_type t; SIMDJSON_TRY(type().get(t)); switch (t) { case json_type::array: - return (*this).get_array().at_path_with_wildcard(json_path); + return (*this).get_array().for_each_at_path_with_wildcard(json_path, std::forward(callback)); case json_type::object: - return (*this).get_object().at_path_with_wildcard(json_path); + return (*this).get_object().for_each_at_path_with_wildcard(json_path, std::forward(callback)); default: return INVALID_JSON_POINTER; } @@ -138217,12 +139058,18 @@ simdjson_inline simdjson_result simdjson_result> simdjson_result::at_path_with_wildcard( - std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +inline error_code simdjson_result::for_each_at_path_with_wildcard( + std::string_view json_path, Func&& callback) noexcept { if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); + return first.for_each_at_path_with_wildcard(json_path, std::forward(callback)); } } // namespace simdjson @@ -138598,8 +139445,14 @@ simdjson_inline simdjson_result document::at_path(std::string_view json_p } } -simdjson_inline simdjson_result> document::at_path_with_wildcard(std::string_view json_path) noexcept { - rewind(); // Rewind the document each time at_path_with_wildcard is called +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code document::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { + rewind(); // Rewind the document each time for_each_at_path_with_wildcard is called if (json_path.empty()) { return INVALID_JSON_POINTER; } @@ -138607,9 +139460,9 @@ simdjson_inline simdjson_result> document::at_path_with_wildc SIMDJSON_TRY(type().get(t)); switch (t) { case json_type::array: - return (*this).get_array().at_path_with_wildcard(json_path); + return (*this).get_array().for_each_at_path_with_wildcard(json_path, std::forward(callback)); case json_type::object: - return (*this).get_object().at_path_with_wildcard(json_path); + return (*this).get_object().for_each_at_path_with_wildcard(json_path, std::forward(callback)); default: return INVALID_JSON_POINTER; } @@ -138946,9 +139799,15 @@ simdjson_inline simdjson_result simdjson_result> simdjson_result::at_path_with_wildcard(std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code simdjson_result::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); + return first.for_each_at_path_with_wildcard(json_path, std::forward(callback)); } #if SIMDJSON_STATIC_REFLECTION @@ -139059,7 +139918,13 @@ simdjson_inline simdjson_result document_reference::get_number() noexcep simdjson_inline simdjson_result document_reference::raw_json_token() noexcept { return doc->raw_json_token(); } simdjson_inline simdjson_result document_reference::at_pointer(std::string_view json_pointer) noexcept { return doc->at_pointer(json_pointer); } simdjson_inline simdjson_result document_reference::at_path(std::string_view json_path) noexcept { return doc->at_path(json_path); } -simdjson_inline simdjson_result> document_reference::at_path_with_wildcard(std::string_view json_path) noexcept { return doc->at_path_with_wildcard(json_path); } +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code document_reference::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { return doc->for_each_at_path_with_wildcard(json_path, std::forward(callback)); } simdjson_inline simdjson_result document_reference::raw_json() noexcept { return doc->raw_json();} simdjson_inline document_reference::operator document&() const noexcept { return *doc; } #if SIMDJSON_SUPPORTS_CONCEPTS && SIMDJSON_STATIC_REFLECTION @@ -139324,11 +140189,17 @@ simdjson_inline simdjson_result simdjson_result> simdjson_result::at_path_with_wildcard(std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code simdjson_result::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); + return first.for_each_at_path_with_wildcard(json_path, std::forward(callback)); } #if SIMDJSON_STATIC_REFLECTION template @@ -139702,7 +140573,7 @@ simdjson_inline std::string_view document_stream::iterator::source() const noexc // TODO: We could remove trailing whitespaces // This returns a string spanning from start of value to the beginning of the next document (excluded) { - auto next_index = stream->parser->implementation->structural_indexes[++cur_struct_index]; + auto next_index = stream->batch_start + stream->parser->implementation->structural_indexes[++cur_struct_index]; // normally the length would be next_index - current_index() - 1, except for the last document size_t svlen = next_index - current_index(); const char *start = reinterpret_cast(stream->buf) + current_index(); @@ -140906,46 +141777,44 @@ inline simdjson_result object::at_path(std::string_view json_path) noexce return at_pointer(json_pointer); } -inline simdjson_result> object::at_path_with_wildcard(std::string_view json_path) noexcept { - std::vector result; - +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +inline error_code object::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { auto result_pair = get_next_key_and_json_path(json_path); std::string_view key = result_pair.first; std::string_view remaining_path = result_pair.second; + // Empty key means the path segment could not be parsed (including malformed + // bracket-quoted keys). Match DOM at_path_with_wildcard and avoid treating + // "no progress" as a recursive lookup of "". + if (key.empty()) { + return INVALID_JSON_POINTER; + } // Handle when its the case for wildcard if (key == "*") { // Loop through each field in the object for (auto field : *this) { value val; SIMDJSON_TRY(field.value().get(val)); - if (remaining_path.empty()) { - result.push_back(std::move(val)); + callback(val); } else { - auto nested_result = val.at_path_with_wildcard(remaining_path); - - if (nested_result.error()) { - return nested_result.error(); - } - // Extract and append all nested matches to our result - std::vector nested_vec; - SIMDJSON_TRY(std::move(nested_result).get(nested_vec)); - - result.insert(result.end(), - std::make_move_iterator(nested_vec.begin()), - std::make_move_iterator(nested_vec.end())); + SIMDJSON_TRY(val.for_each_at_path_with_wildcard(remaining_path, callback)); } } - return result; + return SUCCESS; } else { value val; SIMDJSON_TRY(find_field(key).get(val)); if (remaining_path.empty()) { - result.push_back(std::move(val)); - return result; + callback(val); + return SUCCESS; } else { - return val.at_path_with_wildcard(remaining_path); + return val.for_each_at_path_with_wildcard(remaining_path, callback); } } } @@ -141076,9 +141945,15 @@ simdjson_inline simdjson_result simdjson_result> simdjson_result::at_path_with_wildcard(std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code simdjson_result::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); + return first.for_each_at_path_with_wildcard(json_path, std::forward(callback)); } inline simdjson_result simdjson_result::reset() noexcept { @@ -141283,7 +142158,7 @@ simdjson_inline parser::parser(size_t max_capacity) noexcept simdjson_warn_unused simdjson_inline error_code parser::allocate(size_t new_capacity, size_t new_max_depth) noexcept { if (new_capacity > max_capacity()) { return CAPACITY; } - if (string_buf && new_capacity == capacity() && new_max_depth == max_depth()) { return SUCCESS; } + if (string_buf && new_capacity <= capacity() && new_max_depth <= max_depth()) { return SUCCESS; } // string_capacity copied from document::allocate _capacity = 0; @@ -141459,7 +142334,7 @@ simdjson_inline simdjson_warn_unused ondemand::parser& parser::get_parser() { return *parser::get_parser_instance(); } -simdjson_inline bool release_parser() { +simdjson_inline bool parser::release_parser() { auto &parser_instance = parser::get_threadlocal_parser_if_exists(); if (parser_instance) { parser_instance.reset(); @@ -141576,13 +142451,9 @@ simdjson_inline bool raw_json_string::unsafe_is_equal(std::string_view target) c if(target.size() <= SIMDJSON_PADDING) { return (raw()[target.size()] == '"') && !memcmp(raw(), target.data(), target.size()); } - const char * r{raw()}; - size_t pos{0}; - for(;pos < target.size();pos++) { - if(r[pos] != target[pos]) { return false; } - } - if(r[pos] != '"') { return false; } - return true; + // Past SIMDJSON_PADDING we can no longer rely on the padding to keep the + // comparison in bounds, so we must stop at the quote terminating the key. + return is_equal(target); } simdjson_inline bool raw_json_string::is_equal(std::string_view target) const noexcept { @@ -146110,13 +146981,21 @@ class value { simdjson_inline simdjson_result at_path(std::string_view at_path) noexcept; /** - * Get all values matching the given JSONPath expression with wildcard support. + * Call the provided callback for each value matching the given JSONPath + * expression with wildcard support. * Supports wildcard character (*) for arrays or ".*" for objects. * * @param json_path JSONPath expression with wildcards - * @return Vector of values matching the wildcard pattern + * @param callback Function called for each matching value + * @return error_code indicating success or failure */ - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; protected: /** @@ -146309,9 +147188,23 @@ struct simdjson_result : public lsx::implementation_simdjs simdjson_inline simdjson_result current_depth() const noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; }; +// Forward-declare explicit specializations so MSVC /permissive- sees them before +// any template instantiation that would resolve element.get(val) to the primary. +template<> simdjson_inline error_code +simdjson_result::get( + lsx::ondemand::value &out) noexcept; +template<> simdjson_inline simdjson_result +simdjson_result::get() noexcept; + } // namespace simdjson #endif // SIMDJSON_GENERIC_ONDEMAND_VALUE_H @@ -147650,8 +148543,6 @@ class parser { static simdjson_inline bool release_parser(); private: - friend bool release_parser(); - friend ondemand::parser& get_parser(); /** Get the thread-local parser instance, allocates it if needed */ static simdjson_inline simdjson_warn_unused std::unique_ptr& get_parser_instance(); /** Get the thread-local parser instance, it might be null */ @@ -147814,13 +148705,21 @@ class array { inline simdjson_result at_path(std::string_view json_path) noexcept; /** - * Get all values matching the given JSONPath expression with wildcard support. + * Call the provided callback for each value matching the given JSONPath + * expression with wildcard support. * Supports wildcard patterns like "[*]" to match all array elements. * * @param json_path JSONPath expression with wildcards - * @return Vector of values matching the wildcard pattern + * @param callback Function called for each matching value + * @return error_code indicating success or failure */ - inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; /** * Consumes the array and returns a string_view instance corresponding to the @@ -147944,7 +148843,13 @@ struct simdjson_result : public lsx::implementation_simdjs simdjson_inline simdjson_result at(size_t index) noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; simdjson_inline simdjson_result raw_json() noexcept; #if SIMDJSON_SUPPORTS_CONCEPTS // TODO: move this code into object-inl.h @@ -148844,21 +149749,24 @@ class document { simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; /** - * Get all values matching the given JSONPath expression with wildcard support. + * Call the provided callback for each value matching the given JSONPath + * expression with wildcard support. * * Supports wildcard patterns like "$.array[*]" or "$.object.*" to match multiple elements. * - * This method materializes all matching values into a vector. * The document will be consumed after this call. * * @param json_path JSONPath expression with wildcards - * @return Vector of values matching the wildcard pattern, or: - * - INVALID_JSON_POINTER if the JSONPath cannot be parsed - * - NO_SUCH_FIELD if a field does not exist - * - INDEX_OUT_OF_BOUNDS if an array index is out of bounds - * - INCORRECT_TYPE if path traversal encounters wrong type + * @param callback Function called for each matching value + * @return error_code indicating success or failure */ - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; /** * Consumes the document and returns a string_view instance corresponding to the @@ -149079,7 +149987,13 @@ class document_reference { simdjson_inline simdjson_result raw_json_token() noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; private: document *doc{nullptr}; @@ -149165,7 +150079,13 @@ struct simdjson_result : public lsx::implementation_sim simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; #if SIMDJSON_STATIC_REFLECTION template requires(std::is_class_v && (sizeof...(FieldNames) > 0)) @@ -149250,7 +150170,13 @@ struct simdjson_result : public lsx::implemen simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; #if SIMDJSON_STATIC_REFLECTION template requires(std::is_class_v && (sizeof...(FieldNames) > 0)) @@ -149383,6 +150309,18 @@ class document_stream { * } * size_t truncated = stream.truncated_bytes(); * + * IMPORTANT: this value is only meaningful under the conditions below. It is + * computed from stage-1 bookkeeping, and outside these conditions it is not + * merely imprecise, it is arbitrary -- it can exceed size_in_bytes() or wrap + * around to a huge value. Check it only when both of the following hold: + * + * - you iterated all the way to the end of the stream; + * - no document reported an error. Iteration stops at the first failed + * document, which can leave the bookkeeping from a mid-stream batch. + * + * If you need to know about a truncated tail outside those conditions, track + * it yourself from the last successful document (see iterator::current_index() + * and iterator::source()). */ inline size_t truncated_bytes() const noexcept; @@ -149903,13 +150841,21 @@ class object { inline simdjson_result at_path(std::string_view json_path) noexcept; /** - * Get all values matching the given JSONPath expression with wildcard support. + * Call the provided callback for each value matching the given JSONPath + * expression with wildcard support. * Supports wildcard patterns like ".*" to match all object fields. * * @param json_path JSONPath expression with wildcards - * @return Vector of values matching the wildcard pattern + * @param callback Function called for each matching value + * @return error_code indicating success or failure */ - inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; /** * Reset the iterator so that we are pointing back at the @@ -150059,7 +151005,13 @@ struct simdjson_result : public lsx::implementation_simdj simdjson_inline simdjson_result operator[](std::string_view key) && noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; inline simdjson_result reset() noexcept; inline simdjson_result is_empty() noexcept; inline simdjson_result count_fields() & noexcept; @@ -151073,46 +152025,40 @@ inline simdjson_result array::at_path(std::string_view json_path) noexcep return at_pointer(json_pointer); } -inline simdjson_result> array::at_path_with_wildcard(std::string_view json_path) noexcept { - std::vector result; - +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +inline error_code array::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { auto result_pair = get_next_key_and_json_path(json_path); std::string_view key = result_pair.first; std::string_view remaining_path = result_pair.second; + // Empty key means the path segment could not be parsed (including malformed + // bracket-quoted keys). Without this check, an empty key is treated as index + // 0 and remaining_path may be unchanged, causing infinite recursion. + if (key.empty()) { + return INVALID_JSON_POINTER; + } // Wildcard case - if(key=="*"){ - for(auto element: *this){ - - if(element.error()){ - return element.error(); - } - - if(remaining_path.empty()){ - // Use value_unsafe() because we've already checked for errors above. - // The 'element' is a simdjson_result wrapper, and we need to extract - // the underlying value. value_unsafe() is safe here because error() returned false. - result.push_back(std::move(element).value_unsafe()); - - }else{ - auto nested_result = element.at_path_with_wildcard(remaining_path); - - if(nested_result.error()){ - return nested_result.error(); - } - // Same logic as above. - std::vector nested_matches = std::move(nested_result).value_unsafe(); - - result.insert(result.end(), - std::make_move_iterator(nested_matches.begin()), - std::make_move_iterator(nested_matches.end())); + if (key=="*"){ + for(auto element: *this) { + value val; + SIMDJSON_TRY(element.get(val)); + if (remaining_path.empty()) { + callback(val); + } else { + error_code err = element.for_each_at_path_with_wildcard(remaining_path, callback); + if(err) { return err; } } } - return result; - }else{ + return SUCCESS; + } else { // Specific index case in which we access the element at the given index - size_t idx=0; + size_t idx = 0; - for(char c:key){ + for (char c : key) { if(c < '0' || c > '9'){ return INVALID_JSON_POINTER; } @@ -151120,16 +152066,13 @@ inline simdjson_result> array::at_path_with_wildcard(std::str } auto element = at(idx); - - if(element.error()){ - return element.error(); - } - - if(remaining_path.empty()){ - result.push_back(std::move(element).value_unsafe()); - return result; - }else{ - return element.at_path_with_wildcard(remaining_path); + value val; + SIMDJSON_TRY(element.get(val)); + if (remaining_path.empty()){ + callback(val); + return SUCCESS; + } else { + return element.for_each_at_path_with_wildcard(remaining_path, callback); } } } @@ -151192,9 +152135,15 @@ simdjson_inline simdjson_result simdjson_result> simdjson_result::at_path_with_wildcard(std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code simdjson_result::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); + return first.for_each_at_path_with_wildcard(json_path, std::forward(callback)); } simdjson_inline simdjson_result simdjson_result::raw_json() noexcept { if (error()) { return error(); } @@ -151615,14 +152564,21 @@ simdjson_inline simdjson_result value::at_path(std::string_view json_path } } -inline simdjson_result> value::at_path_with_wildcard(std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +inline error_code value::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { + if (size_t(iter.depth()) >= iter.json_iter().parser->max_depth()) { return DEPTH_ERROR; } json_type t; SIMDJSON_TRY(type().get(t)); switch (t) { case json_type::array: - return (*this).get_array().at_path_with_wildcard(json_path); + return (*this).get_array().for_each_at_path_with_wildcard(json_path, std::forward(callback)); case json_type::object: - return (*this).get_object().at_path_with_wildcard(json_path); + return (*this).get_object().for_each_at_path_with_wildcard(json_path, std::forward(callback)); default: return INVALID_JSON_POINTER; } @@ -151886,12 +152842,18 @@ simdjson_inline simdjson_result simdjson_result> simdjson_result::at_path_with_wildcard( - std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +inline error_code simdjson_result::for_each_at_path_with_wildcard( + std::string_view json_path, Func&& callback) noexcept { if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); + return first.for_each_at_path_with_wildcard(json_path, std::forward(callback)); } } // namespace simdjson @@ -152267,8 +153229,14 @@ simdjson_inline simdjson_result document::at_path(std::string_view json_p } } -simdjson_inline simdjson_result> document::at_path_with_wildcard(std::string_view json_path) noexcept { - rewind(); // Rewind the document each time at_path_with_wildcard is called +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code document::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { + rewind(); // Rewind the document each time for_each_at_path_with_wildcard is called if (json_path.empty()) { return INVALID_JSON_POINTER; } @@ -152276,9 +153244,9 @@ simdjson_inline simdjson_result> document::at_path_with_wildc SIMDJSON_TRY(type().get(t)); switch (t) { case json_type::array: - return (*this).get_array().at_path_with_wildcard(json_path); + return (*this).get_array().for_each_at_path_with_wildcard(json_path, std::forward(callback)); case json_type::object: - return (*this).get_object().at_path_with_wildcard(json_path); + return (*this).get_object().for_each_at_path_with_wildcard(json_path, std::forward(callback)); default: return INVALID_JSON_POINTER; } @@ -152615,9 +153583,15 @@ simdjson_inline simdjson_result simdjson_result> simdjson_result::at_path_with_wildcard(std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code simdjson_result::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); + return first.for_each_at_path_with_wildcard(json_path, std::forward(callback)); } #if SIMDJSON_STATIC_REFLECTION @@ -152728,7 +153702,13 @@ simdjson_inline simdjson_result document_reference::get_number() noexcep simdjson_inline simdjson_result document_reference::raw_json_token() noexcept { return doc->raw_json_token(); } simdjson_inline simdjson_result document_reference::at_pointer(std::string_view json_pointer) noexcept { return doc->at_pointer(json_pointer); } simdjson_inline simdjson_result document_reference::at_path(std::string_view json_path) noexcept { return doc->at_path(json_path); } -simdjson_inline simdjson_result> document_reference::at_path_with_wildcard(std::string_view json_path) noexcept { return doc->at_path_with_wildcard(json_path); } +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code document_reference::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { return doc->for_each_at_path_with_wildcard(json_path, std::forward(callback)); } simdjson_inline simdjson_result document_reference::raw_json() noexcept { return doc->raw_json();} simdjson_inline document_reference::operator document&() const noexcept { return *doc; } #if SIMDJSON_SUPPORTS_CONCEPTS && SIMDJSON_STATIC_REFLECTION @@ -152993,11 +153973,17 @@ simdjson_inline simdjson_result simdjson_result> simdjson_result::at_path_with_wildcard(std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code simdjson_result::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); + return first.for_each_at_path_with_wildcard(json_path, std::forward(callback)); } #if SIMDJSON_STATIC_REFLECTION template @@ -153371,7 +154357,7 @@ simdjson_inline std::string_view document_stream::iterator::source() const noexc // TODO: We could remove trailing whitespaces // This returns a string spanning from start of value to the beginning of the next document (excluded) { - auto next_index = stream->parser->implementation->structural_indexes[++cur_struct_index]; + auto next_index = stream->batch_start + stream->parser->implementation->structural_indexes[++cur_struct_index]; // normally the length would be next_index - current_index() - 1, except for the last document size_t svlen = next_index - current_index(); const char *start = reinterpret_cast(stream->buf) + current_index(); @@ -154575,46 +155561,44 @@ inline simdjson_result object::at_path(std::string_view json_path) noexce return at_pointer(json_pointer); } -inline simdjson_result> object::at_path_with_wildcard(std::string_view json_path) noexcept { - std::vector result; - +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +inline error_code object::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { auto result_pair = get_next_key_and_json_path(json_path); std::string_view key = result_pair.first; std::string_view remaining_path = result_pair.second; + // Empty key means the path segment could not be parsed (including malformed + // bracket-quoted keys). Match DOM at_path_with_wildcard and avoid treating + // "no progress" as a recursive lookup of "". + if (key.empty()) { + return INVALID_JSON_POINTER; + } // Handle when its the case for wildcard if (key == "*") { // Loop through each field in the object for (auto field : *this) { value val; SIMDJSON_TRY(field.value().get(val)); - if (remaining_path.empty()) { - result.push_back(std::move(val)); + callback(val); } else { - auto nested_result = val.at_path_with_wildcard(remaining_path); - - if (nested_result.error()) { - return nested_result.error(); - } - // Extract and append all nested matches to our result - std::vector nested_vec; - SIMDJSON_TRY(std::move(nested_result).get(nested_vec)); - - result.insert(result.end(), - std::make_move_iterator(nested_vec.begin()), - std::make_move_iterator(nested_vec.end())); + SIMDJSON_TRY(val.for_each_at_path_with_wildcard(remaining_path, callback)); } } - return result; + return SUCCESS; } else { value val; SIMDJSON_TRY(find_field(key).get(val)); if (remaining_path.empty()) { - result.push_back(std::move(val)); - return result; + callback(val); + return SUCCESS; } else { - return val.at_path_with_wildcard(remaining_path); + return val.for_each_at_path_with_wildcard(remaining_path, callback); } } } @@ -154745,9 +155729,15 @@ simdjson_inline simdjson_result simdjson_result> simdjson_result::at_path_with_wildcard(std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code simdjson_result::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); + return first.for_each_at_path_with_wildcard(json_path, std::forward(callback)); } inline simdjson_result simdjson_result::reset() noexcept { @@ -154952,7 +155942,7 @@ simdjson_inline parser::parser(size_t max_capacity) noexcept simdjson_warn_unused simdjson_inline error_code parser::allocate(size_t new_capacity, size_t new_max_depth) noexcept { if (new_capacity > max_capacity()) { return CAPACITY; } - if (string_buf && new_capacity == capacity() && new_max_depth == max_depth()) { return SUCCESS; } + if (string_buf && new_capacity <= capacity() && new_max_depth <= max_depth()) { return SUCCESS; } // string_capacity copied from document::allocate _capacity = 0; @@ -155128,7 +156118,7 @@ simdjson_inline simdjson_warn_unused ondemand::parser& parser::get_parser() { return *parser::get_parser_instance(); } -simdjson_inline bool release_parser() { +simdjson_inline bool parser::release_parser() { auto &parser_instance = parser::get_threadlocal_parser_if_exists(); if (parser_instance) { parser_instance.reset(); @@ -155245,13 +156235,9 @@ simdjson_inline bool raw_json_string::unsafe_is_equal(std::string_view target) c if(target.size() <= SIMDJSON_PADDING) { return (raw()[target.size()] == '"') && !memcmp(raw(), target.data(), target.size()); } - const char * r{raw()}; - size_t pos{0}; - for(;pos < target.size();pos++) { - if(r[pos] != target[pos]) { return false; } - } - if(r[pos] != '"') { return false; } - return true; + // Past SIMDJSON_PADDING we can no longer rely on the padding to keep the + // comparison in bounds, so we must stop at the quote terminating the key. + return is_equal(target); } simdjson_inline bool raw_json_string::is_equal(std::string_view target) const noexcept { @@ -159802,13 +160788,21 @@ class value { simdjson_inline simdjson_result at_path(std::string_view at_path) noexcept; /** - * Get all values matching the given JSONPath expression with wildcard support. + * Call the provided callback for each value matching the given JSONPath + * expression with wildcard support. * Supports wildcard character (*) for arrays or ".*" for objects. * * @param json_path JSONPath expression with wildcards - * @return Vector of values matching the wildcard pattern + * @param callback Function called for each matching value + * @return error_code indicating success or failure */ - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; protected: /** @@ -160001,9 +160995,23 @@ struct simdjson_result : public lasx::implementation_simd simdjson_inline simdjson_result current_depth() const noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; }; +// Forward-declare explicit specializations so MSVC /permissive- sees them before +// any template instantiation that would resolve element.get(val) to the primary. +template<> simdjson_inline error_code +simdjson_result::get( + lasx::ondemand::value &out) noexcept; +template<> simdjson_inline simdjson_result +simdjson_result::get() noexcept; + } // namespace simdjson #endif // SIMDJSON_GENERIC_ONDEMAND_VALUE_H @@ -161342,8 +162350,6 @@ class parser { static simdjson_inline bool release_parser(); private: - friend bool release_parser(); - friend ondemand::parser& get_parser(); /** Get the thread-local parser instance, allocates it if needed */ static simdjson_inline simdjson_warn_unused std::unique_ptr& get_parser_instance(); /** Get the thread-local parser instance, it might be null */ @@ -161506,13 +162512,21 @@ class array { inline simdjson_result at_path(std::string_view json_path) noexcept; /** - * Get all values matching the given JSONPath expression with wildcard support. + * Call the provided callback for each value matching the given JSONPath + * expression with wildcard support. * Supports wildcard patterns like "[*]" to match all array elements. * * @param json_path JSONPath expression with wildcards - * @return Vector of values matching the wildcard pattern + * @param callback Function called for each matching value + * @return error_code indicating success or failure */ - inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; /** * Consumes the array and returns a string_view instance corresponding to the @@ -161636,7 +162650,13 @@ struct simdjson_result : public lasx::implementation_simd simdjson_inline simdjson_result at(size_t index) noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; simdjson_inline simdjson_result raw_json() noexcept; #if SIMDJSON_SUPPORTS_CONCEPTS // TODO: move this code into object-inl.h @@ -162536,21 +163556,24 @@ class document { simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; /** - * Get all values matching the given JSONPath expression with wildcard support. + * Call the provided callback for each value matching the given JSONPath + * expression with wildcard support. * * Supports wildcard patterns like "$.array[*]" or "$.object.*" to match multiple elements. * - * This method materializes all matching values into a vector. * The document will be consumed after this call. * * @param json_path JSONPath expression with wildcards - * @return Vector of values matching the wildcard pattern, or: - * - INVALID_JSON_POINTER if the JSONPath cannot be parsed - * - NO_SUCH_FIELD if a field does not exist - * - INDEX_OUT_OF_BOUNDS if an array index is out of bounds - * - INCORRECT_TYPE if path traversal encounters wrong type + * @param callback Function called for each matching value + * @return error_code indicating success or failure */ - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; /** * Consumes the document and returns a string_view instance corresponding to the @@ -162771,7 +163794,13 @@ class document_reference { simdjson_inline simdjson_result raw_json_token() noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; private: document *doc{nullptr}; @@ -162857,7 +163886,13 @@ struct simdjson_result : public lasx::implementation_s simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; #if SIMDJSON_STATIC_REFLECTION template requires(std::is_class_v && (sizeof...(FieldNames) > 0)) @@ -162942,7 +163977,13 @@ struct simdjson_result : public lasx::implem simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; #if SIMDJSON_STATIC_REFLECTION template requires(std::is_class_v && (sizeof...(FieldNames) > 0)) @@ -163075,6 +164116,18 @@ class document_stream { * } * size_t truncated = stream.truncated_bytes(); * + * IMPORTANT: this value is only meaningful under the conditions below. It is + * computed from stage-1 bookkeeping, and outside these conditions it is not + * merely imprecise, it is arbitrary -- it can exceed size_in_bytes() or wrap + * around to a huge value. Check it only when both of the following hold: + * + * - you iterated all the way to the end of the stream; + * - no document reported an error. Iteration stops at the first failed + * document, which can leave the bookkeeping from a mid-stream batch. + * + * If you need to know about a truncated tail outside those conditions, track + * it yourself from the last successful document (see iterator::current_index() + * and iterator::source()). */ inline size_t truncated_bytes() const noexcept; @@ -163595,13 +164648,21 @@ class object { inline simdjson_result at_path(std::string_view json_path) noexcept; /** - * Get all values matching the given JSONPath expression with wildcard support. + * Call the provided callback for each value matching the given JSONPath + * expression with wildcard support. * Supports wildcard patterns like ".*" to match all object fields. * * @param json_path JSONPath expression with wildcards - * @return Vector of values matching the wildcard pattern + * @param callback Function called for each matching value + * @return error_code indicating success or failure */ - inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; /** * Reset the iterator so that we are pointing back at the @@ -163751,7 +164812,13 @@ struct simdjson_result : public lasx::implementation_sim simdjson_inline simdjson_result operator[](std::string_view key) && noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; inline simdjson_result reset() noexcept; inline simdjson_result is_empty() noexcept; inline simdjson_result count_fields() & noexcept; @@ -164765,46 +165832,40 @@ inline simdjson_result array::at_path(std::string_view json_path) noexcep return at_pointer(json_pointer); } -inline simdjson_result> array::at_path_with_wildcard(std::string_view json_path) noexcept { - std::vector result; - +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +inline error_code array::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { auto result_pair = get_next_key_and_json_path(json_path); std::string_view key = result_pair.first; std::string_view remaining_path = result_pair.second; + // Empty key means the path segment could not be parsed (including malformed + // bracket-quoted keys). Without this check, an empty key is treated as index + // 0 and remaining_path may be unchanged, causing infinite recursion. + if (key.empty()) { + return INVALID_JSON_POINTER; + } // Wildcard case - if(key=="*"){ - for(auto element: *this){ - - if(element.error()){ - return element.error(); - } - - if(remaining_path.empty()){ - // Use value_unsafe() because we've already checked for errors above. - // The 'element' is a simdjson_result wrapper, and we need to extract - // the underlying value. value_unsafe() is safe here because error() returned false. - result.push_back(std::move(element).value_unsafe()); - - }else{ - auto nested_result = element.at_path_with_wildcard(remaining_path); - - if(nested_result.error()){ - return nested_result.error(); - } - // Same logic as above. - std::vector nested_matches = std::move(nested_result).value_unsafe(); - - result.insert(result.end(), - std::make_move_iterator(nested_matches.begin()), - std::make_move_iterator(nested_matches.end())); + if (key=="*"){ + for(auto element: *this) { + value val; + SIMDJSON_TRY(element.get(val)); + if (remaining_path.empty()) { + callback(val); + } else { + error_code err = element.for_each_at_path_with_wildcard(remaining_path, callback); + if(err) { return err; } } } - return result; - }else{ + return SUCCESS; + } else { // Specific index case in which we access the element at the given index - size_t idx=0; + size_t idx = 0; - for(char c:key){ + for (char c : key) { if(c < '0' || c > '9'){ return INVALID_JSON_POINTER; } @@ -164812,16 +165873,13 @@ inline simdjson_result> array::at_path_with_wildcard(std::str } auto element = at(idx); - - if(element.error()){ - return element.error(); - } - - if(remaining_path.empty()){ - result.push_back(std::move(element).value_unsafe()); - return result; - }else{ - return element.at_path_with_wildcard(remaining_path); + value val; + SIMDJSON_TRY(element.get(val)); + if (remaining_path.empty()){ + callback(val); + return SUCCESS; + } else { + return element.for_each_at_path_with_wildcard(remaining_path, callback); } } } @@ -164884,9 +165942,15 @@ simdjson_inline simdjson_result simdjson_result> simdjson_result::at_path_with_wildcard(std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code simdjson_result::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); + return first.for_each_at_path_with_wildcard(json_path, std::forward(callback)); } simdjson_inline simdjson_result simdjson_result::raw_json() noexcept { if (error()) { return error(); } @@ -165307,14 +166371,21 @@ simdjson_inline simdjson_result value::at_path(std::string_view json_path } } -inline simdjson_result> value::at_path_with_wildcard(std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +inline error_code value::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { + if (size_t(iter.depth()) >= iter.json_iter().parser->max_depth()) { return DEPTH_ERROR; } json_type t; SIMDJSON_TRY(type().get(t)); switch (t) { case json_type::array: - return (*this).get_array().at_path_with_wildcard(json_path); + return (*this).get_array().for_each_at_path_with_wildcard(json_path, std::forward(callback)); case json_type::object: - return (*this).get_object().at_path_with_wildcard(json_path); + return (*this).get_object().for_each_at_path_with_wildcard(json_path, std::forward(callback)); default: return INVALID_JSON_POINTER; } @@ -165578,12 +166649,18 @@ simdjson_inline simdjson_result simdjson_result> simdjson_result::at_path_with_wildcard( - std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +inline error_code simdjson_result::for_each_at_path_with_wildcard( + std::string_view json_path, Func&& callback) noexcept { if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); + return first.for_each_at_path_with_wildcard(json_path, std::forward(callback)); } } // namespace simdjson @@ -165959,8 +167036,14 @@ simdjson_inline simdjson_result document::at_path(std::string_view json_p } } -simdjson_inline simdjson_result> document::at_path_with_wildcard(std::string_view json_path) noexcept { - rewind(); // Rewind the document each time at_path_with_wildcard is called +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code document::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { + rewind(); // Rewind the document each time for_each_at_path_with_wildcard is called if (json_path.empty()) { return INVALID_JSON_POINTER; } @@ -165968,9 +167051,9 @@ simdjson_inline simdjson_result> document::at_path_with_wildc SIMDJSON_TRY(type().get(t)); switch (t) { case json_type::array: - return (*this).get_array().at_path_with_wildcard(json_path); + return (*this).get_array().for_each_at_path_with_wildcard(json_path, std::forward(callback)); case json_type::object: - return (*this).get_object().at_path_with_wildcard(json_path); + return (*this).get_object().for_each_at_path_with_wildcard(json_path, std::forward(callback)); default: return INVALID_JSON_POINTER; } @@ -166307,9 +167390,15 @@ simdjson_inline simdjson_result simdjson_result> simdjson_result::at_path_with_wildcard(std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code simdjson_result::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); + return first.for_each_at_path_with_wildcard(json_path, std::forward(callback)); } #if SIMDJSON_STATIC_REFLECTION @@ -166420,7 +167509,13 @@ simdjson_inline simdjson_result document_reference::get_number() noexcep simdjson_inline simdjson_result document_reference::raw_json_token() noexcept { return doc->raw_json_token(); } simdjson_inline simdjson_result document_reference::at_pointer(std::string_view json_pointer) noexcept { return doc->at_pointer(json_pointer); } simdjson_inline simdjson_result document_reference::at_path(std::string_view json_path) noexcept { return doc->at_path(json_path); } -simdjson_inline simdjson_result> document_reference::at_path_with_wildcard(std::string_view json_path) noexcept { return doc->at_path_with_wildcard(json_path); } +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code document_reference::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { return doc->for_each_at_path_with_wildcard(json_path, std::forward(callback)); } simdjson_inline simdjson_result document_reference::raw_json() noexcept { return doc->raw_json();} simdjson_inline document_reference::operator document&() const noexcept { return *doc; } #if SIMDJSON_SUPPORTS_CONCEPTS && SIMDJSON_STATIC_REFLECTION @@ -166685,11 +167780,17 @@ simdjson_inline simdjson_result simdjson_result> simdjson_result::at_path_with_wildcard(std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code simdjson_result::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); + return first.for_each_at_path_with_wildcard(json_path, std::forward(callback)); } #if SIMDJSON_STATIC_REFLECTION template @@ -167063,7 +168164,7 @@ simdjson_inline std::string_view document_stream::iterator::source() const noexc // TODO: We could remove trailing whitespaces // This returns a string spanning from start of value to the beginning of the next document (excluded) { - auto next_index = stream->parser->implementation->structural_indexes[++cur_struct_index]; + auto next_index = stream->batch_start + stream->parser->implementation->structural_indexes[++cur_struct_index]; // normally the length would be next_index - current_index() - 1, except for the last document size_t svlen = next_index - current_index(); const char *start = reinterpret_cast(stream->buf) + current_index(); @@ -168267,46 +169368,44 @@ inline simdjson_result object::at_path(std::string_view json_path) noexce return at_pointer(json_pointer); } -inline simdjson_result> object::at_path_with_wildcard(std::string_view json_path) noexcept { - std::vector result; - +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +inline error_code object::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { auto result_pair = get_next_key_and_json_path(json_path); std::string_view key = result_pair.first; std::string_view remaining_path = result_pair.second; + // Empty key means the path segment could not be parsed (including malformed + // bracket-quoted keys). Match DOM at_path_with_wildcard and avoid treating + // "no progress" as a recursive lookup of "". + if (key.empty()) { + return INVALID_JSON_POINTER; + } // Handle when its the case for wildcard if (key == "*") { // Loop through each field in the object for (auto field : *this) { value val; SIMDJSON_TRY(field.value().get(val)); - if (remaining_path.empty()) { - result.push_back(std::move(val)); + callback(val); } else { - auto nested_result = val.at_path_with_wildcard(remaining_path); - - if (nested_result.error()) { - return nested_result.error(); - } - // Extract and append all nested matches to our result - std::vector nested_vec; - SIMDJSON_TRY(std::move(nested_result).get(nested_vec)); - - result.insert(result.end(), - std::make_move_iterator(nested_vec.begin()), - std::make_move_iterator(nested_vec.end())); + SIMDJSON_TRY(val.for_each_at_path_with_wildcard(remaining_path, callback)); } } - return result; + return SUCCESS; } else { value val; SIMDJSON_TRY(find_field(key).get(val)); if (remaining_path.empty()) { - result.push_back(std::move(val)); - return result; + callback(val); + return SUCCESS; } else { - return val.at_path_with_wildcard(remaining_path); + return val.for_each_at_path_with_wildcard(remaining_path, callback); } } } @@ -168437,9 +169536,15 @@ simdjson_inline simdjson_result simdjson_result> simdjson_result::at_path_with_wildcard(std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code simdjson_result::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); + return first.for_each_at_path_with_wildcard(json_path, std::forward(callback)); } inline simdjson_result simdjson_result::reset() noexcept { @@ -168644,7 +169749,7 @@ simdjson_inline parser::parser(size_t max_capacity) noexcept simdjson_warn_unused simdjson_inline error_code parser::allocate(size_t new_capacity, size_t new_max_depth) noexcept { if (new_capacity > max_capacity()) { return CAPACITY; } - if (string_buf && new_capacity == capacity() && new_max_depth == max_depth()) { return SUCCESS; } + if (string_buf && new_capacity <= capacity() && new_max_depth <= max_depth()) { return SUCCESS; } // string_capacity copied from document::allocate _capacity = 0; @@ -168820,7 +169925,7 @@ simdjson_inline simdjson_warn_unused ondemand::parser& parser::get_parser() { return *parser::get_parser_instance(); } -simdjson_inline bool release_parser() { +simdjson_inline bool parser::release_parser() { auto &parser_instance = parser::get_threadlocal_parser_if_exists(); if (parser_instance) { parser_instance.reset(); @@ -168937,13 +170042,9 @@ simdjson_inline bool raw_json_string::unsafe_is_equal(std::string_view target) c if(target.size() <= SIMDJSON_PADDING) { return (raw()[target.size()] == '"') && !memcmp(raw(), target.data(), target.size()); } - const char * r{raw()}; - size_t pos{0}; - for(;pos < target.size();pos++) { - if(r[pos] != target[pos]) { return false; } - } - if(r[pos] != '"') { return false; } - return true; + // Past SIMDJSON_PADDING we can no longer rely on the padding to keep the + // comparison in bounds, so we must stop at the quote terminating the key. + return is_equal(target); } simdjson_inline bool raw_json_string::is_equal(std::string_view target) const noexcept { @@ -173498,13 +174599,21 @@ class value { simdjson_inline simdjson_result at_path(std::string_view at_path) noexcept; /** - * Get all values matching the given JSONPath expression with wildcard support. + * Call the provided callback for each value matching the given JSONPath + * expression with wildcard support. * Supports wildcard character (*) for arrays or ".*" for objects. * * @param json_path JSONPath expression with wildcards - * @return Vector of values matching the wildcard pattern + * @param callback Function called for each matching value + * @return error_code indicating success or failure */ - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; protected: /** @@ -173697,9 +174806,23 @@ struct simdjson_result : public rvv_vls::implementatio simdjson_inline simdjson_result current_depth() const noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; }; +// Forward-declare explicit specializations so MSVC /permissive- sees them before +// any template instantiation that would resolve element.get(val) to the primary. +template<> simdjson_inline error_code +simdjson_result::get( + rvv_vls::ondemand::value &out) noexcept; +template<> simdjson_inline simdjson_result +simdjson_result::get() noexcept; + } // namespace simdjson #endif // SIMDJSON_GENERIC_ONDEMAND_VALUE_H @@ -175038,8 +176161,6 @@ class parser { static simdjson_inline bool release_parser(); private: - friend bool release_parser(); - friend ondemand::parser& get_parser(); /** Get the thread-local parser instance, allocates it if needed */ static simdjson_inline simdjson_warn_unused std::unique_ptr& get_parser_instance(); /** Get the thread-local parser instance, it might be null */ @@ -175202,13 +176323,21 @@ class array { inline simdjson_result at_path(std::string_view json_path) noexcept; /** - * Get all values matching the given JSONPath expression with wildcard support. + * Call the provided callback for each value matching the given JSONPath + * expression with wildcard support. * Supports wildcard patterns like "[*]" to match all array elements. * * @param json_path JSONPath expression with wildcards - * @return Vector of values matching the wildcard pattern + * @param callback Function called for each matching value + * @return error_code indicating success or failure */ - inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; /** * Consumes the array and returns a string_view instance corresponding to the @@ -175332,7 +176461,13 @@ struct simdjson_result : public rvv_vls::implementatio simdjson_inline simdjson_result at(size_t index) noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; simdjson_inline simdjson_result raw_json() noexcept; #if SIMDJSON_SUPPORTS_CONCEPTS // TODO: move this code into object-inl.h @@ -176232,21 +177367,24 @@ class document { simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; /** - * Get all values matching the given JSONPath expression with wildcard support. + * Call the provided callback for each value matching the given JSONPath + * expression with wildcard support. * * Supports wildcard patterns like "$.array[*]" or "$.object.*" to match multiple elements. * - * This method materializes all matching values into a vector. * The document will be consumed after this call. * * @param json_path JSONPath expression with wildcards - * @return Vector of values matching the wildcard pattern, or: - * - INVALID_JSON_POINTER if the JSONPath cannot be parsed - * - NO_SUCH_FIELD if a field does not exist - * - INDEX_OUT_OF_BOUNDS if an array index is out of bounds - * - INCORRECT_TYPE if path traversal encounters wrong type + * @param callback Function called for each matching value + * @return error_code indicating success or failure */ - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; /** * Consumes the document and returns a string_view instance corresponding to the @@ -176467,7 +177605,13 @@ class document_reference { simdjson_inline simdjson_result raw_json_token() noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; private: document *doc{nullptr}; @@ -176553,7 +177697,13 @@ struct simdjson_result : public rvv_vls::implementa simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; #if SIMDJSON_STATIC_REFLECTION template requires(std::is_class_v && (sizeof...(FieldNames) > 0)) @@ -176638,7 +177788,13 @@ struct simdjson_result : public rvv_vls:: simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; #if SIMDJSON_STATIC_REFLECTION template requires(std::is_class_v && (sizeof...(FieldNames) > 0)) @@ -176771,6 +177927,18 @@ class document_stream { * } * size_t truncated = stream.truncated_bytes(); * + * IMPORTANT: this value is only meaningful under the conditions below. It is + * computed from stage-1 bookkeeping, and outside these conditions it is not + * merely imprecise, it is arbitrary -- it can exceed size_in_bytes() or wrap + * around to a huge value. Check it only when both of the following hold: + * + * - you iterated all the way to the end of the stream; + * - no document reported an error. Iteration stops at the first failed + * document, which can leave the bookkeeping from a mid-stream batch. + * + * If you need to know about a truncated tail outside those conditions, track + * it yourself from the last successful document (see iterator::current_index() + * and iterator::source()). */ inline size_t truncated_bytes() const noexcept; @@ -177291,13 +178459,21 @@ class object { inline simdjson_result at_path(std::string_view json_path) noexcept; /** - * Get all values matching the given JSONPath expression with wildcard support. + * Call the provided callback for each value matching the given JSONPath + * expression with wildcard support. * Supports wildcard patterns like ".*" to match all object fields. * * @param json_path JSONPath expression with wildcards - * @return Vector of values matching the wildcard pattern + * @param callback Function called for each matching value + * @return error_code indicating success or failure */ - inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; /** * Reset the iterator so that we are pointing back at the @@ -177447,7 +178623,13 @@ struct simdjson_result : public rvv_vls::implementati simdjson_inline simdjson_result operator[](std::string_view key) && noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(std::string_view json_path) noexcept; +#if SIMDJSON_SUPPORTS_CONCEPTS + template + requires std::invocable +#else + template +#endif + simdjson_inline error_code for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept; inline simdjson_result reset() noexcept; inline simdjson_result is_empty() noexcept; inline simdjson_result count_fields() & noexcept; @@ -178461,46 +179643,40 @@ inline simdjson_result array::at_path(std::string_view json_path) noexcep return at_pointer(json_pointer); } -inline simdjson_result> array::at_path_with_wildcard(std::string_view json_path) noexcept { - std::vector result; - +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +inline error_code array::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { auto result_pair = get_next_key_and_json_path(json_path); std::string_view key = result_pair.first; std::string_view remaining_path = result_pair.second; + // Empty key means the path segment could not be parsed (including malformed + // bracket-quoted keys). Without this check, an empty key is treated as index + // 0 and remaining_path may be unchanged, causing infinite recursion. + if (key.empty()) { + return INVALID_JSON_POINTER; + } // Wildcard case - if(key=="*"){ - for(auto element: *this){ - - if(element.error()){ - return element.error(); - } - - if(remaining_path.empty()){ - // Use value_unsafe() because we've already checked for errors above. - // The 'element' is a simdjson_result wrapper, and we need to extract - // the underlying value. value_unsafe() is safe here because error() returned false. - result.push_back(std::move(element).value_unsafe()); - - }else{ - auto nested_result = element.at_path_with_wildcard(remaining_path); - - if(nested_result.error()){ - return nested_result.error(); - } - // Same logic as above. - std::vector nested_matches = std::move(nested_result).value_unsafe(); - - result.insert(result.end(), - std::make_move_iterator(nested_matches.begin()), - std::make_move_iterator(nested_matches.end())); + if (key=="*"){ + for(auto element: *this) { + value val; + SIMDJSON_TRY(element.get(val)); + if (remaining_path.empty()) { + callback(val); + } else { + error_code err = element.for_each_at_path_with_wildcard(remaining_path, callback); + if(err) { return err; } } } - return result; - }else{ + return SUCCESS; + } else { // Specific index case in which we access the element at the given index - size_t idx=0; + size_t idx = 0; - for(char c:key){ + for (char c : key) { if(c < '0' || c > '9'){ return INVALID_JSON_POINTER; } @@ -178508,16 +179684,13 @@ inline simdjson_result> array::at_path_with_wildcard(std::str } auto element = at(idx); - - if(element.error()){ - return element.error(); - } - - if(remaining_path.empty()){ - result.push_back(std::move(element).value_unsafe()); - return result; - }else{ - return element.at_path_with_wildcard(remaining_path); + value val; + SIMDJSON_TRY(element.get(val)); + if (remaining_path.empty()){ + callback(val); + return SUCCESS; + } else { + return element.for_each_at_path_with_wildcard(remaining_path, callback); } } } @@ -178580,9 +179753,15 @@ simdjson_inline simdjson_result simdjson_result> simdjson_result::at_path_with_wildcard(std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code simdjson_result::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); + return first.for_each_at_path_with_wildcard(json_path, std::forward(callback)); } simdjson_inline simdjson_result simdjson_result::raw_json() noexcept { if (error()) { return error(); } @@ -179003,14 +180182,21 @@ simdjson_inline simdjson_result value::at_path(std::string_view json_path } } -inline simdjson_result> value::at_path_with_wildcard(std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +inline error_code value::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { + if (size_t(iter.depth()) >= iter.json_iter().parser->max_depth()) { return DEPTH_ERROR; } json_type t; SIMDJSON_TRY(type().get(t)); switch (t) { case json_type::array: - return (*this).get_array().at_path_with_wildcard(json_path); + return (*this).get_array().for_each_at_path_with_wildcard(json_path, std::forward(callback)); case json_type::object: - return (*this).get_object().at_path_with_wildcard(json_path); + return (*this).get_object().for_each_at_path_with_wildcard(json_path, std::forward(callback)); default: return INVALID_JSON_POINTER; } @@ -179274,12 +180460,18 @@ simdjson_inline simdjson_result simdjson_result> simdjson_result::at_path_with_wildcard( - std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +inline error_code simdjson_result::for_each_at_path_with_wildcard( + std::string_view json_path, Func&& callback) noexcept { if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); + return first.for_each_at_path_with_wildcard(json_path, std::forward(callback)); } } // namespace simdjson @@ -179655,8 +180847,14 @@ simdjson_inline simdjson_result document::at_path(std::string_view json_p } } -simdjson_inline simdjson_result> document::at_path_with_wildcard(std::string_view json_path) noexcept { - rewind(); // Rewind the document each time at_path_with_wildcard is called +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code document::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { + rewind(); // Rewind the document each time for_each_at_path_with_wildcard is called if (json_path.empty()) { return INVALID_JSON_POINTER; } @@ -179664,9 +180862,9 @@ simdjson_inline simdjson_result> document::at_path_with_wildc SIMDJSON_TRY(type().get(t)); switch (t) { case json_type::array: - return (*this).get_array().at_path_with_wildcard(json_path); + return (*this).get_array().for_each_at_path_with_wildcard(json_path, std::forward(callback)); case json_type::object: - return (*this).get_object().at_path_with_wildcard(json_path); + return (*this).get_object().for_each_at_path_with_wildcard(json_path, std::forward(callback)); default: return INVALID_JSON_POINTER; } @@ -180003,9 +181201,15 @@ simdjson_inline simdjson_result simdjson_result> simdjson_result::at_path_with_wildcard(std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code simdjson_result::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); + return first.for_each_at_path_with_wildcard(json_path, std::forward(callback)); } #if SIMDJSON_STATIC_REFLECTION @@ -180116,7 +181320,13 @@ simdjson_inline simdjson_result document_reference::get_number() noexcep simdjson_inline simdjson_result document_reference::raw_json_token() noexcept { return doc->raw_json_token(); } simdjson_inline simdjson_result document_reference::at_pointer(std::string_view json_pointer) noexcept { return doc->at_pointer(json_pointer); } simdjson_inline simdjson_result document_reference::at_path(std::string_view json_path) noexcept { return doc->at_path(json_path); } -simdjson_inline simdjson_result> document_reference::at_path_with_wildcard(std::string_view json_path) noexcept { return doc->at_path_with_wildcard(json_path); } +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code document_reference::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { return doc->for_each_at_path_with_wildcard(json_path, std::forward(callback)); } simdjson_inline simdjson_result document_reference::raw_json() noexcept { return doc->raw_json();} simdjson_inline document_reference::operator document&() const noexcept { return *doc; } #if SIMDJSON_SUPPORTS_CONCEPTS && SIMDJSON_STATIC_REFLECTION @@ -180381,11 +181591,17 @@ simdjson_inline simdjson_result simdjson_result> simdjson_result::at_path_with_wildcard(std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code simdjson_result::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); + return first.for_each_at_path_with_wildcard(json_path, std::forward(callback)); } #if SIMDJSON_STATIC_REFLECTION template @@ -180759,7 +181975,7 @@ simdjson_inline std::string_view document_stream::iterator::source() const noexc // TODO: We could remove trailing whitespaces // This returns a string spanning from start of value to the beginning of the next document (excluded) { - auto next_index = stream->parser->implementation->structural_indexes[++cur_struct_index]; + auto next_index = stream->batch_start + stream->parser->implementation->structural_indexes[++cur_struct_index]; // normally the length would be next_index - current_index() - 1, except for the last document size_t svlen = next_index - current_index(); const char *start = reinterpret_cast(stream->buf) + current_index(); @@ -181963,46 +183179,44 @@ inline simdjson_result object::at_path(std::string_view json_path) noexce return at_pointer(json_pointer); } -inline simdjson_result> object::at_path_with_wildcard(std::string_view json_path) noexcept { - std::vector result; - +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +inline error_code object::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { auto result_pair = get_next_key_and_json_path(json_path); std::string_view key = result_pair.first; std::string_view remaining_path = result_pair.second; + // Empty key means the path segment could not be parsed (including malformed + // bracket-quoted keys). Match DOM at_path_with_wildcard and avoid treating + // "no progress" as a recursive lookup of "". + if (key.empty()) { + return INVALID_JSON_POINTER; + } // Handle when its the case for wildcard if (key == "*") { // Loop through each field in the object for (auto field : *this) { value val; SIMDJSON_TRY(field.value().get(val)); - if (remaining_path.empty()) { - result.push_back(std::move(val)); + callback(val); } else { - auto nested_result = val.at_path_with_wildcard(remaining_path); - - if (nested_result.error()) { - return nested_result.error(); - } - // Extract and append all nested matches to our result - std::vector nested_vec; - SIMDJSON_TRY(std::move(nested_result).get(nested_vec)); - - result.insert(result.end(), - std::make_move_iterator(nested_vec.begin()), - std::make_move_iterator(nested_vec.end())); + SIMDJSON_TRY(val.for_each_at_path_with_wildcard(remaining_path, callback)); } } - return result; + return SUCCESS; } else { value val; SIMDJSON_TRY(find_field(key).get(val)); if (remaining_path.empty()) { - result.push_back(std::move(val)); - return result; + callback(val); + return SUCCESS; } else { - return val.at_path_with_wildcard(remaining_path); + return val.for_each_at_path_with_wildcard(remaining_path, callback); } } } @@ -182133,9 +183347,15 @@ simdjson_inline simdjson_result simdjson_result> simdjson_result::at_path_with_wildcard(std::string_view json_path) noexcept { +#if SIMDJSON_SUPPORTS_CONCEPTS +template + requires std::invocable +#else +template +#endif +simdjson_inline error_code simdjson_result::for_each_at_path_with_wildcard(std::string_view json_path, Func&& callback) noexcept { if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); + return first.for_each_at_path_with_wildcard(json_path, std::forward(callback)); } inline simdjson_result simdjson_result::reset() noexcept { @@ -182340,7 +183560,7 @@ simdjson_inline parser::parser(size_t max_capacity) noexcept simdjson_warn_unused simdjson_inline error_code parser::allocate(size_t new_capacity, size_t new_max_depth) noexcept { if (new_capacity > max_capacity()) { return CAPACITY; } - if (string_buf && new_capacity == capacity() && new_max_depth == max_depth()) { return SUCCESS; } + if (string_buf && new_capacity <= capacity() && new_max_depth <= max_depth()) { return SUCCESS; } // string_capacity copied from document::allocate _capacity = 0; @@ -182516,7 +183736,7 @@ simdjson_inline simdjson_warn_unused ondemand::parser& parser::get_parser() { return *parser::get_parser_instance(); } -simdjson_inline bool release_parser() { +simdjson_inline bool parser::release_parser() { auto &parser_instance = parser::get_threadlocal_parser_if_exists(); if (parser_instance) { parser_instance.reset(); @@ -182633,13 +183853,9 @@ simdjson_inline bool raw_json_string::unsafe_is_equal(std::string_view target) c if(target.size() <= SIMDJSON_PADDING) { return (raw()[target.size()] == '"') && !memcmp(raw(), target.data(), target.size()); } - const char * r{raw()}; - size_t pos{0}; - for(;pos < target.size();pos++) { - if(r[pos] != target[pos]) { return false; } - } - if(r[pos] != '"') { return false; } - return true; + // Past SIMDJSON_PADDING we can no longer rely on the padding to keep the + // comparison in bounds, so we must stop at the quote terminating the key. + return is_equal(target); } simdjson_inline bool raw_json_string::is_equal(std::string_view target) const noexcept {