From f7050c919e4716e41e35838a76859f4a2b8ba67b Mon Sep 17 00:00:00 2001 From: Vincent Gao Date: Mon, 20 Jul 2026 00:50:34 +0200 Subject: [PATCH] Fix JSON5 number and escape grammar conformance Tighten the number-literal scanner and string-escape handling in src/_decoder.pyx to match the JSON5 grammar (verified against the json5 reference implementation): - reject numeric underscores, a leading '.' without a fractional digit, doubled or trailing dots, dots inside the exponent, stray/doubled signs and empty exponents - reject '\0' followed by a decimal digit (a forbidden legacy octal escape) - accept 0.e1 and parse out-of-range magnitudes to +/-Infinity per ECMAScript ToNumber The float scanner now tracks the grammar explicitly (at most one dot, one exponent, a sign only after e/E), and the leading-zero path reuses it, so float() is only reached to rescue tokens fast_double_parser rejects but ECMAScript accepts (overflow and 0.e1). Add scripts/run-conformance-test.py, a value-level table for the whole class, to CI and `make test`. --- .github/workflows/ci.yml | 3 + CHANGELOG.md | 8 +++ Makefile | 1 + scripts/run-conformance-test.py | 75 ++++++++++++++++++++++ src/_decoder.pyx | 110 ++++++++++++++------------------ src/_reader_callback.pyx | 7 ++ src/_reader_ucs.pyx | 9 +++ src/_readers.pyx | 7 ++ src/_unicode.pyx | 10 --- 9 files changed, 157 insertions(+), 73 deletions(-) create mode 100755 scripts/run-conformance-test.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bfef97c..d9ddb65 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,6 +71,9 @@ jobs: - name: Run JSON5 tests suite run: python scripts/run-tests.py + - name: Run JSON5 conformance checks + run: python scripts/run-conformance-test.py + - name: Run "JSON is a Minefield" suite run: python scripts/run-minefield-test.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 2431732..1e0cd4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +**Unreleased** + +* Fix number and string-escape grammar conformance in the parser: reject + numeric underscores, a leading `.` without a fractional digit, doubled or + trailing dots, stray signs and empty exponents, and `\0` followed by a digit; + accept `0.e1` and parse out-of-range magnitudes to `±Infinity` per ECMAScript + `ToNumber` (by gaoflow, [#153](https://github.com/Kijewski/pyjson5/pull/153)) + **2.0.1 (2026-05-15)** * Support Python free-threaded builds (PEP 703). The extension now opts into diff --git a/Makefile b/Makefile index 80a585c..1eeb1ca 100644 --- a/Makefile +++ b/Makefile @@ -57,4 +57,5 @@ test: wheel pip install --force dist/pyjson5-*.whl python scripts/run-minefield-test.py python scripts/run-tests.py + python scripts/run-conformance-test.py python scripts/run-threaded-test.py diff --git a/scripts/run-conformance-test.py b/scripts/run-conformance-test.py new file mode 100755 index 0000000..8aa4b56 --- /dev/null +++ b/scripts/run-conformance-test.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python + +# Value-level conformance checks for the JSON5 number and string-escape +# grammar. The expected results follow the JSON5 spec and match the reference +# implementation (json5 npm 2.2.3), stdlib json and dpranke's json5 where those +# apply. run-tests.py only checks accept/reject; this pins the parsed values. + +from math import inf, isnan, nan + +from pyjson5 import Json5Exception, decode + +THROW = object() + +CASES = [ + # JSON5 has no numeric separators; '_' is not part of any number + ("1000", 1000), ("1_000", THROW), ("1__0", THROW), ("1_", THROW), + ("10_", THROW), ("1.5_", THROW), ("1e3_", THROW), ("-1_0", THROW), + ("0x1_0", THROW), ("0x_F", THROW), ("0x10_", THROW), ("1_e3", THROW), + # a leading '.' needs a fractional digit before any exponent or the end + (".5", 0.5), (".0", 0.0), ("-.5", -0.5), ("+.25", 0.25), + (".e5", THROW), (".e+5", THROW), (".", THROW), + # at most one '.', never after the exponent, no doubled/trailing dots + ("5.", 5.0), ("0.", 0.0), ("1.e5", 100000.0), ("0.e1", 0.0), + ("10.e-2", 0.1), ("5.8.", THROW), (".5.", THROW), ("2e400.", THROW), + # a sign is only valid at the start (consumed) or right after 'e'/'E' + ("+7", 7), ("++7", THROW), ("+-.4", THROW), ("1e+5", 100000.0), + ("1e-5", 1e-05), + # the exponent must have at least one digit + ("0e", THROW), ("0e+", THROW), ("1e", THROW), ("1e+", THROW), ("0e5", 0.0), + # overflow follows ECMAScript ToNumber (-> +/-Infinity); underflow -> 0 + ("1e999", inf), ("-1e999", -inf), ("1e400", inf), ("9e999", inf), + ("1e308", 1e308), ("1e-999", 0.0), + # hexadecimal and leading-zero integers + ("0xFF", 255), ("0x10", 16), ("-0xff", -255), ("0x", THROW), + ("01", THROW), ("007", THROW), ("00", THROW), + # \0 is NUL only when not followed by a decimal digit (no legacy octal) + (r'"\0"', "\x00"), (r'"\0a"', "\x00a"), (r'"\09"', THROW), + (r'"\07"', THROW), (r'"\00"', THROW), (r"'\09'", THROW), + # unrelated escapes and literals stay valid + (r'"\x41"', "A"), (r'"A"', "A"), ("Infinity", inf), ("NaN", nan), +] + + +def check(): + bad = 0 + for src, expected in CASES: + try: + got = decode(src) + threw = False + except Json5Exception: + got = None + threw = True + + if expected is THROW: + ok = threw + elif threw: + ok = False + elif isinstance(expected, float) and isnan(expected): + ok = isinstance(got, float) and isnan(got) + else: + ok = type(got) is type(expected) and got == expected + + if not ok: + bad += 1 + outcome = "was rejected" if threw else f"gave {got!r}" + want = "to be rejected" if expected is THROW else repr(expected) + print(f"BAD {src!r}: {outcome}, expected {want}") + + return bad + + +if __name__ == "__main__": + bad = check() + print(f"\n{len(CASES) - bad} of {len(CASES)} conformance cases passed") + raise SystemExit(1 if bad else 0) diff --git a/src/_decoder.pyx b/src/_decoder.pyx index 6038863..9d1b98e 100644 --- a/src/_decoder.pyx +++ b/src/_decoder.pyx @@ -141,6 +141,7 @@ cdef int32_t _get_escaped_unicode_maybe_surrogate(ReaderRef reader, Py_ssize_t s cdef int32_t _get_escape_sequence(ReaderRef reader, Py_ssize_t start) except 0x7ffffff: cdef uint32_t c0 + cdef int32_t c1 c0 = _reader_get(reader) if expect(not _reader_good(reader), False): @@ -159,6 +160,11 @@ cdef int32_t _get_escape_sequence(ReaderRef reader, elif c0 == b'v': return 0x000b elif c0 == b'0': + # \0 is NUL only if not followed by a decimal digit; \00..\09 would be + # a legacy octal escape, which JSON5/ECMAScript forbids. + c1 = _reader_peek(reader) + if expect(_is_decimal( c1), False): + _raise_expected_s('escape sequence', start, c1) return 0x0000 elif c0 == b'x': return _get_hex_character(reader, 2) @@ -250,6 +256,13 @@ cdef object _decode_double(StackHeapString[char] &buf, Py_ssize_t start): if end_of_double != NULL and end_of_double[0] == b'\0': return PyFloat_FromDouble(d0) + # fast_double_parser rejects out-of-range magnitudes (1e999 -> Infinity) + # and 0.e1-style tokens; ECMAScript ToNumber accepts both, so fall back. + try: + return float( buf.data()) + except ValueError: + pass + _raise_unclosed('NumericLiteral', start) @@ -272,7 +285,7 @@ cdef object _decode_number_leading_zero(ReaderRef reader, StackHeapString[char] c0 = _reader_get(reader) if _is_hexadecimal(c0): buf.push_back( c0) - elif c0 != b'_': + else: c1 = cast_to_int32(c0) break @@ -283,47 +296,11 @@ cdef object _decode_number_leading_zero(ReaderRef reader, StackHeapString[char] return PyLong_FromString(buf.data(), NULL, 16) except Exception: _raise_unclosed('NumericLiteral', start) - elif c0 == b'.': + elif c0 == b'.' or _is_e(c0): + # a leading zero is the integer part of "0.xxx" / "0exxx" buf.push_back(b'0') - buf.push_back(b'.') - - while True: - if not _reader_good(reader): - c1 = -1 - break - - c0 = _reader_get(reader) - if _is_in_float_representation(c0): - buf.push_back( c0) - elif c0 != b'_': - c1 = cast_to_int32(c0) - break - - c_in_out[0] = c1 - - if buf.data()[buf.size() - 1] == b'.': - ( buf.data())[buf.size() - 1] = b'\0' - else: - buf.push_back(b'\0') - - return _decode_double(buf, start) - elif _is_e(c0): - while True: - if not _reader_good(reader): - c1 = -1 - break - - c0 = _reader_get(reader) - if _is_in_float_representation(c0): - pass - elif c0 == b'_': - pass - else: - c1 = cast_to_int32(c0) - break - - c_in_out[0] = c1 - return 0.0 + c_in_out[0] = cast_to_int32(c0) + return _decode_number_any(reader, buf, c_in_out, start, True) else: c1 = cast_to_int32(c0) c_in_out[0] = c1 @@ -331,42 +308,49 @@ cdef object _decode_number_leading_zero(ReaderRef reader, StackHeapString[char] cdef object _decode_number_any(ReaderRef reader, StackHeapString[char] &buf, - int32_t *c_in_out, Py_ssize_t start): + int32_t *c_in_out, Py_ssize_t start, + boolean have_int_part=False): cdef uint32_t c0 cdef int32_t c1 - cdef boolean is_float = False - cdef boolean was_point = False - cdef boolean leading_point = False + cdef uint32_t prev = 0 + cdef boolean is_float = have_int_part + cdef boolean seen_dot = False + cdef boolean seen_exp = False + cdef boolean need_frac_digit = False c1 = c_in_out[0] c0 = cast_to_uint32(c1) - if c0 == b'.': + if not have_int_part and c0 == b'.': + # a leading '.' has no integer part; synthesize a zero and require a + # fractional digit ('.e5' and a lone '.' are both invalid) buf.push_back(b'0') is_float = True - leading_point = True + need_frac_digit = True while True: if _is_decimal(c0): - pass - elif _is_in_float_representation(c0): + if not seen_exp: + need_frac_digit = False + elif c0 == b'.': + if expect(seen_dot or seen_exp, False): + _raise_unclosed('NumericLiteral', start) + seen_dot = True + is_float = True + elif _is_e(c0): + if expect(seen_exp or need_frac_digit, False): + _raise_unclosed('NumericLiteral', start) + seen_exp = True is_float = True - elif c0 != b'_': + elif c0 == b'+' or c0 == b'-': + if expect(not _is_e(prev), False): # a sign is only valid after e/E + _raise_unclosed('NumericLiteral', start) + else: c1 = cast_to_int32(c0) break - if c0 == b'_': - pass - elif c0 != b'.': - if was_point: - was_point = False - if not _is_e(c0): - buf.push_back(b'.') - buf.push_back( c0) - elif not was_point: - was_point = True - else: - _raise_unclosed('NumericLiteral', start) + buf.push_back( c0) + prev = c0 if not _reader_good(reader): c1 = -1 @@ -376,7 +360,7 @@ cdef object _decode_number_any(ReaderRef reader, StackHeapString[char] &buf, c_in_out[0] = c1 - if leading_point and buf.size() == 1: # single '.' + if expect(need_frac_digit, False): _raise_unclosed('NumericLiteral', start) buf.push_back(b'\0') diff --git a/src/_reader_callback.pyx b/src/_reader_callback.pyx index f99249c..b84bb30 100644 --- a/src/_reader_callback.pyx +++ b/src/_reader_callback.pyx @@ -48,3 +48,10 @@ cdef int32_t _reader_Callback_good(ReaderCallbackRef self) except -1: self.lookahead = c return True + + +cdef inline int32_t _reader_Callback_peek(ReaderCallbackRef self) except -2: + if not _reader_Callback_good(self): + return -1 + + return self.lookahead diff --git a/src/_reader_ucs.pyx b/src/_reader_ucs.pyx index 0f10134..38fd91e 100644 --- a/src/_reader_ucs.pyx +++ b/src/_reader_ucs.pyx @@ -86,3 +86,12 @@ cdef inline uint32_t _reader_utf8_get(ReaderUCSRef self) noexcept nogil: c0 = (c0 << 6) | (_reader_ucs_get(self) & 0b00_111111) return c0 + + +cdef inline int32_t _reader_ucs_peek(ReaderUCSRef self) noexcept nogil: + # Next raw code unit without consuming, or -1 if exhausted. For UTF-8 this + # is the next byte, which is enough to test for a trailing ASCII digit. + if self.base.remaining <= 0: + return -1 + + return self.string[0] diff --git a/src/_readers.pyx b/src/_readers.pyx index e0a910e..fd351a2 100644 --- a/src/_readers.pyx +++ b/src/_readers.pyx @@ -39,3 +39,10 @@ cdef int32_t _reader_good(ReaderRef self) except -1: return _reader_ucs_good(self) elif ReaderRef is ReaderCallbackRef: return _reader_Callback_good(self) + + +cdef int32_t _reader_peek(ReaderRef self) except -2: + if ReaderRef in ReaderUCSRef: + return _reader_ucs_peek(self) + elif ReaderRef is ReaderCallbackRef: + return _reader_Callback_peek(self) diff --git a/src/_unicode.pyx b/src/_unicode.pyx index 4aefc03..1abcf5a 100644 --- a/src/_unicode.pyx +++ b/src/_unicode.pyx @@ -33,13 +33,3 @@ cdef inline boolean _is_hex(uint32_t c) nogil: cdef inline boolean _is_hexadecimal(uint32_t c) nogil: return _is_decimal(c) or _is_hex(c) - -cdef boolean _is_in_float_representation(uint32_t c) nogil: - if _is_decimal(c): - return True - if _is_e(c): - return True - elif c in b'.+-': - return True - else: - return False