Fix JSON5 number and escape grammar conformance

.github/workflows/ci.yml

@@ -71,6 +71,9 @@ jobs:
       - name: Run JSON5 tests suite
         run: python scripts/run-tests.py
 
+      - name: Run JSON5 conformance checks
+        run: python scripts/run-conformance-test.py
+
       - name: Run "JSON is a Minefield" suite
         run: python scripts/run-minefield-test.py
 

CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+**Unreleased**
+
+* Fix number and string-escape grammar conformance in the parser: reject
+  numeric underscores, a leading `.` without a fractional digit, doubled or
+  trailing dots, stray signs and empty exponents, and `\0` followed by a digit;
+  accept `0.e1` and parse out-of-range magnitudes to `±Infinity` per ECMAScript
+  `ToNumber` (by gaoflow, [#153](https://github.com/Kijewski/pyjson5/pull/153))
+
 **2.0.1 (2026-05-15)**
 
 * Support Python free-threaded builds (PEP 703). The extension now opts into

Makefile

@@ -57,4 +57,5 @@ test: wheel
 	pip install --force dist/pyjson5-*.whl
 	python scripts/run-minefield-test.py
 	python scripts/run-tests.py
+	python scripts/run-conformance-test.py
 	python scripts/run-threaded-test.py

scripts/run-conformance-test.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+
+# Value-level conformance checks for the JSON5 number and string-escape
+# grammar. The expected results follow the JSON5 spec and match the reference
+# implementation (json5 npm 2.2.3), stdlib json and dpranke's json5 where those
+# apply. run-tests.py only checks accept/reject; this pins the parsed values.
+
+from math import inf, isnan, nan
+
+from pyjson5 import Json5Exception, decode
+
+THROW = object()
+
+CASES = [
+    # JSON5 has no numeric separators; '_' is not part of any number
+    ("1000", 1000), ("1_000", THROW), ("1__0", THROW), ("1_", THROW),
+    ("10_", THROW), ("1.5_", THROW), ("1e3_", THROW), ("-1_0", THROW),
+    ("0x1_0", THROW), ("0x_F", THROW), ("0x10_", THROW), ("1_e3", THROW),
+    # a leading '.' needs a fractional digit before any exponent or the end
+    (".5", 0.5), (".0", 0.0), ("-.5", -0.5), ("+.25", 0.25),
+    (".e5", THROW), (".e+5", THROW), (".", THROW),
+    # at most one '.', never after the exponent, no doubled/trailing dots
+    ("5.", 5.0), ("0.", 0.0), ("1.e5", 100000.0), ("0.e1", 0.0),
+    ("10.e-2", 0.1), ("5.8.", THROW), (".5.", THROW), ("2e400.", THROW),
+    # a sign is only valid at the start (consumed) or right after 'e'/'E'
+    ("+7", 7), ("++7", THROW), ("+-.4", THROW), ("1e+5", 100000.0),
+    ("1e-5", 1e-05),
+    # the exponent must have at least one digit
+    ("0e", THROW), ("0e+", THROW), ("1e", THROW), ("1e+", THROW), ("0e5", 0.0),
+    # overflow follows ECMAScript ToNumber (-> +/-Infinity); underflow -> 0
+    ("1e999", inf), ("-1e999", -inf), ("1e400", inf), ("9e999", inf),
+    ("1e308", 1e308), ("1e-999", 0.0),
+    # hexadecimal and leading-zero integers
+    ("0xFF", 255), ("0x10", 16), ("-0xff", -255), ("0x", THROW),
+    ("01", THROW), ("007", THROW), ("00", THROW),
+    # \0 is NUL only when not followed by a decimal digit (no legacy octal)
+    (r'"\0"', "\x00"), (r'"\0a"', "\x00a"), (r'"\09"', THROW),
+    (r'"\07"', THROW), (r'"\00"', THROW), (r"'\09'", THROW),
+    # unrelated escapes and literals stay valid
+    (r'"\x41"', "A"), (r'"A"', "A"), ("Infinity", inf), ("NaN", nan),
+]
+
+
+def check():
+    bad = 0
+    for src, expected in CASES:
+        try:
+            got = decode(src)
+            threw = False
+        except Json5Exception:
+            got = None
+            threw = True
+
+        if expected is THROW:
+            ok = threw
+        elif threw:
+            ok = False
+        elif isinstance(expected, float) and isnan(expected):
+            ok = isinstance(got, float) and isnan(got)
+        else:
+            ok = type(got) is type(expected) and got == expected
+
+        if not ok:
+            bad += 1
+            outcome = "was rejected" if threw else f"gave {got!r}"
+            want = "to be rejected" if expected is THROW else repr(expected)
+            print(f"BAD {src!r}: {outcome}, expected {want}")
+
+    return bad
+
+
+if __name__ == "__main__":
+    bad = check()
+    print(f"\n{len(CASES) - bad} of {len(CASES)} conformance cases passed")
+    raise SystemExit(1 if bad else 0)

src/_decoder.pyx

@@ -141,6 +141,7 @@ cdef int32_t _get_escaped_unicode_maybe_surrogate(ReaderRef reader, Py_ssize_t s
 cdef int32_t _get_escape_sequence(ReaderRef reader,
                                   Py_ssize_t start) except 0x7ffffff:
     cdef uint32_t c0
+    cdef int32_t c1
 
     c0 = _reader_get(reader)
     if expect(not _reader_good(reader), False):
@@ -159,6 +160,11 @@ cdef int32_t _get_escape_sequence(ReaderRef reader,
     elif c0 == b'v':
         return 0x000b
     elif c0 == b'0':
+        # \0 is NUL only if not followed by a decimal digit; \00..\09 would be
+        # a legacy octal escape, which JSON5/ECMAScript forbids.
+        c1 = _reader_peek(reader)
+        if expect(_is_decimal(<uint32_t> c1), False):
+            _raise_expected_s('escape sequence', start, <uint32_t> c1)
         return 0x0000
     elif c0 == b'x':
         return _get_hex_character(reader, 2)
@@ -250,6 +256,13 @@ cdef object _decode_double(StackHeapString[char] &buf, Py_ssize_t start):
     if end_of_double != NULL and end_of_double[0] == b'\0':
         return PyFloat_FromDouble(d0)
 
+    # fast_double_parser rejects out-of-range magnitudes (1e999 -> Infinity)
+    # and 0.e1-style tokens; ECMAScript ToNumber accepts both, so fall back.
+    try:
+        return float(<bytes> buf.data())
+    except ValueError:
+        pass
+
     _raise_unclosed('NumericLiteral', start)
 
 
@@ -272,7 +285,7 @@ cdef object _decode_number_leading_zero(ReaderRef reader, StackHeapString[char]
             c0 = _reader_get(reader)
             if _is_hexadecimal(c0):
                 buf.push_back(<char> <unsigned char> c0)
-            elif c0 != b'_':
+            else:
                 c1 = cast_to_int32(c0)
                 break
 
@@ -283,90 +296,61 @@ cdef object _decode_number_leading_zero(ReaderRef reader, StackHeapString[char]
             return PyLong_FromString(buf.data(), NULL, 16)
         except Exception:
             _raise_unclosed('NumericLiteral', start)
-    elif c0 == b'.':
+    elif c0 == b'.' or _is_e(c0):
+        # a leading zero is the integer part of "0.xxx" / "0exxx"
         buf.push_back(b'0')
-        buf.push_back(b'.')
-
-        while True:
-            if not _reader_good(reader):
-                c1 = -1
-                break
-
-            c0 = _reader_get(reader)
-            if _is_in_float_representation(c0):
-                buf.push_back(<char> <unsigned char> c0)
-            elif c0 != b'_':
-                c1 = cast_to_int32(c0)
-                break
-
-        c_in_out[0] = c1
-
-        if buf.data()[buf.size() - 1] == b'.':
-            (<char*> buf.data())[buf.size() - 1] = b'\0'
-        else:
-            buf.push_back(b'\0')
-
-        return _decode_double(buf, start)
-    elif _is_e(c0):
-        while True:
-            if not _reader_good(reader):
-                c1 = -1
-                break
-
-            c0 = _reader_get(reader)
-            if _is_in_float_representation(c0):
-                pass
-            elif c0 == b'_':
-                pass
-            else:
-                c1 = cast_to_int32(c0)
-                break
-
-        c_in_out[0] = c1
-        return 0.0
+        c_in_out[0] = cast_to_int32(c0)
+        return _decode_number_any(reader, buf, c_in_out, start, True)
     else:
         c1 = cast_to_int32(c0)
         c_in_out[0] = c1
         return 0
 
 
 cdef object _decode_number_any(ReaderRef reader, StackHeapString[char] &buf,
-                               int32_t *c_in_out, Py_ssize_t start):
+                               int32_t *c_in_out, Py_ssize_t start,
+                               boolean have_int_part=False):
     cdef uint32_t c0
     cdef int32_t c1
-    cdef boolean is_float = False
-    cdef boolean was_point = False
-    cdef boolean leading_point = False
+    cdef uint32_t prev = 0
+    cdef boolean is_float = have_int_part
+    cdef boolean seen_dot = False
+    cdef boolean seen_exp = False
+    cdef boolean need_frac_digit = False
 
     c1 = c_in_out[0]
     c0 = cast_to_uint32(c1)
 
-    if c0 == b'.':
+    if not have_int_part and c0 == b'.':
+        # a leading '.' has no integer part; synthesize a zero and require a
+        # fractional digit ('.e5' and a lone '.' are both invalid)
         buf.push_back(b'0')
         is_float = True
-        leading_point = True
+        need_frac_digit = True
 
     while True:
         if _is_decimal(c0):
-            pass
-        elif _is_in_float_representation(c0):
+            if not seen_exp:
+                need_frac_digit = False
+        elif c0 == b'.':
+            if expect(seen_dot or seen_exp, False):
+                _raise_unclosed('NumericLiteral', start)
+            seen_dot = True
+            is_float = True
+        elif _is_e(c0):
+            if expect(seen_exp or need_frac_digit, False):
+                _raise_unclosed('NumericLiteral', start)
+            seen_exp = True
             is_float = True
-        elif c0 != b'_':
+        elif c0 == b'+' or c0 == b'-':
+            if expect(not _is_e(prev), False):  # a sign is only valid after e/E
+                _raise_unclosed('NumericLiteral', start)
+        else:
             c1 = cast_to_int32(c0)
             break
 
-        if c0 == b'_':
-            pass
-        elif c0 != b'.':
-            if was_point:
-                was_point = False
-                if not _is_e(c0):
-                    buf.push_back(b'.')
-            buf.push_back(<char> <unsigned char> c0)
-        elif not was_point:
-            was_point = True
-        else:
-            _raise_unclosed('NumericLiteral', start)
+        buf.push_back(<char> <unsigned char> c0)
+        prev = c0
 
         if not _reader_good(reader):
             c1 = -1
@@ -376,7 +360,7 @@ cdef object _decode_number_any(ReaderRef reader, StackHeapString[char] &buf,
 
     c_in_out[0] = c1
 
-    if leading_point and buf.size() == 1:  # single '.'
+    if expect(need_frac_digit, False):
         _raise_unclosed('NumericLiteral', start)
 
     buf.push_back(b'\0')

src/_reader_callback.pyx

@@ -48,3 +48,10 @@ cdef int32_t _reader_Callback_good(ReaderCallbackRef self) except -1:
     self.lookahead = c
 
     return True
+
+
+cdef inline int32_t _reader_Callback_peek(ReaderCallbackRef self) except -2:
+    if not _reader_Callback_good(self):
+        return -1
+
+    return self.lookahead

src/_reader_ucs.pyx

@@ -86,3 +86,12 @@ cdef inline uint32_t _reader_utf8_get(ReaderUCSRef self) noexcept nogil:
         c0 = (c0 << 6) | (_reader_ucs_get(self) & 0b00_111111)
 
     return c0
+
+
+cdef inline int32_t _reader_ucs_peek(ReaderUCSRef self) noexcept nogil:
+    # Next raw code unit without consuming, or -1 if exhausted. For UTF-8 this
+    # is the next byte, which is enough to test for a trailing ASCII digit.
+    if self.base.remaining <= 0:
+        return -1
+
+    return <int32_t> self.string[0]

src/_readers.pyx

@@ -39,3 +39,10 @@ cdef int32_t _reader_good(ReaderRef self) except -1:
         return _reader_ucs_good(self)
     elif ReaderRef is ReaderCallbackRef:
         return _reader_Callback_good(self)
+
+
+cdef int32_t _reader_peek(ReaderRef self) except -2:
+    if ReaderRef in ReaderUCSRef:
+        return _reader_ucs_peek(self)
+    elif ReaderRef is ReaderCallbackRef:
+        return _reader_Callback_peek(self)

src/_unicode.pyx

@@ -33,13 +33,3 @@ cdef inline boolean _is_hex(uint32_t c) nogil:
 
 cdef inline boolean _is_hexadecimal(uint32_t c) nogil:
     return _is_decimal(c) or _is_hex(c)
-
-cdef boolean _is_in_float_representation(uint32_t c) nogil:
-    if _is_decimal(c):
-        return True
-    if _is_e(c):
-        return True
-    elif c in b'.+-':
-        return True
-    else:
-        return False