fix: distinguish malformed documents from incomplete json

src/index.test.ts

@@ -0,0 +1,28 @@
+import { describe, expect, it } from "vitest";
+import { MalformedJSON, parse } from "./index";
+
+describe("malformed JSON handling", () => {
+    it("throws for malformed numbers inside arrays", () => {
+        expect(() => parse("[1, .05, 2]")).toThrow(MalformedJSON);
+    });
+
+    it("throws for malformed numbers inside objects", () => {
+        expect(() => parse('{"a": .05, "b": 2}')).toThrow(MalformedJSON);
+    });
+
+    it("keeps parsing after an empty array with spaces", () => {
+        expect(parse('[{"id":1,"arr":["hello"]},{"id":2,"arr":[        ],"more":"yaya"},{"id":3,"arr":["!"]}]')).toEqual([
+            { id: 1, arr: ["hello"] },
+            { id: 2, arr: [], more: "yaya" },
+            { id: 3, arr: ["!"] },
+        ]);
+    });
+
+    it("keeps parsing after an object comma with spaces", () => {
+        expect(parse('{"a":1,   }')).toEqual({ a: 1 });
+    });
+
+    it("keeps parsing after an array comma with spaces", () => {
+        expect(parse("[1,   ]")).toEqual([1]);
+    });
+});

src/index.ts

@@ -20,7 +20,8 @@ function parseJSON(jsonString: string, allowPartial: number = Allow.ALL): any {
     if (!jsonString.trim()) {
         throw new Error(`${jsonString} is empty`);
     }
-    return _parseJSON(jsonString.trim(), allowPartial);
+    // Trailing whitespace may belong to an unfinished string value.
+    return _parseJSON(jsonString.trimStart(), allowPartial);
 }
 
 const _parseJSON = (jsonString: string, allow: number) => {
@@ -35,6 +36,8 @@ const _parseJSON = (jsonString: string, allow: number) => {
         throw new MalformedJSON(`${msg} at position ${index}`);
     };
 
+    const isPartialJSON = (e: unknown) => e instanceof PartialJSON;
+
     const parseAny: () => any = () => {
         skipBlank();
         if (index >= length) markPartialJSON("Unexpected end of input");
@@ -100,40 +103,54 @@ const _parseJSON = (jsonString: string, allow: number) => {
         try {
             while (jsonString[index] !== "}") {
                 skipBlank();
-                if (index >= length && Allow.OBJ & allow) return obj;
+                if (index >= length) markPartialJSON("Expected an object key");
+                if (jsonString[index] !== '"') throwMalformedError("Expected a quoted object key");
                 const key = parseStr();
                 skipBlank();
+                if (index >= length) markPartialJSON("Expected ':' after object key");
+                if (jsonString[index] !== ":") throwMalformedError("Expected ':' after object key");
                 index++; // skip colon
                 try {
                     const value = parseAny();
                     obj[key] = value;
                 } catch (e) {
-                    if (Allow.OBJ & allow) return obj;
-                    else throw e;
+                    if (Allow.OBJ & allow && isPartialJSON(e)) return obj;
+                    throw e;
                 }
                 skipBlank();
-                if (jsonString[index] === ",") index++; // skip comma
+                if (jsonString[index] === ",") {
+                    index++; // skip comma
+                    skipBlank();
+                } else if (index < length && jsonString[index] !== "}") {
+                    throwMalformedError("Expected ',' or '}' after object value");
+                }
             }
         } catch (e) {
+            if (!isPartialJSON(e)) throw e;
             if (Allow.OBJ & allow) return obj;
-            else markPartialJSON("Expected '}' at end of object");
+            markPartialJSON("Expected '}' at end of object");
         }
         index++; // skip final brace
         return obj;
     };
 
     const parseArr = () => {
         index++; // skip initial bracket
+        skipBlank();
         const arr = [];
         try {
             while (jsonString[index] !== "]") {
                 arr.push(parseAny());
                 skipBlank();
                 if (jsonString[index] === ",") {
                     index++; // skip comma
+                    skipBlank();
+                } else if (index < length && jsonString[index] !== "]") {
+                    throwMalformedError("Expected ',' or ']' after array value");
                 }
             }
         } catch (e) {
+            if (!isPartialJSON(e)) throw e;
             if (Allow.ARR & allow) {
                 return arr;
             }
@@ -144,44 +161,40 @@ const _parseJSON = (jsonString: string, allow: number) => {
     };
 
     const parseNum = () => {
-        if (index === 0) {
-            if (jsonString === "-") throwMalformedError("Not sure what '-' is");
-            try {
-                return JSON.parse(jsonString);
-            } catch (e) {
-                if (Allow.NUM & allow)
-                    try {
-                        return JSON.parse(jsonString.substring(0, jsonString.lastIndexOf("e")));
-                    } catch (e) {}
-                throwMalformedError(String(e));
-            }
-        }
-
         const start = index;
+        while (index < length && !",]} \n\r\t".includes(jsonString[index])) index++;
+        const token = jsonString.substring(start, index);
+        const atEnd = index === length;
 
-        if (jsonString[index] === "-") index++;
-        while (jsonString[index] && ",]}".indexOf(jsonString[index]) === -1) index++;
-
-        if (index == length && !(Allow.NUM & allow)) markPartialJSON("Unterminated number literal");
-
+        let value;
         try {
-            return JSON.parse(jsonString.substring(start, index));
+            value = JSON.parse(token);
         } catch (e) {
-            if (jsonString.substring(start, index) === "-") markPartialJSON("Not sure what '-' is");
-            try {
-                return JSON.parse(jsonString.substring(start, jsonString.lastIndexOf("e")));
-            } catch (e) {
-                throwMalformedError(String(e));
+            if (token === "-" && atEnd && start > 0) markPartialJSON("Not sure what '-' is");
+            if (atEnd) {
+                const partial = token.match(/^(-?(?:0|[1-9]\d*)(?:\.\d+)?)[eE][+-]?$/)
+                    ?? token.match(/^(-?(?:0|[1-9]\d*))\.$/);
+                if (partial) {
+                    if (Allow.NUM & allow) return JSON.parse(partial[1]);
+                    if (start > 0) markPartialJSON("Unterminated number literal");
+                }
             }
+            throwMalformedError(String(e));
         }
+        // Only valid numeric tokens can be incomplete streaming numbers.
+        if (start > 0 && atEnd && !(Allow.NUM & allow)) markPartialJSON("Unterminated number literal");
+        return value;
     };
 
     const skipBlank = () => {
         while (index < length && " \n\r\t".includes(jsonString[index])) {
             index++;
         }
     };
-    return parseAny();
+    const value = parseAny();
+    skipBlank();
+    if (index < length) throwMalformedError("Unexpected content after JSON value");
+    return value;
 };
 
 const parse = parseJSON;

src/number-regressions.test.ts

@@ -0,0 +1,22 @@
+import { describe, expect, it } from "vitest";
+import { Allow, MalformedJSON, parse } from "./index";
+
+describe("partial number suffixes", () => {
+    it.each(["123.", "123e", "123E", "123e+", "123E-", "-123."])("completes only a valid unfinished suffix: %s", (input) => {
+        const expected = input.startsWith("-") ? -123 : 123;
+        expect(parse(input)).toBe(expected);
+        expect(parse(`[${input}`)).toEqual([expected]);
+    });
+    it.each(["1evil", "1e+oops", "01e", "1.e", "1e2e"])("rejects malformed exponent text: %s", (input) => {
+        expect(() => parse(input)).toThrow(MalformedJSON);
+        expect(() => parse(`[${input}]`)).toThrow(MalformedJSON);
+        expect(() => parse(`{"value":${input}`)).toThrow(MalformedJSON);
+    });
+    it.each(["[1e,2]", "[1.,2]", "[1E+]", '[1e+2junk]'])("does not repair numbers terminated by a delimiter: %s", (input) => {
+        expect(() => parse(input)).toThrow(MalformedJSON);
+    });
+    it("keeps complete numbers and the partial-number opt-out", () => {
+        expect(parse("-1.25E+2", 0)).toBe(-125);
+        expect(parse("[1.25", Allow.ARR)).toEqual([]);
+    });
+});

src/partial-string-whitespace.test.ts

@@ -0,0 +1,17 @@
+import { describe, expect, it } from "vitest";
+import { Allow, parse } from "./index";
+
+describe("whitespace inside unfinished strings", () => {
+    it.each([" ", "  ", "\u00a0", "\u2003"])("preserves a trailing %j", (suffix) => {
+        expect(parse('"text' + suffix)).toBe("text" + suffix);
+        expect(parse('["text' + suffix)).toEqual(["text" + suffix]);
+        expect(parse('{"key":"text' + suffix)).toEqual({ key: "text" + suffix });
+    });
+    it("still ignores surrounding whitespace outside a complete value", () => {
+        expect(parse('  "text "  \n')).toBe("text ");
+        expect(parse('  [1,2]  \n')).toEqual([1,2]);
+    });
+    it("retains the partial-string opt-out", () => {
+        expect(parse('["text ', Allow.ARR)).toEqual([]);
+    });
+});

src/separator-regressions.test.ts

@@ -0,0 +1,17 @@
+import { describe, expect, it } from "vitest";
+import { Allow, MalformedJSON, PartialJSON, parse } from "./index";
+
+describe("object and array separators", () => {
+    it.each(['{"a" 1}', '{"a"::1}', '{foo:1}', '[true false]', '["a" "b"]', '[1 2]', '{"a":1 "b":2}', '[{},{} {}]'])("rejects a missing separator or unquoted key: %s", (input) => {
+        expect(() => parse(input)).toThrow(MalformedJSON);
+    });
+    it.each(['{', '{"a"', '{"a":'])("still recovers unfinished object prefixes: %s", (input) => {
+        expect(parse(input)).toEqual({});
+        expect(() => parse(input, 0)).toThrow(PartialJSON);
+    });
+    it("retains valid nesting and permitted incomplete containers", () => {
+        expect(parse('{"a":[1,true,{"b":"c"}]}')).toEqual({a:[1,true,{b:"c"}]});
+        expect(parse('{"a":1')).toEqual({a:1});
+        expect(parse('[1,2', Allow.ARR | Allow.NUM)).toEqual([1,2]);
+    });
+});

src/strict-number-classification.test.ts

@@ -0,0 +1,15 @@
+import { expect, it } from "vitest";
+import { Allow, MalformedJSON, parse } from "./index";
+
+it.each(["[oops", "[01", "[1evil"])("rejects malformed array numbers with NUM disabled: %s", (input) => {
+    expect(() => parse(input, Allow.ARR)).toThrow(MalformedJSON);
+});
+it.each(['{"value":oops', '{"value":01', '{"value":1evil'])("rejects malformed object numbers with NUM disabled: %s", (input) => {
+    expect(() => parse(input, Allow.OBJ)).toThrow(MalformedJSON);
+});
+it("preserves strict root whitespace and streaming numeric opt-out", () => {
+    expect(parse(" 1", 0)).toBe(1);
+    expect(parse("\n-1.5\t", 0)).toBe(-1.5);
+    expect(parse("[1.25", Allow.ARR)).toEqual([]);
+    expect(parse("[1e+", Allow.ARR)).toEqual([]);
+});

src/trailing-content.test.ts

@@ -0,0 +1,18 @@
+import { describe, expect, it } from "vitest";
+import { MalformedJSON, parse } from "./index";
+
+describe("complete document boundary", () => {
+    it.each(['true false', 'null garbage', '"hello" "world"', '{} []', '[] trailing', 'Infinityx', 'NaNtail'])("rejects content after the first value: %s", (input) => {
+        expect(() => parse(input)).toThrow(MalformedJSON);
+    });
+    it.each(['true', 'null', '"hello"', '{}', '[]', '123', '-1.5e2'])("accepts trailing whitespace: %s", (input) => {
+        expect(parse(`${input} \n\t`)).toEqual(JSON.parse(input));
+    });
+    it("still accepts incomplete atoms and containers", () => {
+        expect(parse('tr')).toBe(true);
+        expect(parse('Inf')).toBe(Infinity);
+        expect(parse('1e+')).toBe(1);
+        expect(parse('[1,')).toEqual([1]);
+        expect(parse('{"a":')).toEqual({});
+    });
+});

tsconfig.json

@@ -11,5 +11,11 @@
     "esModuleInterop": true,
     "forceConsistentCasingInFileNames": true,
     "strict": true
-  }
+  },
+  "include": [
+    "src/**/*.ts"
+  ],
+  "exclude": [
+    "src/**/*.test.ts"
+  ]
 }