From 7ae8b122facb605c35c9ebe2c566abd49aea252a Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 7 Oct 2026 13:36:40 +0000 Subject: [PATCH] feat: opt-in safe JSON parsing for IDs above MAX_SAFE_INTEGER (v1.7.0) Changelog: - Add quoteLossyIntegers / parseJSONSafeIds scanner (no new dependencies) - REST: keepIdsAsString + customParseJSONFn via axios text + transformResponse - WebSocket: same options in resolveEmittableEvents JSON.parse path - Unit tests for scanner edge cases; benchmark script for 1000-level order book Addresses sieblyio/gateio-api#101 (PoC on JJ-Cro fork). Co-authored-by: Jerko J --- package.json | 2 +- scripts/benchmark-json-parse.ts | 42 +++++++++++ src/WebsocketClient.ts | 6 +- src/lib/BaseRestClient.ts | 14 ++++ src/lib/jsonParse.ts | 122 ++++++++++++++++++++++++++++++++ src/lib/requestUtils.ts | 9 +++ src/types/websockets/client.ts | 9 +++ test/lib/jsonParse.test.ts | 90 +++++++++++++++++++++++ 8 files changed, 292 insertions(+), 2 deletions(-) create mode 100644 scripts/benchmark-json-parse.ts create mode 100644 src/lib/jsonParse.ts create mode 100644 test/lib/jsonParse.test.ts diff --git a/package.json b/package.json index 7df721f..5f2207a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gateio-api", - "version": "1.6.4", + "version": "1.7.0", "description": "Complete & Robust Node.js SDK for Gate.com's REST APIs, WebSockets & WebSocket APIs, with TypeScript declarations.", "scripts": { "clean": "rm -rf dist/*", diff --git a/scripts/benchmark-json-parse.ts b/scripts/benchmark-json-parse.ts new file mode 100644 index 0000000..cee7ce4 --- /dev/null +++ b/scripts/benchmark-json-parse.ts @@ -0,0 +1,42 @@ +import { performance } from 'node:perf_hooks'; + +import { parseJSONSafeIds } from '../dist/mjs/lib/jsonParse.js'; + +const LOSSY_ID = '9007199254740992'; +const LEVELS = 1000; + +function buildOrderBookJson(levels: number): string { + const bids: string[] = []; + const asks: string[] = []; + for (let i = 0; i < levels; i++) { + const price = (50000 - i * 0.01).toFixed(2); + const size = (i + 1).toString(); + bids.push( + `{"p":"${price}","s":"${size}","order_id":${LOSSY_ID}}`, + ); + asks.push( + `{"p":"${(50000 + i * 0.01).toFixed(2)}","s":"${size}","order_id":${LOSSY_ID}}`, + ); + } + return `{"channel":"futures.order_book","result":{"bids":[${bids.join(',')}],"asks":[${asks.join(',')}]}}`; +} + +const payload = buildOrderBookJson(LEVELS); +const iterations = 200; + +function bench(label: string, fn: (text: string) => unknown): number { + const start = performance.now(); + for (let i = 0; i < iterations; i++) { + fn(payload); + } + const ms = performance.now() - start; + console.log(`${label}: ${ms.toFixed(2)} ms total (${(ms / iterations).toFixed(3)} ms/op)`); + return ms; +} + +const plainMs = bench('JSON.parse', (text) => JSON.parse(text)); +const safeMs = bench('parseJSONSafeIds', (text) => parseJSONSafeIds(text)); + +console.log( + `ratio parseJSONSafeIds / JSON.parse: ${(safeMs / plainMs).toFixed(2)}x`, +); diff --git a/src/WebsocketClient.ts b/src/WebsocketClient.ts index 5da337a..9a281e7 100644 --- a/src/WebsocketClient.ts +++ b/src/WebsocketClient.ts @@ -1,4 +1,5 @@ import { BaseWebsocketClient, EmittableEvent } from './lib/BaseWSClient.js'; +import { resolveJsonParseFn } from './lib/jsonParse.js'; import { neverGuard } from './lib/misc-util.js'; import { CHANNEL_ID, MessageEventLike } from './lib/requestUtils.js'; import { @@ -378,7 +379,10 @@ export class WebsocketClient extends BaseWebsocketClient { const results: EmittableEvent[] = []; try { - const parsed = JSON.parse(event.data); + const parseFn = + resolveJsonParseFn(this.options) ?? + ((text: string) => JSON.parse(text)); + const parsed = parseFn(event.data); const responseEvents = ['subscribe', 'unsubscribe']; const authenticatedEvents = ['auth']; diff --git a/src/lib/BaseRestClient.ts b/src/lib/BaseRestClient.ts index 85ba9aa..0244dab 100644 --- a/src/lib/BaseRestClient.ts +++ b/src/lib/BaseRestClient.ts @@ -3,6 +3,7 @@ import axios, { AxiosRequestConfig, AxiosResponse, Method } from 'axios'; // Browser builds (via webpack) exclude this module - see webpack.config.js fallback settings import https from 'https'; +import { resolveJsonParseFn } from './jsonParse.js'; import { neverGuard } from './misc-util.js'; import { CHANNEL_ID, @@ -173,6 +174,19 @@ export abstract class BaseRestClient { // NOTE: This is Node.js-only functionality. In browser environments, this code is skipped // as the 'https' module is excluded via webpack fallback configuration. // Browser connection pooling is handled automatically by the browser itself. + const jsonParseFn = resolveJsonParseFn(this.options); + if (jsonParseFn) { + this.globalRequestOptions.responseType = 'text'; + this.globalRequestOptions.transformResponse = [ + (data: string) => { + if (typeof data === 'string') { + return jsonParseFn(data); + } + return data; + }, + ]; + } + if (this.options.keepAlive) { // Extract existing https agent parameters, if provided, to prevent the keepAlive flag from overwriting an existing https agent completely const existingHttpsAgent = this.globalRequestOptions.httpsAgent as diff --git a/src/lib/jsonParse.ts b/src/lib/jsonParse.ts new file mode 100644 index 0000000..2f895c3 --- /dev/null +++ b/src/lib/jsonParse.ts @@ -0,0 +1,122 @@ +const MAX_SAFE_INTEGER_STR = '9007199254740991'; + +function isLossyIntegerLiteral(token: string): boolean { + let i = 0; + if (token[i] === '-') { + i++; + } + const digits = token.slice(i); + if (!digits.length || !/^\d+$/.test(digits)) { + return false; + } + if (digits.length < MAX_SAFE_INTEGER_STR.length) { + return false; + } + if (digits.length > MAX_SAFE_INTEGER_STR.length) { + return true; + } + return digits > MAX_SAFE_INTEGER_STR; +} + +/** + * Walk JSON text and wrap unsafe integer literals in quotes so JSON.parse keeps them as strings. + */ +export function quoteLossyIntegers(text: string): string { + let out = ''; + let i = 0; + let inString = false; + + while (i < text.length) { + const c = text[i]; + + if (inString) { + out += c; + if (c === '\\') { + i++; + if (i < text.length) { + out += text[i]; + } + } else if (c === '"') { + inString = false; + } + i++; + continue; + } + + if (c === '"') { + inString = true; + out += c; + i++; + continue; + } + + if (c === '-' || (c >= '0' && c <= '9')) { + let j = i; + if (text[j] === '-') { + j++; + } + const intStart = j; + while (j < text.length && text[j] >= '0' && text[j] <= '9') { + j++; + } + if (j === intStart && text[i] === '-') { + out += c; + i++; + continue; + } + + let k = j; + if (k < text.length && text[k] === '.') { + k++; + while (k < text.length && text[k] >= '0' && text[k] <= '9') { + k++; + } + } + if (k < text.length && (text[k] === 'e' || text[k] === 'E')) { + k++; + if (k < text.length && (text[k] === '+' || text[k] === '-')) { + k++; + } + while (k < text.length && text[k] >= '0' && text[k] <= '9') { + k++; + } + } + + const token = text.slice(i, k); + const isIntegerOnly = k === j; + if (isIntegerOnly && isLossyIntegerLiteral(token)) { + out += `"${token}"`; + } else { + out += token; + } + i = k; + continue; + } + + out += c; + i++; + } + + return out; +} + +export function parseJSONSafeIds(text: string): unknown { + return JSON.parse(quoteLossyIntegers(text)); +} + +export type SafeIdJsonParseOptions = { + keepIdsAsString?: boolean; + customParseJSONFn?: (text: string) => unknown; +}; + +export function resolveJsonParseFn( + options: SafeIdJsonParseOptions, +): ((text: string) => unknown) | undefined { + if (typeof options.customParseJSONFn === 'function') { + return options.customParseJSONFn; + } + if (options.keepIdsAsString) { + return parseJSONSafeIds; + } + return undefined; +} diff --git a/src/lib/requestUtils.ts b/src/lib/requestUtils.ts index ae40631..2c75b20 100644 --- a/src/lib/requestUtils.ts +++ b/src/lib/requestUtils.ts @@ -49,6 +49,15 @@ export interface RestClientOptions { * Look in the examples folder for a demonstration on using node's createHmac instead. */ customSignMessageFn?: (message: string, secret: string) => Promise; + + /** + * When true, integer literals above Number.MAX_SAFE_INTEGER are parsed as strings (REST responses). + * Default: false. + */ + keepIdsAsString?: boolean; + + /** Override JSON parsing for REST responses (e.g. custom safe-id parser). */ + customParseJSONFn?: (text: string) => unknown; } export function serializeParams | undefined = {}>( diff --git a/src/types/websockets/client.ts b/src/types/websockets/client.ts index 7485936..c830a05 100644 --- a/src/types/websockets/client.ts +++ b/src/types/websockets/client.ts @@ -58,6 +58,15 @@ export interface WSClientConfigurableOptions { * If you authenticated the WS API before, automatically try to re-authenticate the WS API if you're disconnected/reconnected for any reason. */ reauthWSAPIOnReconnect?: boolean; + + /** + * When true, integer literals above Number.MAX_SAFE_INTEGER are parsed as strings (WS messages). + * Default: false. + */ + keepIdsAsString?: boolean; + + /** Override JSON parsing for WS messages (e.g. custom safe-id parser). */ + customParseJSONFn?: (text: string) => unknown; } /** diff --git a/test/lib/jsonParse.test.ts b/test/lib/jsonParse.test.ts new file mode 100644 index 0000000..22f6bd9 --- /dev/null +++ b/test/lib/jsonParse.test.ts @@ -0,0 +1,90 @@ +import { + parseJSONSafeIds, + quoteLossyIntegers, + resolveJsonParseFn, +} from '../../src/lib/jsonParse'; + +const LOSSY_ID = '9007199254740992'; +const SAFE_ID = '9007199254740991'; + +describe('jsonParse safe integer scanner', () => { + describe('quoteLossyIntegers', () => { + it('wraps a lossy id at the root', () => { + const text = `{"id":${LOSSY_ID}}`; + expect(quoteLossyIntegers(text)).toBe(`{"id":"${LOSSY_ID}"}`); + }); + + it('wraps ids in nested objects and arrays', () => { + const text = `{"order":{"id":${LOSSY_ID}},"ids":[${LOSSY_ID},${SAFE_ID}]}`; + const quoted = quoteLossyIntegers(text); + expect(quoted).toContain(`"id":"${LOSSY_ID}"`); + expect(quoted).toContain(`["${LOSSY_ID}",${SAFE_ID}]`); + }); + + it('wraps negative lossy integers', () => { + const text = `{"id":-${LOSSY_ID}}`; + expect(quoteLossyIntegers(text)).toBe(`{"id":"-${LOSSY_ID}"}`); + }); + + it('leaves integers inside strings untouched', () => { + const text = `{"note":"order ${LOSSY_ID} pending"}`; + expect(quoteLossyIntegers(text)).toBe(text); + }); + + it('handles escaped quotes inside strings', () => { + const text = `{"note":"id \\"${LOSSY_ID}\\" in text","id":${LOSSY_ID}}`; + const quoted = quoteLossyIntegers(text); + expect(quoted).toContain(`"id \\"${LOSSY_ID}\\" in text"`); + expect(quoted).toContain(`"id":"${LOSSY_ID}"`); + }); + + it('does not wrap decimals or exponents', () => { + const text = `{"a":${LOSSY_ID}.5,"b":1e20}`; + expect(quoteLossyIntegers(text)).toBe(text); + }); + + it('keeps safe integers as bare numbers', () => { + const text = `{"id":${SAFE_ID},"small":42}`; + expect(quoteLossyIntegers(text)).toBe(text); + }); + }); + + describe('parseJSONSafeIds', () => { + it('parses lossy ids as strings and safe ids as numbers', () => { + const parsed = parseJSONSafeIds( + `{"id":${LOSSY_ID},"safe":${SAFE_ID},"nested":{"ids":[${LOSSY_ID}]}}`, + ) as { + id: string; + safe: number; + nested: { ids: string[] }; + }; + + expect(parsed.id).toBe(LOSSY_ID); + expect(typeof parsed.id).toBe('string'); + expect(parsed.safe).toBe(Number(SAFE_ID)); + expect(typeof parsed.safe).toBe('number'); + expect(parsed.nested.ids[0]).toBe(LOSSY_ID); + }); + + it('matches JSON.parse when no lossy integers are present', () => { + const text = `{"price":"1.25","amount":100,"tags":["a","b"]}`; + expect(parseJSONSafeIds(text)).toEqual(JSON.parse(text)); + }); + }); + + describe('resolveJsonParseFn', () => { + it('returns undefined by default', () => { + expect(resolveJsonParseFn({})).toBeUndefined(); + }); + + it('prefers customParseJSONFn over keepIdsAsString', () => { + const custom = (t: string) => ({ custom: t.length }); + expect( + resolveJsonParseFn({ + keepIdsAsString: true, + customParseJSONFn: custom, + }), + ).toBe(custom); + }); + }); +});