From 1f700ee1731ea27401270aa6f7ca2e19e8576d0d Mon Sep 17 00:00:00 2001 From: Gareth Johnson <75035963+garethj2@users.noreply.github.com> Date: Thu, 29 Oct 2026 09:32:17 +0000 Subject: [PATCH 1/3] Support new unicode syntax --- spec/core/HStr.spec.ts | 10 ++++++++ spec/core/HUri.spec.ts | 10 ++++++++ spec/core/ZincReader.spec.ts | 44 +++++++++++++++++++++++++++++++++ spec/filter/FilterLexer.spec.ts | 12 +++++++++ src/core/HStr.ts | 17 +++++++------ src/core/HUri.ts | 17 +++++++------ src/core/ZincReader.ts | 43 ++++++++++++++++++++++++++------ src/filter/FilterParser.ts | 2 +- 8 files changed, 133 insertions(+), 22 deletions(-) diff --git a/spec/core/HStr.spec.ts b/spec/core/HStr.spec.ts index ee5cfc0..df8cd9c 100644 --- a/spec/core/HStr.spec.ts +++ b/spec/core/HStr.spec.ts @@ -3,6 +3,7 @@ */ import { HStr } from '../../src/core/HStr' +import { ZincReader } from '../../src/core/ZincReader' import { Kind } from '../../src/core/Kind' import { HGrid } from '../../src/core/grid/HGrid' import { HList } from '../../src/core/list/HList' @@ -160,6 +161,15 @@ describe('HStr', function (): void { expect(HStr.make(str).toZinc()).toBe('"' + str + '"') }) + it('encodes characters above U+FFFF as a surrogate pair', function (): void { + expect(HStr.make('a😀b').toZinc()).toBe('"a\\ud83d\\ude00b"') + }) + + it('round trips characters above U+FFFF', function (): void { + const str = HStr.make('a😀b') + expect(ZincReader.readValue(str.toZinc())).toEqual(str) + }) + it('encodes non-ascii characters to unicode', function (): void { expect(HStr.make('ݧݩᴆᴁaḀCD').toZinc()).toBe( '"\\u0767\\u0769\\u1d06\\u1d01a\\u1e00CD"' diff --git a/spec/core/HUri.spec.ts b/spec/core/HUri.spec.ts index 00da5d9..e4c7211 100644 --- a/spec/core/HUri.spec.ts +++ b/spec/core/HUri.spec.ts @@ -3,6 +3,7 @@ */ import { HUri } from '../../src/core/HUri' +import { ZincReader } from '../../src/core/ZincReader' import { Kind } from '../../src/core/Kind' import { HGrid } from '../../src/core/grid/HGrid' import { HList } from '../../src/core/list/HList' @@ -121,6 +122,15 @@ describe('HUri', function (): void { expect(HUri.make(str).toZinc()).toBe('`' + str + '`') }) + it('encodes characters above U+FFFF as a surrogate pair', function (): void { + expect(HUri.make('a😀b').toZinc()).toBe('`a\\ud83d\\ude00b`') + }) + + it('round trips characters above U+FFFF', function (): void { + const uri = HUri.make('a😀b') + expect(ZincReader.readValue(uri.toZinc())).toEqual(uri) + }) + it('encodes non-ascii characters to unicode', function (): void { expect(HUri.make('ݧݩᴆᴁaḀCD').toZinc()).toBe( '`\\u0767\\u0769\\u1d06\\u1d01a\\u1e00CD`' diff --git a/spec/core/ZincReader.spec.ts b/spec/core/ZincReader.spec.ts index dfa036f..c9bace2 100644 --- a/spec/core/ZincReader.spec.ts +++ b/spec/core/ZincReader.spec.ts @@ -61,6 +61,38 @@ describe('ZincReader', function (): void { ) }) + it('parse a \\u{x} unicode character', function (): void { + expect(makeReader('"Hell\\u{D3}"').readValue()).toEqual( + HStr.make('HellÓ') + ) + }) + + it('parse a \\u{x} character above U+FFFF', function (): void { + expect(makeReader('"a\\u{1F600}b"').readValue()).toEqual( + HStr.make('a😀b') + ) + }) + + it('parse the smallest and largest \\u{x} values', function (): void { + expect(makeReader('"\\u{0}\\u{10FFFF}"').readValue()).toEqual( + HStr.make('\u0000\u{10FFFF}') + ) + }) + + it('throws an error for an invalid \\u{x} escape', function (): void { + for (const bad of [ + '"\\u{}"', + '"\\u{1234567}"', + '"\\u{110000}"', + '"\\u{12G}"', + '"\\u{12', + ]) { + expect((): void => { + makeReader(bad).readValue() + }).toThrow() + } + }) + it('parse a line break', function (): void { expect(makeReader('"Hell\\b"').readValue()).toEqual( HStr.make('Hell\b') @@ -127,6 +159,18 @@ describe('ZincReader', function (): void { ) }) + it('parse a \\u{x} unicode character', function (): void { + expect(makeReader('`Hell\\u{D3}`').readValue()).toEqual( + HUri.make('HellÓ') + ) + }) + + it('parse a \\u{x} character above U+FFFF', function (): void { + expect(makeReader('`a\\u{1F600}b`').readValue()).toEqual( + HUri.make('a😀b') + ) + }) + it('throws an error for a missing tick', function (): void { expect((): void => { makeReader('`/foo/bag').readValue() diff --git a/spec/filter/FilterLexer.spec.ts b/spec/filter/FilterLexer.spec.ts index 8253d63..76eecfa 100644 --- a/spec/filter/FilterLexer.spec.ts +++ b/spec/filter/FilterLexer.spec.ts @@ -106,6 +106,12 @@ describe('FilterLexer', function (): void { expect(token.toString()).toBe('HellÓ') }) + it('parse a \\u{x} unicode character', function (): void { + const token = makeLexer('"a\\u{1F600}b"').nextToken() + expect(token.type).toBe(TokenType.string) + expect(token.toString()).toBe('a😀b') + }) + it('parse a line break', function (): void { const token = makeLexer('"Hell\\b"').nextToken() expect(token.type).toBe(TokenType.string) @@ -184,6 +190,12 @@ describe('FilterLexer', function (): void { expect(token.toString()).toBe('HellÓ') }) + it('parse a \\u{x} unicode character', function (): void { + const token = makeLexer('`a\\u{1F600}b`').nextToken() + expect(token.type).toBe(TokenType.uri) + expect(token.toString()).toBe('a😀b') + }) + it('throws an error for a missing tick', function (): void { expect((): void => { makeLexer('`/foo/bag').nextToken() diff --git a/src/core/HStr.ts b/src/core/HStr.ts index b06879b..86f8581 100644 --- a/src/core/HStr.ts +++ b/src/core/HStr.ts @@ -203,14 +203,17 @@ export class HStr implements HVal { buf += '$' break default: - let seq = code.toString(16) - - // Make sure we have four characters for the unicode sequence. - while (seq.length < 4) { - seq = `0${seq}` + // Escape each UTF-16 code unit so characters above U+FFFF + // are written as a surrogate pair. + for (let i = 0; i < c.length; ++i) { + // Make sure we have four characters for the unicode sequence. + const seq = c + .charCodeAt(i) + .toString(16) + .padStart(4, '0') + + buf += `${i > 0 ? '\\' : ''}u${seq}` } - - buf += `u${seq}` } } else { buf += c diff --git a/src/core/HUri.ts b/src/core/HUri.ts index 5caf62e..2566d35 100644 --- a/src/core/HUri.ts +++ b/src/core/HUri.ts @@ -202,14 +202,17 @@ export class HUri implements HVal { buf += '`' break default: - let seq = code.toString(16) - - // Make sure we have four characters for the unicode sequence. - while (seq.length < 4) { - seq = `0${seq}` + // Escape each UTF-16 code unit so characters above U+FFFF + // are written as a surrogate pair. + for (let i = 0; i < c.length; ++i) { + // Make sure we have four characters for the unicode sequence. + const seq = c + .charCodeAt(i) + .toString(16) + .padStart(4, '0') + + buf += `${i > 0 ? '\\' : ''}u${seq}` } - - buf += `u${seq}` } } else { buf += c diff --git a/src/core/ZincReader.ts b/src/core/ZincReader.ts index 14bcbf4..216b448 100644 --- a/src/core/ZincReader.ts +++ b/src/core/ZincReader.ts @@ -328,6 +328,19 @@ export class ZincReader { return HRef.make(str, displayName) } + /** + * @param esc The invalid escape characters. + * @returns An error for an invalid unicode escape. + */ + private invalidUnicodeEscape(esc: string): LocalizedError { + return new LocalizedError({ + message: `Invalid unicode escape ${esc}`, + lex: 'lexerInvalidUnicodeEscape', + args: { arg: esc }, + index: this.scanner.index, + }) + } + /** * Consume an escape sequence within a string. * @@ -362,7 +375,28 @@ export class ZincReader { } } - if (this.scanner.is('u')) { + if (this.scanner.is('u') && this.scanner.next === '{') { + // Handle \u{x} where x is 1 to 6 hex digits. + this.scanner.consume() // u + this.scanner.consume() // { + let esc = '' + + while (!this.scanner.is('}')) { + esc += this.scanner.current ?? '' + + if (!this.scanner.isHex() || esc.length > 6) { + throw this.invalidUnicodeEscape(esc) + } + this.scanner.consume() + } + + const codePoint = parseInt(esc, 16) + if (esc.length === 0 || codePoint > 0x10ffff) { + throw this.invalidUnicodeEscape(esc) + } + + return String.fromCodePoint(codePoint) + } else if (this.scanner.is('u')) { let esc = '' for (let i = 0; i < 4; ++i) { @@ -370,12 +404,7 @@ export class ZincReader { esc += this.scanner.current if (!this.scanner.isHex()) { - throw new LocalizedError({ - message: `Invalid unicode escape ${esc}`, - lex: 'lexerInvalidUnicodeEscape', - args: { arg: esc }, - index: this.scanner.index, - }) + throw this.invalidUnicodeEscape(esc) } } diff --git a/src/filter/FilterParser.ts b/src/filter/FilterParser.ts index 562dc58..9d48676 100644 --- a/src/filter/FilterParser.ts +++ b/src/filter/FilterParser.ts @@ -74,7 +74,7 @@ export const enum booleanOperators { * := any 16-bit Unicode char >= 0x20 (except str/uri quote) * := "\b" | "\f" | "\n" | "\r" | "\r" | "\t" | "\"" | "\\" | "\$" | * := "\:" | "\/" | "\?" | "\#" | "\[" | "\]" | "\@" | "\`" | "\\" | "\&" | "\=" | "\;" | - * := "\u" + * := "\u" | "\u{" {1,6} "}" * := ('a'-'f') | ('A'-'F') | digit * := ( | | | '_')* * := ('a' - 'z') From 00c8e0ef488b30d80230742726c3b74c317ebf2e Mon Sep 17 00:00:00 2001 From: Gareth Johnson <75035963+garethj2@users.noreply.github.com> Date: Thu, 29 Oct 2026 10:02:56 +0000 Subject: [PATCH 2/3] Enhance escaping --- spec/core/HVal.spec.ts | 12 ++++++++++++ spec/core/ZincReader.spec.ts | 27 +++++++++++++++++++-------- src/core/HStr.ts | 14 +++----------- src/core/HUri.ts | 14 +++----------- src/core/HVal.ts | 17 +++++++++++++++++ src/core/ZincReader.ts | 14 ++++++++++++-- 6 files changed, 66 insertions(+), 32 deletions(-) diff --git a/spec/core/HVal.spec.ts b/spec/core/HVal.spec.ts index 0cbf4c2..a8d1d2a 100644 --- a/spec/core/HVal.spec.ts +++ b/spec/core/HVal.spec.ts @@ -10,6 +10,7 @@ import { valueEquals, valueToZinc, valueToAxon, + toUnicodeEscape, HVal, } from '../../src/core/HVal' import { HBool } from '../../src/core/HBool' @@ -104,4 +105,15 @@ describe('HVal', function (): void { expect(valueToAxon(null)).toBe('null') }) }) // .valueToAxon() + + describe('.toUnicodeEscape()', function (): void { + it('returns a four digit escape', function (): void { + expect(toUnicodeEscape('\u0001')).toBe('\\u0001') + expect(toUnicodeEscape('ῶ')).toBe('\\u1ff6') + }) + + it('returns a surrogate pair for characters above U+FFFF', function (): void { + expect(toUnicodeEscape('😀')).toBe('\\ud83d\\ude00') + }) + }) // .toUnicodeEscape() }) diff --git a/spec/core/ZincReader.spec.ts b/spec/core/ZincReader.spec.ts index c9bace2..c4df9bf 100644 --- a/spec/core/ZincReader.spec.ts +++ b/spec/core/ZincReader.spec.ts @@ -23,6 +23,7 @@ import { HList } from '../../src/core/list/HList' import { HGrid } from '../../src/core/grid/HGrid' import { HVal } from '../../src/core/HVal' import { readFile } from '../file' +import { LocalizedError } from '../../src/util/LocalizedError' describe('ZincReader', function (): void { function makeReader(input: string): ZincReader { @@ -80,16 +81,26 @@ describe('ZincReader', function (): void { }) it('throws an error for an invalid \\u{x} escape', function (): void { - for (const bad of [ - '"\\u{}"', - '"\\u{1234567}"', - '"\\u{110000}"', - '"\\u{12G}"', - '"\\u{12', + for (const [bad, arg] of [ + ['"\\u{}"', ''], + ['"\\u{1234567}"', '1234567'], + ['"\\u{110000}"', '110000'], + ['"\\u{12G}"', '12G'], + ['"\\u{12"', '12'], + ['"\\u{12', '12'], ]) { - expect((): void => { + let error: unknown + try { makeReader(bad).readValue() - }).toThrow() + } catch (err) { + error = err + } + + expect(error).toBeInstanceOf(LocalizedError) + expect((error as LocalizedError).lex).toBe( + 'lexerInvalidUnicodeEscape' + ) + expect((error as LocalizedError).args).toEqual({ arg }) } }) diff --git a/src/core/HStr.ts b/src/core/HStr.ts index 86f8581..f64f5ac 100644 --- a/src/core/HStr.ts +++ b/src/core/HStr.ts @@ -10,6 +10,7 @@ import { valueIsKind, valueMatches, TEXT_ENCODER, + toUnicodeEscape, } from './HVal' import { Node } from '../filter/Node' import { HGrid } from './grid/HGrid' @@ -203,17 +204,8 @@ export class HStr implements HVal { buf += '$' break default: - // Escape each UTF-16 code unit so characters above U+FFFF - // are written as a surrogate pair. - for (let i = 0; i < c.length; ++i) { - // Make sure we have four characters for the unicode sequence. - const seq = c - .charCodeAt(i) - .toString(16) - .padStart(4, '0') - - buf += `${i > 0 ? '\\' : ''}u${seq}` - } + // The leading backslash has already been written. + buf += toUnicodeEscape(c).substring(1) } } else { buf += c diff --git a/src/core/HUri.ts b/src/core/HUri.ts index 2566d35..e90c271 100644 --- a/src/core/HUri.ts +++ b/src/core/HUri.ts @@ -10,6 +10,7 @@ import { valueIsKind, valueMatches, TEXT_ENCODER, + toUnicodeEscape, } from './HVal' import { HaysonUri } from './hayson' import { Node } from '../filter/Node' @@ -202,17 +203,8 @@ export class HUri implements HVal { buf += '`' break default: - // Escape each UTF-16 code unit so characters above U+FFFF - // are written as a surrogate pair. - for (let i = 0; i < c.length; ++i) { - // Make sure we have four characters for the unicode sequence. - const seq = c - .charCodeAt(i) - .toString(16) - .padStart(4, '0') - - buf += `${i > 0 ? '\\' : ''}u${seq}` - } + // The leading backslash has already been written. + buf += toUnicodeEscape(c).substring(1) } } else { buf += c diff --git a/src/core/HVal.ts b/src/core/HVal.ts index 2788284..ada25eb 100644 --- a/src/core/HVal.ts +++ b/src/core/HVal.ts @@ -123,6 +123,23 @@ export function valueToZinc(val: HVal | null): string { return val?.toZinc() ?? ZINC_NULL } +/** + * Returns a character as a `\uXXXX` escape sequence. + * + * Characters above U+FFFF are returned as a surrogate pair of escapes + * so they can be read by older readers that don't support `\u{x}`. + * + * @param c The character to escape. + * @returns The escape sequence. + */ +export function toUnicodeEscape(c: string): string { + let esc = '' + for (let i = 0; i < c.length; ++i) { + esc += `\\u${c.charCodeAt(i).toString(16).padStart(4, '0')}` + } + return esc +} + /** * The Axon symbol for null */ diff --git a/src/core/ZincReader.ts b/src/core/ZincReader.ts index 216b448..2f1d8c0 100644 --- a/src/core/ZincReader.ts +++ b/src/core/ZincReader.ts @@ -382,9 +382,19 @@ export class ZincReader { let esc = '' while (!this.scanner.is('}')) { - esc += this.scanner.current ?? '' + if (!this.scanner.isHex()) { + // Only report the bad char if it's part of the escape. + throw this.invalidUnicodeEscape( + this.scanner.isEof() || + this.scanner.is('"') || + this.scanner.is('`') + ? esc + : esc + this.scanner.current + ) + } - if (!this.scanner.isHex() || esc.length > 6) { + esc += this.scanner.current + if (esc.length > 6) { throw this.invalidUnicodeEscape(esc) } this.scanner.consume() From f654c81add323a84f5d6444be4103b79a25c257e Mon Sep 17 00:00:00 2001 From: Gareth Johnson <75035963+garethj2@users.noreply.github.com> Date: Mon, 2 Nov 2026 08:51:05 +0000 Subject: [PATCH 3/3] Add isNaN check after parseInt --- src/core/HNum.ts | 6 +++--- src/core/HTime.ts | 2 +- src/core/HUnit.ts | 2 +- src/core/ZincReader.ts | 14 +++++++++----- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/core/HNum.ts b/src/core/HNum.ts index cac5cae..1e6af43 100644 --- a/src/core/HNum.ts +++ b/src/core/HNum.ts @@ -237,7 +237,7 @@ export class HNum implements HVal { return POSITIVE_INFINITY_ZINC } else if (this.value === Number.NEGATIVE_INFINITY) { return NEGATIVE_INFINITY_ZINC - } else if (isNaN(this.value)) { + } else if (Number.isNaN(this.value)) { return NOT_A_NUMBER_ZINC } else { const value = this.value.toLocaleString(locale, { @@ -291,7 +291,7 @@ export class HNum implements HVal { return POSITIVE_INFINITY_ZINC } else if (this.value === Number.NEGATIVE_INFINITY) { return NEGATIVE_INFINITY_ZINC - } else if (isNaN(this.value)) { + } else if (Number.isNaN(this.value)) { return NOT_A_NUMBER_ZINC } else { return this.#unitSymbol @@ -380,7 +380,7 @@ export class HNum implements HVal { val = POSITIVE_INFINITY_ZINC } else if (this.value === Number.NEGATIVE_INFINITY) { val = NEGATIVE_INFINITY_ZINC - } else if (isNaN(this.value)) { + } else if (Number.isNaN(this.value)) { val = NOT_A_NUMBER_ZINC } else { val = this.value diff --git a/src/core/HTime.ts b/src/core/HTime.ts index 9714d53..955a3bc 100644 --- a/src/core/HTime.ts +++ b/src/core/HTime.ts @@ -80,7 +80,7 @@ export class HTime implements HVal { private static toJsDate(value: string): Date { const time = Date.parse('1970-01-01T' + value + 'Z') - if (isNaN(time)) { + if (Number.isNaN(time)) { throw new Error(`Invalid Time format ${value}`) } return new Date(time) diff --git a/src/core/HUnit.ts b/src/core/HUnit.ts index 89a09a7..f788dd4 100644 --- a/src/core/HUnit.ts +++ b/src/core/HUnit.ts @@ -251,7 +251,7 @@ export class HUnit { private static parseNumber(str: string): number { const num = Number(str.trim()) - if (isNaN(num) || typeof num !== 'number') { + if (Number.isNaN(num) || typeof num !== 'number') { throw new Error(`Unable to convert '${str}' to a number`) } diff --git a/src/core/ZincReader.ts b/src/core/ZincReader.ts index 2f1d8c0..b443146 100644 --- a/src/core/ZincReader.ts +++ b/src/core/ZincReader.ts @@ -400,8 +400,12 @@ export class ZincReader { this.scanner.consume() } - const codePoint = parseInt(esc, 16) - if (esc.length === 0 || codePoint > 0x10ffff) { + const codePoint = Number.parseInt(esc, 16) + if ( + Number.isNaN(codePoint) || + esc.length === 0 || + codePoint > 0x10ffff + ) { throw this.invalidUnicodeEscape(esc) } @@ -418,7 +422,7 @@ export class ZincReader { } } - return String.fromCharCode(parseInt(esc, 16)) + return String.fromCharCode(Number.parseInt(esc, 16)) } else { throw new LocalizedError({ message: `Invalid escape sequence ${this.scanner.current}`, @@ -533,7 +537,7 @@ export class ZincReader { } // If we have a number then attempt to parse it and throw any errors. - if (isNaN(Number(str))) { + if (Number.isNaN(Number(str))) { throw new LocalizedError({ message: `Invalid number: ${str}`, lex: 'lexerInvalidNumber', @@ -661,7 +665,7 @@ export class ZincReader { const num = Number(buf) - if (isNaN(num)) { + if (Number.isNaN(num)) { throw new LocalizedError({ message: `Invalid decimal number ${buf}`, lex: 'lexerInvalidDecimal',