Support new unicode syntax

spec/core/HStr.spec.ts

@@ -3,6 +3,7 @@
  */
 
 import { HStr } from '../../src/core/HStr'
+import { ZincReader } from '../../src/core/ZincReader'
 import { Kind } from '../../src/core/Kind'
 import { HGrid } from '../../src/core/grid/HGrid'
 import { HList } from '../../src/core/list/HList'
@@ -160,6 +161,15 @@ describe('HStr', function (): void {
 			expect(HStr.make(str).toZinc()).toBe('"' + str + '"')
 		})
 
+		it('encodes characters above U+FFFF as a surrogate pair', function (): void {
+			expect(HStr.make('a😀b').toZinc()).toBe('"a\\ud83d\\ude00b"')
+		})
+
+		it('round trips characters above U+FFFF', function (): void {
+			const str = HStr.make('a😀b')
+			expect(ZincReader.readValue(str.toZinc())).toEqual(str)
+		})
+
 		it('encodes non-ascii characters to unicode', function (): void {
 			expect(HStr.make('ݧݩᴆᴁaḀCD').toZinc()).toBe(
 				'"\\u0767\\u0769\\u1d06\\u1d01a\\u1e00CD"'

spec/core/HUri.spec.ts

@@ -3,6 +3,7 @@
  */
 
 import { HUri } from '../../src/core/HUri'
+import { ZincReader } from '../../src/core/ZincReader'
 import { Kind } from '../../src/core/Kind'
 import { HGrid } from '../../src/core/grid/HGrid'
 import { HList } from '../../src/core/list/HList'
@@ -121,6 +122,15 @@ describe('HUri', function (): void {
 			expect(HUri.make(str).toZinc()).toBe('`' + str + '`')
 		})
 
+		it('encodes characters above U+FFFF as a surrogate pair', function (): void {
+			expect(HUri.make('a😀b').toZinc()).toBe('`a\\ud83d\\ude00b`')
+		})
+
+		it('round trips characters above U+FFFF', function (): void {
+			const uri = HUri.make('a😀b')
+			expect(ZincReader.readValue(uri.toZinc())).toEqual(uri)
+		})
+
 		it('encodes non-ascii characters to unicode', function (): void {
 			expect(HUri.make('ݧݩᴆᴁaḀCD').toZinc()).toBe(
 				'`\\u0767\\u0769\\u1d06\\u1d01a\\u1e00CD`'

spec/core/HVal.spec.ts

@@ -10,6 +10,7 @@ import {
 	valueEquals,
 	valueToZinc,
 	valueToAxon,
+	toUnicodeEscape,
 	HVal,
 } from '../../src/core/HVal'
 import { HBool } from '../../src/core/HBool'
@@ -104,4 +105,15 @@ describe('HVal', function (): void {
 			expect(valueToAxon(null)).toBe('null')
 		})
 	}) // .valueToAxon()
+
+	describe('.toUnicodeEscape()', function (): void {
+		it('returns a four digit escape', function (): void {
+			expect(toUnicodeEscape('\u0001')).toBe('\\u0001')
+			expect(toUnicodeEscape('ῶ')).toBe('\\u1ff6')
+		})
+
+		it('returns a surrogate pair for characters above U+FFFF', function (): void {
+			expect(toUnicodeEscape('😀')).toBe('\\ud83d\\ude00')
+		})
+	}) // .toUnicodeEscape()
 })

spec/core/ZincReader.spec.ts

@@ -23,6 +23,7 @@ import { HList } from '../../src/core/list/HList'
 import { HGrid } from '../../src/core/grid/HGrid'
 import { HVal } from '../../src/core/HVal'
 import { readFile } from '../file'
+import { LocalizedError } from '../../src/util/LocalizedError'
 
 describe('ZincReader', function (): void {
 	function makeReader(input: string): ZincReader {
@@ -61,6 +62,48 @@ describe('ZincReader', function (): void {
 				)
 			})
 
+			it('parse a \\u{x} unicode character', function (): void {
+				expect(makeReader('"Hell\\u{D3}"').readValue()).toEqual(
+					HStr.make('HellÓ')
+				)
+			})
+
+			it('parse a \\u{x} character above U+FFFF', function (): void {
+				expect(makeReader('"a\\u{1F600}b"').readValue()).toEqual(
+					HStr.make('a😀b')
+				)
+			})
+
+			it('parse the smallest and largest \\u{x} values', function (): void {
+				expect(makeReader('"\\u{0}\\u{10FFFF}"').readValue()).toEqual(
+					HStr.make('\u0000\u{10FFFF}')
+				)
+			})
+
+			it('throws an error for an invalid \\u{x} escape', function (): void {
+				for (const [bad, arg] of [
+					['"\\u{}"', ''],
+					['"\\u{1234567}"', '1234567'],
+					['"\\u{110000}"', '110000'],
+					['"\\u{12G}"', '12G'],
+					['"\\u{12"', '12'],
+					['"\\u{12', '12'],
+				]) {
+					let error: unknown
+					try {
+						makeReader(bad).readValue()
+					} catch (err) {
+						error = err
+					}
+
+					expect(error).toBeInstanceOf(LocalizedError)
+					expect((error as LocalizedError).lex).toBe(
+						'lexerInvalidUnicodeEscape'
+					)
+					expect((error as LocalizedError).args).toEqual({ arg })
+				}
+			})
+
 			it('parse a line break', function (): void {
 				expect(makeReader('"Hell\\b"').readValue()).toEqual(
 					HStr.make('Hell\b')
@@ -127,6 +170,18 @@ describe('ZincReader', function (): void {
 				)
 			})
 
+			it('parse a \\u{x} unicode character', function (): void {
+				expect(makeReader('`Hell\\u{D3}`').readValue()).toEqual(
+					HUri.make('HellÓ')
+				)
+			})
+
+			it('parse a \\u{x} character above U+FFFF', function (): void {
+				expect(makeReader('`a\\u{1F600}b`').readValue()).toEqual(
+					HUri.make('a😀b')
+				)
+			})
+
 			it('throws an error for a missing tick', function (): void {
 				expect((): void => {
 					makeReader('`/foo/bag').readValue()

spec/filter/FilterLexer.spec.ts

@@ -106,6 +106,12 @@ describe('FilterLexer', function (): void {
 				expect(token.toString()).toBe('HellÓ')
 			})
 
+			it('parse a \\u{x} unicode character', function (): void {
+				const token = makeLexer('"a\\u{1F600}b"').nextToken()
+				expect(token.type).toBe(TokenType.string)
+				expect(token.toString()).toBe('a😀b')
+			})
+
 			it('parse a line break', function (): void {
 				const token = makeLexer('"Hell\\b"').nextToken()
 				expect(token.type).toBe(TokenType.string)
@@ -184,6 +190,12 @@ describe('FilterLexer', function (): void {
 				expect(token.toString()).toBe('HellÓ')
 			})
 
+			it('parse a \\u{x} unicode character', function (): void {
+				const token = makeLexer('`a\\u{1F600}b`').nextToken()
+				expect(token.type).toBe(TokenType.uri)
+				expect(token.toString()).toBe('a😀b')
+			})
+
 			it('throws an error for a missing tick', function (): void {
 				expect((): void => {
 					makeLexer('`/foo/bag').nextToken()

src/core/HNum.ts

@@ -237,7 +237,7 @@ export class HNum implements HVal {
 			return POSITIVE_INFINITY_ZINC
 		} else if (this.value === Number.NEGATIVE_INFINITY) {
 			return NEGATIVE_INFINITY_ZINC
-		} else if (isNaN(this.value)) {
+		} else if (Number.isNaN(this.value)) {
 			return NOT_A_NUMBER_ZINC
 		} else {
 			const value = this.value.toLocaleString(locale, {
@@ -291,7 +291,7 @@ export class HNum implements HVal {
 			return POSITIVE_INFINITY_ZINC
 		} else if (this.value === Number.NEGATIVE_INFINITY) {
 			return NEGATIVE_INFINITY_ZINC
-		} else if (isNaN(this.value)) {
+		} else if (Number.isNaN(this.value)) {
 			return NOT_A_NUMBER_ZINC
 		} else {
 			return this.#unitSymbol
@@ -380,7 +380,7 @@ export class HNum implements HVal {
 			val = POSITIVE_INFINITY_ZINC
 		} else if (this.value === Number.NEGATIVE_INFINITY) {
 			val = NEGATIVE_INFINITY_ZINC
-		} else if (isNaN(this.value)) {
+		} else if (Number.isNaN(this.value)) {
 			val = NOT_A_NUMBER_ZINC
 		} else {
 			val = this.value

src/core/HStr.ts

@@ -10,6 +10,7 @@ import {
 	valueIsKind,
 	valueMatches,
 	TEXT_ENCODER,
+	toUnicodeEscape,
 } from './HVal'
 import { Node } from '../filter/Node'
 import { HGrid } from './grid/HGrid'
@@ -203,14 +204,8 @@ export class HStr implements HVal {
 						buf += '$'
 						break
 					default:
-						let seq = code.toString(16)
-
-						// Make sure we have four characters for the unicode sequence.
-						while (seq.length < 4) {
-							seq = `0${seq}`
-						}
-
-						buf += `u${seq}`
+						// The leading backslash has already been written.
+						buf += toUnicodeEscape(c).substring(1)
 				}
 			} else {
 				buf += c

src/core/HTime.ts

@@ -80,7 +80,7 @@ export class HTime implements HVal {
 
 	private static toJsDate(value: string): Date {
 		const time = Date.parse('1970-01-01T' + value + 'Z')
-		if (isNaN(time)) {
+		if (Number.isNaN(time)) {
 			throw new Error(`Invalid Time format ${value}`)
 		}
 		return new Date(time)

src/core/HUnit.ts

@@ -251,7 +251,7 @@ export class HUnit {
 	private static parseNumber(str: string): number {
 		const num = Number(str.trim())
 
-		if (isNaN(num) || typeof num !== 'number') {
+		if (Number.isNaN(num) || typeof num !== 'number') {
 			throw new Error(`Unable to convert '${str}' to a number`)
 		}
 

src/core/HUri.ts

@@ -10,6 +10,7 @@ import {
 	valueIsKind,
 	valueMatches,
 	TEXT_ENCODER,
+	toUnicodeEscape,
 } from './HVal'
 import { HaysonUri } from './hayson'
 import { Node } from '../filter/Node'
@@ -202,14 +203,8 @@ export class HUri implements HVal {
 						buf += '`'
 						break
 					default:
-						let seq = code.toString(16)
-
-						// Make sure we have four characters for the unicode sequence.
-						while (seq.length < 4) {
-							seq = `0${seq}`
-						}
-
-						buf += `u${seq}`
+						// The leading backslash has already been written.
+						buf += toUnicodeEscape(c).substring(1)
 				}
 			} else {
 				buf += c

src/core/HVal.ts

@@ -123,6 +123,23 @@ export function valueToZinc(val: HVal | null): string {
 	return val?.toZinc() ?? ZINC_NULL
 }
 
+/**
+ * Returns a character as a `\uXXXX` escape sequence.
+ *
+ * Characters above U+FFFF are returned as a surrogate pair of escapes
+ * so they can be read by older readers that don't support `\u{x}`.
+ *
+ * @param c The character to escape.
+ * @returns The escape sequence.
+ */
+export function toUnicodeEscape(c: string): string {
+	let esc = ''
+	for (let i = 0; i < c.length; ++i) {
+		esc += `\\u${c.charCodeAt(i).toString(16).padStart(4, '0')}`
+	}
+	return esc
+}
+
 /**
  * The Axon symbol for null
  */

src/core/ZincReader.ts

@@ -328,6 +328,19 @@ export class ZincReader {
 		return HRef.make(str, displayName)
 	}
 
+	/**
+	 * @param esc The invalid escape characters.
+	 * @returns An error for an invalid unicode escape.
+	 */
+	private invalidUnicodeEscape(esc: string): LocalizedError {
+		return new LocalizedError({
+			message: `Invalid unicode escape ${esc}`,
+			lex: 'lexerInvalidUnicodeEscape',
+			args: { arg: esc },
+			index: this.scanner.index,
+		})
+	}
+
 	/**
 	 * Consume an escape sequence within a string.
 	 *
@@ -362,24 +375,54 @@ export class ZincReader {
 			}
 		}
 
-		if (this.scanner.is('u')) {
+		if (this.scanner.is('u') && this.scanner.next === '{') {
+			// Handle \u{x} where x is 1 to 6 hex digits.
+			this.scanner.consume() // u
+			this.scanner.consume() // {
+			let esc = ''
+
+			while (!this.scanner.is('}')) {
+				if (!this.scanner.isHex()) {
+					// Only report the bad char if it's part of the escape.
+					throw this.invalidUnicodeEscape(
+						this.scanner.isEof() ||
+							this.scanner.is('"') ||
+							this.scanner.is('`')
+							? esc
+							: esc + this.scanner.current
+					)
+				}
+
+				esc += this.scanner.current
+				if (esc.length > 6) {
+					throw this.invalidUnicodeEscape(esc)
+				}
+				this.scanner.consume()
+			}
+
+			const codePoint = Number.parseInt(esc, 16)
+			if (
+				Number.isNaN(codePoint) ||
+				esc.length === 0 ||
+				codePoint > 0x10ffff
+			) {
+				throw this.invalidUnicodeEscape(esc)
+			}
+
+			return String.fromCodePoint(codePoint)
+		} else if (this.scanner.is('u')) {
 			let esc = ''
 
 			for (let i = 0; i < 4; ++i) {
 				this.scanner.consume()
 				esc += this.scanner.current
 
 				if (!this.scanner.isHex()) {
-					throw new LocalizedError({
-						message: `Invalid unicode escape ${esc}`,
-						lex: 'lexerInvalidUnicodeEscape',
-						args: { arg: esc },
-						index: this.scanner.index,
-					})
+					throw this.invalidUnicodeEscape(esc)
 				}
 			}
 
-			return String.fromCharCode(parseInt(esc, 16))
+			return String.fromCharCode(Number.parseInt(esc, 16))
 		} else {
 			throw new LocalizedError({
 				message: `Invalid escape sequence ${this.scanner.current}`,
@@ -494,7 +537,7 @@ export class ZincReader {
 			}
 
 			// If we have a number then attempt to parse it and throw any errors.
-			if (isNaN(Number(str))) {
+			if (Number.isNaN(Number(str))) {
 				throw new LocalizedError({
 					message: `Invalid number: ${str}`,
 					lex: 'lexerInvalidNumber',
@@ -622,7 +665,7 @@ export class ZincReader {
 
 		const num = Number(buf)
 
-		if (isNaN(num)) {
+		if (Number.isNaN(num)) {
 			throw new LocalizedError({
 				message: `Invalid decimal number ${buf}`,
 				lex: 'lexerInvalidDecimal',