diff --git a/spec/core/HStr.spec.ts b/spec/core/HStr.spec.ts index ee5cfc0..df8cd9c 100644 --- a/spec/core/HStr.spec.ts +++ b/spec/core/HStr.spec.ts @@ -3,6 +3,7 @@ */ import { HStr } from '../../src/core/HStr' +import { ZincReader } from '../../src/core/ZincReader' import { Kind } from '../../src/core/Kind' import { HGrid } from '../../src/core/grid/HGrid' import { HList } from '../../src/core/list/HList' @@ -160,6 +161,15 @@ describe('HStr', function (): void { expect(HStr.make(str).toZinc()).toBe('"' + str + '"') }) + it('encodes characters above U+FFFF as a surrogate pair', function (): void { + expect(HStr.make('a😀b').toZinc()).toBe('"a\\ud83d\\ude00b"') + }) + + it('round trips characters above U+FFFF', function (): void { + const str = HStr.make('a😀b') + expect(ZincReader.readValue(str.toZinc())).toEqual(str) + }) + it('encodes non-ascii characters to unicode', function (): void { expect(HStr.make('ݧݩᴆᴁaḀCD').toZinc()).toBe( '"\\u0767\\u0769\\u1d06\\u1d01a\\u1e00CD"' diff --git a/spec/core/HUri.spec.ts b/spec/core/HUri.spec.ts index 00da5d9..e4c7211 100644 --- a/spec/core/HUri.spec.ts +++ b/spec/core/HUri.spec.ts @@ -3,6 +3,7 @@ */ import { HUri } from '../../src/core/HUri' +import { ZincReader } from '../../src/core/ZincReader' import { Kind } from '../../src/core/Kind' import { HGrid } from '../../src/core/grid/HGrid' import { HList } from '../../src/core/list/HList' @@ -121,6 +122,15 @@ describe('HUri', function (): void { expect(HUri.make(str).toZinc()).toBe('`' + str + '`') }) + it('encodes characters above U+FFFF as a surrogate pair', function (): void { + expect(HUri.make('a😀b').toZinc()).toBe('`a\\ud83d\\ude00b`') + }) + + it('round trips characters above U+FFFF', function (): void { + const uri = HUri.make('a😀b') + expect(ZincReader.readValue(uri.toZinc())).toEqual(uri) + }) + it('encodes non-ascii characters to unicode', function (): void { expect(HUri.make('ݧݩᴆᴁaḀCD').toZinc()).toBe( '`\\u0767\\u0769\\u1d06\\u1d01a\\u1e00CD`' diff --git a/spec/core/HVal.spec.ts b/spec/core/HVal.spec.ts index 0cbf4c2..a8d1d2a 100644 --- a/spec/core/HVal.spec.ts +++ b/spec/core/HVal.spec.ts @@ -10,6 +10,7 @@ import { valueEquals, valueToZinc, valueToAxon, + toUnicodeEscape, HVal, } from '../../src/core/HVal' import { HBool } from '../../src/core/HBool' @@ -104,4 +105,15 @@ describe('HVal', function (): void { expect(valueToAxon(null)).toBe('null') }) }) // .valueToAxon() + + describe('.toUnicodeEscape()', function (): void { + it('returns a four digit escape', function (): void { + expect(toUnicodeEscape('\u0001')).toBe('\\u0001') + expect(toUnicodeEscape('ῶ')).toBe('\\u1ff6') + }) + + it('returns a surrogate pair for characters above U+FFFF', function (): void { + expect(toUnicodeEscape('😀')).toBe('\\ud83d\\ude00') + }) + }) // .toUnicodeEscape() }) diff --git a/spec/core/ZincReader.spec.ts b/spec/core/ZincReader.spec.ts index dfa036f..c4df9bf 100644 --- a/spec/core/ZincReader.spec.ts +++ b/spec/core/ZincReader.spec.ts @@ -23,6 +23,7 @@ import { HList } from '../../src/core/list/HList' import { HGrid } from '../../src/core/grid/HGrid' import { HVal } from '../../src/core/HVal' import { readFile } from '../file' +import { LocalizedError } from '../../src/util/LocalizedError' describe('ZincReader', function (): void { function makeReader(input: string): ZincReader { @@ -61,6 +62,48 @@ describe('ZincReader', function (): void { ) }) + it('parse a \\u{x} unicode character', function (): void { + expect(makeReader('"Hell\\u{D3}"').readValue()).toEqual( + HStr.make('HellÓ') + ) + }) + + it('parse a \\u{x} character above U+FFFF', function (): void { + expect(makeReader('"a\\u{1F600}b"').readValue()).toEqual( + HStr.make('a😀b') + ) + }) + + it('parse the smallest and largest \\u{x} values', function (): void { + expect(makeReader('"\\u{0}\\u{10FFFF}"').readValue()).toEqual( + HStr.make('\u0000\u{10FFFF}') + ) + }) + + it('throws an error for an invalid \\u{x} escape', function (): void { + for (const [bad, arg] of [ + ['"\\u{}"', ''], + ['"\\u{1234567}"', '1234567'], + ['"\\u{110000}"', '110000'], + ['"\\u{12G}"', '12G'], + ['"\\u{12"', '12'], + ['"\\u{12', '12'], + ]) { + let error: unknown + try { + makeReader(bad).readValue() + } catch (err) { + error = err + } + + expect(error).toBeInstanceOf(LocalizedError) + expect((error as LocalizedError).lex).toBe( + 'lexerInvalidUnicodeEscape' + ) + expect((error as LocalizedError).args).toEqual({ arg }) + } + }) + it('parse a line break', function (): void { expect(makeReader('"Hell\\b"').readValue()).toEqual( HStr.make('Hell\b') @@ -127,6 +170,18 @@ describe('ZincReader', function (): void { ) }) + it('parse a \\u{x} unicode character', function (): void { + expect(makeReader('`Hell\\u{D3}`').readValue()).toEqual( + HUri.make('HellÓ') + ) + }) + + it('parse a \\u{x} character above U+FFFF', function (): void { + expect(makeReader('`a\\u{1F600}b`').readValue()).toEqual( + HUri.make('a😀b') + ) + }) + it('throws an error for a missing tick', function (): void { expect((): void => { makeReader('`/foo/bag').readValue() diff --git a/spec/filter/FilterLexer.spec.ts b/spec/filter/FilterLexer.spec.ts index 8253d63..76eecfa 100644 --- a/spec/filter/FilterLexer.spec.ts +++ b/spec/filter/FilterLexer.spec.ts @@ -106,6 +106,12 @@ describe('FilterLexer', function (): void { expect(token.toString()).toBe('HellÓ') }) + it('parse a \\u{x} unicode character', function (): void { + const token = makeLexer('"a\\u{1F600}b"').nextToken() + expect(token.type).toBe(TokenType.string) + expect(token.toString()).toBe('a😀b') + }) + it('parse a line break', function (): void { const token = makeLexer('"Hell\\b"').nextToken() expect(token.type).toBe(TokenType.string) @@ -184,6 +190,12 @@ describe('FilterLexer', function (): void { expect(token.toString()).toBe('HellÓ') }) + it('parse a \\u{x} unicode character', function (): void { + const token = makeLexer('`a\\u{1F600}b`').nextToken() + expect(token.type).toBe(TokenType.uri) + expect(token.toString()).toBe('a😀b') + }) + it('throws an error for a missing tick', function (): void { expect((): void => { makeLexer('`/foo/bag').nextToken() diff --git a/src/core/HNum.ts b/src/core/HNum.ts index cac5cae..1e6af43 100644 --- a/src/core/HNum.ts +++ b/src/core/HNum.ts @@ -237,7 +237,7 @@ export class HNum implements HVal { return POSITIVE_INFINITY_ZINC } else if (this.value === Number.NEGATIVE_INFINITY) { return NEGATIVE_INFINITY_ZINC - } else if (isNaN(this.value)) { + } else if (Number.isNaN(this.value)) { return NOT_A_NUMBER_ZINC } else { const value = this.value.toLocaleString(locale, { @@ -291,7 +291,7 @@ export class HNum implements HVal { return POSITIVE_INFINITY_ZINC } else if (this.value === Number.NEGATIVE_INFINITY) { return NEGATIVE_INFINITY_ZINC - } else if (isNaN(this.value)) { + } else if (Number.isNaN(this.value)) { return NOT_A_NUMBER_ZINC } else { return this.#unitSymbol @@ -380,7 +380,7 @@ export class HNum implements HVal { val = POSITIVE_INFINITY_ZINC } else if (this.value === Number.NEGATIVE_INFINITY) { val = NEGATIVE_INFINITY_ZINC - } else if (isNaN(this.value)) { + } else if (Number.isNaN(this.value)) { val = NOT_A_NUMBER_ZINC } else { val = this.value diff --git a/src/core/HStr.ts b/src/core/HStr.ts index b06879b..f64f5ac 100644 --- a/src/core/HStr.ts +++ b/src/core/HStr.ts @@ -10,6 +10,7 @@ import { valueIsKind, valueMatches, TEXT_ENCODER, + toUnicodeEscape, } from './HVal' import { Node } from '../filter/Node' import { HGrid } from './grid/HGrid' @@ -203,14 +204,8 @@ export class HStr implements HVal { buf += '$' break default: - let seq = code.toString(16) - - // Make sure we have four characters for the unicode sequence. - while (seq.length < 4) { - seq = `0${seq}` - } - - buf += `u${seq}` + // The leading backslash has already been written. + buf += toUnicodeEscape(c).substring(1) } } else { buf += c diff --git a/src/core/HTime.ts b/src/core/HTime.ts index 9714d53..955a3bc 100644 --- a/src/core/HTime.ts +++ b/src/core/HTime.ts @@ -80,7 +80,7 @@ export class HTime implements HVal { private static toJsDate(value: string): Date { const time = Date.parse('1970-01-01T' + value + 'Z') - if (isNaN(time)) { + if (Number.isNaN(time)) { throw new Error(`Invalid Time format ${value}`) } return new Date(time) diff --git a/src/core/HUnit.ts b/src/core/HUnit.ts index 89a09a7..f788dd4 100644 --- a/src/core/HUnit.ts +++ b/src/core/HUnit.ts @@ -251,7 +251,7 @@ export class HUnit { private static parseNumber(str: string): number { const num = Number(str.trim()) - if (isNaN(num) || typeof num !== 'number') { + if (Number.isNaN(num) || typeof num !== 'number') { throw new Error(`Unable to convert '${str}' to a number`) } diff --git a/src/core/HUri.ts b/src/core/HUri.ts index 5caf62e..e90c271 100644 --- a/src/core/HUri.ts +++ b/src/core/HUri.ts @@ -10,6 +10,7 @@ import { valueIsKind, valueMatches, TEXT_ENCODER, + toUnicodeEscape, } from './HVal' import { HaysonUri } from './hayson' import { Node } from '../filter/Node' @@ -202,14 +203,8 @@ export class HUri implements HVal { buf += '`' break default: - let seq = code.toString(16) - - // Make sure we have four characters for the unicode sequence. - while (seq.length < 4) { - seq = `0${seq}` - } - - buf += `u${seq}` + // The leading backslash has already been written. + buf += toUnicodeEscape(c).substring(1) } } else { buf += c diff --git a/src/core/HVal.ts b/src/core/HVal.ts index 2788284..ada25eb 100644 --- a/src/core/HVal.ts +++ b/src/core/HVal.ts @@ -123,6 +123,23 @@ export function valueToZinc(val: HVal | null): string { return val?.toZinc() ?? ZINC_NULL } +/** + * Returns a character as a `\uXXXX` escape sequence. + * + * Characters above U+FFFF are returned as a surrogate pair of escapes + * so they can be read by older readers that don't support `\u{x}`. + * + * @param c The character to escape. + * @returns The escape sequence. + */ +export function toUnicodeEscape(c: string): string { + let esc = '' + for (let i = 0; i < c.length; ++i) { + esc += `\\u${c.charCodeAt(i).toString(16).padStart(4, '0')}` + } + return esc +} + /** * The Axon symbol for null */ diff --git a/src/core/ZincReader.ts b/src/core/ZincReader.ts index 14bcbf4..b443146 100644 --- a/src/core/ZincReader.ts +++ b/src/core/ZincReader.ts @@ -328,6 +328,19 @@ export class ZincReader { return HRef.make(str, displayName) } + /** + * @param esc The invalid escape characters. + * @returns An error for an invalid unicode escape. + */ + private invalidUnicodeEscape(esc: string): LocalizedError { + return new LocalizedError({ + message: `Invalid unicode escape ${esc}`, + lex: 'lexerInvalidUnicodeEscape', + args: { arg: esc }, + index: this.scanner.index, + }) + } + /** * Consume an escape sequence within a string. * @@ -362,7 +375,42 @@ export class ZincReader { } } - if (this.scanner.is('u')) { + if (this.scanner.is('u') && this.scanner.next === '{') { + // Handle \u{x} where x is 1 to 6 hex digits. + this.scanner.consume() // u + this.scanner.consume() // { + let esc = '' + + while (!this.scanner.is('}')) { + if (!this.scanner.isHex()) { + // Only report the bad char if it's part of the escape. + throw this.invalidUnicodeEscape( + this.scanner.isEof() || + this.scanner.is('"') || + this.scanner.is('`') + ? esc + : esc + this.scanner.current + ) + } + + esc += this.scanner.current + if (esc.length > 6) { + throw this.invalidUnicodeEscape(esc) + } + this.scanner.consume() + } + + const codePoint = Number.parseInt(esc, 16) + if ( + Number.isNaN(codePoint) || + esc.length === 0 || + codePoint > 0x10ffff + ) { + throw this.invalidUnicodeEscape(esc) + } + + return String.fromCodePoint(codePoint) + } else if (this.scanner.is('u')) { let esc = '' for (let i = 0; i < 4; ++i) { @@ -370,16 +418,11 @@ export class ZincReader { esc += this.scanner.current if (!this.scanner.isHex()) { - throw new LocalizedError({ - message: `Invalid unicode escape ${esc}`, - lex: 'lexerInvalidUnicodeEscape', - args: { arg: esc }, - index: this.scanner.index, - }) + throw this.invalidUnicodeEscape(esc) } } - return String.fromCharCode(parseInt(esc, 16)) + return String.fromCharCode(Number.parseInt(esc, 16)) } else { throw new LocalizedError({ message: `Invalid escape sequence ${this.scanner.current}`, @@ -494,7 +537,7 @@ export class ZincReader { } // If we have a number then attempt to parse it and throw any errors. - if (isNaN(Number(str))) { + if (Number.isNaN(Number(str))) { throw new LocalizedError({ message: `Invalid number: ${str}`, lex: 'lexerInvalidNumber', @@ -622,7 +665,7 @@ export class ZincReader { const num = Number(buf) - if (isNaN(num)) { + if (Number.isNaN(num)) { throw new LocalizedError({ message: `Invalid decimal number ${buf}`, lex: 'lexerInvalidDecimal', diff --git a/src/filter/FilterParser.ts b/src/filter/FilterParser.ts index 562dc58..9d48676 100644 --- a/src/filter/FilterParser.ts +++ b/src/filter/FilterParser.ts @@ -74,7 +74,7 @@ export const enum booleanOperators { * := any 16-bit Unicode char >= 0x20 (except str/uri quote) * := "\b" | "\f" | "\n" | "\r" | "\r" | "\t" | "\"" | "\\" | "\$" | * := "\:" | "\/" | "\?" | "\#" | "\[" | "\]" | "\@" | "\`" | "\\" | "\&" | "\=" | "\;" | - * := "\u" + * := "\u" | "\u{" {1,6} "}" * := ('a'-'f') | ('A'-'F') | digit * := ( | | | '_')* * := ('a' - 'z')