From 2e03762232accffc7229183863c66d8df69f39cf Mon Sep 17 00:00:00 2001 From: Gareth Johnson <75035963+garethj2@users.noreply.github.com> Date: Thu, 29 Oct 2026 09:59:31 +0000 Subject: [PATCH] Support new unicode syntax --- .../encoding/zinc/decode/scalar/str.rs | 146 ++++++++++++++++-- .../encoding/zinc/decode/scalar/uri.rs | 26 +++- src/haystack/encoding/zinc/decode/scanner.rs | 42 +++++ src/haystack/encoding/zinc/encode.rs | 7 +- src/haystack/filter/lexer.rs | 18 +++ tests/zinc/test_string.rs | 15 ++ tests/zinc/test_uri.rs | 18 +++ 7 files changed, 256 insertions(+), 16 deletions(-) diff --git a/src/haystack/encoding/zinc/decode/scalar/str.rs b/src/haystack/encoding/zinc/decode/scalar/str.rs index a59f4a0..e2ef67a 100644 --- a/src/haystack/encoding/zinc/decode/scalar/str.rs +++ b/src/haystack/encoding/zinc/decode/scalar/str.rs @@ -57,33 +57,99 @@ pub(super) fn parse_str_escape(scanner: &mut Scanner) -> Result Ok("`".into()), b'\\' => Ok("\\".into()), b'u' => Ok(parse_str_unicode_escape(scanner)?), - _ => scanner.make_generic_err(&format!("Invalid escape sequence '{}'", scanner.cur)), + _ => scanner.make_generic_err(&format!( + "Invalid escape sequence '{}'", + scanner.cur as char + )), } } -// Parse a Str unicode char escape sequence +// Parse a Str unicode char escape sequence, either `\uXXXX` or `\u{X}`. +// Surrogate pairs written as two `\uXXXX` escapes are joined into one char. pub(super) fn parse_str_unicode_escape(scanner: &mut Scanner) -> Result { - if scanner.cur == b'u' { - let mut esc = Vec::new(); + if scanner.cur != b'u' { + return scanner.make_generic_err(&format!( + "Invalid escape sequence '{}'", + scanner.cur as char + )); + } - for _ in 0..4 { + let code = parse_unicode_code(scanner)?; + + if (0xD800..=0xDBFF).contains(&code) && scanner.next_is(b"\\u") { + scanner.read()?; // \ + scanner.read()?; // u + let low = parse_unicode_code(scanner)?; + + if (0xDC00..=0xDFFF).contains(&low) { + let code_point = 0x10000 + ((code - 0xD800) << 10) + (low - 0xDC00); + return Ok(to_char(code_point).to_string()); + } + + return Ok(format!("{}{}", to_char(code), to_char(low))); + } + + Ok(to_char(code).to_string()) +} + +// Convert a code point to a char, using the replacement char for lone surrogates. +fn to_char(code: u32) -> char { + char::from_u32(code).unwrap_or(char::REPLACEMENT_CHARACTER) +} + +// Read the code of a `\uXXXX` or `\u{X}` escape. The scanner must be on the `u` +// and is left on the last character of the escape. +fn parse_unicode_code(scanner: &mut Scanner) -> Result { + let mut esc = String::new(); + + if scanner.next_is(b"{") { + scanner.read()?; // { + + loop { scanner.read()?; + if scanner.cur == b'}' { + break; + } if !scanner.is_hex_digit() { - return scanner - .make_generic_err(&format!("Invalid unicode escape '{}'", scanner.cur)); + return scanner.make_generic_err(&format!( + "Invalid unicode escape '{esc}{}'", + scanner.cur as char + )); } - esc.push(scanner.cur) + if esc.len() == 6 { + return scanner.make_generic_err(&format!( + "Invalid unicode escape '{esc}{}...', must have 1 to 6 hex digits", + scanner.cur as char + )); + } + esc.push(scanner.cur as char) } - let str = String::from_utf8_lossy(&esc).to_string(); - match u16::from_str_radix(&str, 16) { - Ok(char) => Ok(String::from_utf16_lossy(&[char])), - Err(err) => scanner - .make_generic_err(&format!("Invalid unicode escape char '{str}', err: {err}")), + if esc.is_empty() { + return scanner + .make_generic_err("Invalid unicode escape '{}', must have 1 to 6 hex digits"); } } else { - scanner.make_generic_err(&format!("Invalid escape sequence '{}'", scanner.cur)) + for _ in 0..4 { + scanner.read()?; + + if !scanner.is_hex_digit() { + return scanner.make_generic_err(&format!( + "Invalid unicode escape '{}'", + scanner.cur as char + )); + } + esc.push(scanner.cur as char) + } + } + + match u32::from_str_radix(&esc, 16) { + Ok(code) if code <= 0x10FFFF => Ok(code), + Ok(_) => scanner.make_generic_err(&format!("Invalid unicode escape '{esc}', out of range")), + Err(err) => { + scanner.make_generic_err(&format!("Invalid unicode escape '{esc}', err: {err}")) + } } } @@ -149,4 +215,56 @@ mod test { let str = parse_str(&mut scanner); assert_eq!(str.ok(), Some(Str::make("\x08\x0c"))); } + + fn parse(input: &str) -> Result { + let mut input = Cursor::new(input.as_bytes()); + let mut scanner = super::Scanner::make(&mut input).expect("Scanner"); + parse_str(&mut scanner) + } + + #[test] + fn test_zinc_parse_str_unicode_escapes() { + assert_eq!(parse(r#""Hell\u00D3""#).ok(), Some(Str::make("Hellร“"))); + assert_eq!(parse(r#""Hell\u{D3}!""#).ok(), Some(Str::make("Hellร“!"))); + assert_eq!(parse(r#""a\u{1F600}b""#).ok(), Some(Str::make("a๐Ÿ˜€b"))); + assert_eq!( + parse(r#""\u{0}\u{10FFFF}""#).ok(), + Some(Str::make("\0\u{10FFFF}")) + ); + } + + #[test] + fn test_zinc_parse_str_surrogate_pairs() { + assert_eq!(parse(r#""a\ud83d\ude00b""#).ok(), Some(Str::make("a๐Ÿ˜€b"))); + assert_eq!( + parse(r#""a\u{D83D}\u{DE00}b""#).ok(), + Some(Str::make("a๐Ÿ˜€b")) + ); + assert_eq!(parse(r#""a๐Ÿ˜€b""#).ok(), Some(Str::make("a๐Ÿ˜€b"))); + + // Lone and mismatched surrogates become replacement characters. + assert_eq!(parse(r#""\ud83d""#).ok(), Some(Str::make("\u{FFFD}"))); + assert_eq!(parse(r#""\ud83d\n""#).ok(), Some(Str::make("\u{FFFD}\n"))); + assert_eq!( + parse(r#""\ud83d\u0041""#).ok(), + Some(Str::make("\u{FFFD}A")) + ); + } + + #[test] + fn test_zinc_parse_str_bad_unicode_escapes() { + for bad in [ + r#""\u{}""#, + r#""\u{1234567}""#, + r#""\u{110000}""#, + r#""\u{12G}""#, + r#""\u{12"#, + r#""\u12G4""#, + r#""\u12"#, + r#""\ud83d\"#, + r#""\ud83d"#, + ] { + assert!(parse(bad).is_err(), "{bad} should fail"); + } + } } diff --git a/src/haystack/encoding/zinc/decode/scalar/uri.rs b/src/haystack/encoding/zinc/decode/scalar/uri.rs index 2a506a0..b908565 100644 --- a/src/haystack/encoding/zinc/decode/scalar/uri.rs +++ b/src/haystack/encoding/zinc/decode/scalar/uri.rs @@ -31,10 +31,17 @@ pub(crate) fn parse_uri(scanner: &mut Scanner) -> Result str.push(next); scanner.read()?; } - _ => { + b'u' => { + scanner.read()?; // u let unicode = parse_str_unicode_escape(scanner)?; str.extend_from_slice(unicode.as_bytes()); } + _ => { + return scanner.make_generic_err(&format!( + "Invalid Uri escape sequence '{}'", + next as char + )); + } }; } else { str.push(scanner.cur); @@ -78,6 +85,23 @@ mod test { } } + fn parse(input: &str) -> Result { + let mut input = Cursor::new(input.as_bytes()); + let mut scanner = super::Scanner::make(&mut input).expect("Scanner"); + parse_uri(&mut scanner) + } + + #[test] + fn test_zinc_parse_uri_unicode_escape() { + assert_eq!(parse(r#"`/f\u1ff6o`"#).ok(), Some(Uri::make("/fแฟถo"))); + assert_eq!(parse(r#"`a\u{1F600}b`"#).ok(), Some(Uri::make("a๐Ÿ˜€b"))); + assert_eq!(parse(r#"`a\ud83d\ude00b`"#).ok(), Some(Uri::make("a๐Ÿ˜€b"))); + assert_eq!(parse(r#"`a๐Ÿ˜€b`"#).ok(), Some(Uri::make("a๐Ÿ˜€b"))); + assert!(parse(r#"`a\x`"#).is_err()); + assert!(parse(r#"`a\nb`"#).is_err()); + assert!(parse(r#"`a\ud83d`"#).is_ok()); + } + #[test] fn test_zinc_parse_uri_bad() { let mut input = Cursor::new("`".as_bytes()); diff --git a/src/haystack/encoding/zinc/decode/scanner.rs b/src/haystack/encoding/zinc/decode/scanner.rs index 69c1b04..d05b845 100644 --- a/src/haystack/encoding/zinc/decode/scanner.rs +++ b/src/haystack/encoding/zinc/decode/scanner.rs @@ -273,6 +273,26 @@ impl<'a, R: Read> Scanner<'a, R> { Some(self.buffer_peek(next)) } + /// Returns true if the bytes after the current one match `seq`, without + /// consuming them. Bytes already stashed by a previous peek are compared + /// first, and no further input is read after the first mismatch. + pub fn next_is(&mut self, seq: &[u8]) -> bool { + for (i, expected) in seq.iter().enumerate() { + let byte = match self.next.as_ref().and_then(|next| next.get(i)) { + Some(byte) => *byte, + None => match self.safe_peek() { + Some(byte) => byte, + None => return false, + }, + }; + + if byte != *expected { + return false; + } + } + true + } + /// Create am error for an missing expect condition /// Current position and the current char are captured pub fn make_expect_err(&self, item: E) -> Result { @@ -396,6 +416,28 @@ mod test { assert!(scanner.peek().is_err()); } + #[test] + fn test_zinc_scanner_next_is() { + let mut input = Cursor::new("abcdef".as_bytes()); + let mut scanner = super::Scanner::make(&mut input).expect("Scanner"); + + assert!(scanner.next_is(b"bc")); + assert!(!scanner.next_is(b"bx")); + + // Stashed bytes are compared rather than skipped and are still read in order. + assert_eq!(scanner.peek().ok(), Some(b'd')); + assert!(scanner.next_is(b"bcd")); + assert_eq!(scanner.read().ok(), Some(b'b')); + assert!(scanner.next_is(b"cd")); + assert_eq!(scanner.read().ok(), Some(b'c')); + assert_eq!(scanner.read().ok(), Some(b'd')); + + // Stops at the end of input without matching. + assert!(!scanner.next_is(b"efg")); + assert_eq!(scanner.read().ok(), Some(b'e')); + assert_eq!(scanner.read().ok(), Some(b'f')); + } + #[test] fn test_zinc_scanner_is_any_of() { let mut input = Cursor::new("8".as_bytes()); diff --git a/src/haystack/encoding/zinc/encode.rs b/src/haystack/encoding/zinc/encode.rs index e905f25..fec43dd 100644 --- a/src/haystack/encoding/zinc/encode.rs +++ b/src/haystack/encoding/zinc/encode.rs @@ -240,7 +240,12 @@ impl ToZinc for Uri { '`' => writer.write_all(br"\`")?, '\\' => writer.write_all(br"\\")?, '\x20'..='\x7e' => writer.write_all(&[c as u8])?, - _ => writer.write_fmt(format_args!("\\u{:04x}", c as u32))?, + _ => { + // Write chars above U+FFFF as a surrogate pair. + for unit in c.encode_utf16(&mut [0u16; 2]) { + writer.write_fmt(format_args!("\\u{:04x}", unit))? + } + } } } writer.write_all(b"`")?; diff --git a/src/haystack/filter/lexer.rs b/src/haystack/filter/lexer.rs index f0b5788..914eda9 100644 --- a/src/haystack/filter/lexer.rs +++ b/src/haystack/filter/lexer.rs @@ -323,6 +323,15 @@ mod test { "Unicode escape" ); + let mut input = Cursor::new(r#""a \u{1F600} string""#.as_bytes()); + let mut lexer = Lexer::make(&mut input).expect("Should create lexer"); + + assert_eq!( + lexer.read().ok(), + Some(&LexerToken::make_value("a ๐Ÿ˜€ string".into())), + "Braced unicode escape" + ); + let mut input = Cursor::new("\"\\t\\n\\\\\"".as_bytes()); let mut lexer = Lexer::make(&mut input).expect("Should create lexer"); @@ -358,6 +367,15 @@ mod test { "Special characters escape" ); + let mut input = Cursor::new(r#"`/a\u{1F600}`"#.as_bytes()); + let mut lexer = Lexer::make(&mut input).expect("Should create lexer"); + + assert_eq!( + lexer.read().ok(), + Some(&LexerToken::make_value(Uri::from("/a๐Ÿ˜€").into())), + "Braced unicode escape" + ); + let mut input = Cursor::new("`".as_bytes()); let mut lexer = Lexer::make(&mut input).expect("Should create lexer"); diff --git a/tests/zinc/test_string.rs b/tests/zinc/test_string.rs index ed21419..cceafcb 100644 --- a/tests/zinc/test_string.rs +++ b/tests/zinc/test_string.rs @@ -43,3 +43,18 @@ fn test_zinc_str_decode() { fn test_zinc_str_decode_bad() { assert!(from_str(r#""non terminated"#).is_err()); } + +#[test] +fn test_zinc_str_unicode_escapes() { + assert_eq!(from_str(r#""\u{1F600}""#).unwrap(), Value::make_str("๐Ÿ˜€")); + assert_eq!( + from_str(r#""\ud83d\ude00""#).unwrap(), + Value::make_str("๐Ÿ˜€") + ); + + // Non-ASCII is written as UTF-8 and round trips. + let value = Value::make_str("a๐Ÿ˜€b"); + let zinc = to_zinc_string(&value).unwrap(); + assert_eq!(zinc, r#""a๐Ÿ˜€b""#); + assert_eq!(from_str(&zinc).unwrap(), value); +} diff --git a/tests/zinc/test_uri.rs b/tests/zinc/test_uri.rs index f88d4e9..817a475 100644 --- a/tests/zinc/test_uri.rs +++ b/tests/zinc/test_uri.rs @@ -25,3 +25,21 @@ fn test_zinc_uri_decode() { fn test_zinc_uri_decode_bad() { assert!(from_str(r#"`^"#).is_err()); } + +#[test] +fn test_zinc_uri_unicode() { + let value = Value::make_uri("/fแฟถo"); + let zinc = to_zinc_string(&value).unwrap(); + assert_eq!(zinc, r#"`/f\u1ff6o`"#); + assert_eq!(from_str(&zinc).unwrap(), value); + + let value = Value::make_uri("a๐Ÿ˜€b"); + let zinc = to_zinc_string(&value).unwrap(); + assert_eq!(zinc, r#"`a\ud83d\ude00b`"#); + assert_eq!(from_str(&zinc).unwrap(), value); + + assert_eq!( + from_str(r#"`a\u{1F600}b`"#).unwrap(), + Value::make_uri("a๐Ÿ˜€b") + ); +}