Support new unicode syntax

src/haystack/encoding/zinc/decode/scalar/str.rs

@@ -57,33 +57,99 @@ pub(super) fn parse_str_escape<R: Read>(scanner: &mut Scanner<R>) -> Result<Stri
         b'`' => Ok("`".into()),
         b'\\' => Ok("\\".into()),
         b'u' => Ok(parse_str_unicode_escape(scanner)?),
-        _ => scanner.make_generic_err(&format!("Invalid escape sequence '{}'", scanner.cur)),
+        _ => scanner.make_generic_err(&format!(
+            "Invalid escape sequence '{}'",
+            scanner.cur as char
+        )),
     }
 }
 
-// Parse a Str unicode char escape sequence
+// Parse a Str unicode char escape sequence, either `\uXXXX` or `\u{X}`.
+// Surrogate pairs written as two `\uXXXX` escapes are joined into one char.
 pub(super) fn parse_str_unicode_escape<R: Read>(scanner: &mut Scanner<R>) -> Result<String, Error> {
-    if scanner.cur == b'u' {
-        let mut esc = Vec::new();
+    if scanner.cur != b'u' {
+        return scanner.make_generic_err(&format!(
+            "Invalid escape sequence '{}'",
+            scanner.cur as char
+        ));
+    }
 
-        for _ in 0..4 {
+    let code = parse_unicode_code(scanner)?;
+
+    if (0xD800..=0xDBFF).contains(&code) && scanner.next_is(b"\\u") {
+        scanner.read()?; // \
+        scanner.read()?; // u
+        let low = parse_unicode_code(scanner)?;
+
+        if (0xDC00..=0xDFFF).contains(&low) {
+            let code_point = 0x10000 + ((code - 0xD800) << 10) + (low - 0xDC00);
+            return Ok(to_char(code_point).to_string());
+        }
+
+        return Ok(format!("{}{}", to_char(code), to_char(low)));
+    }
+
+    Ok(to_char(code).to_string())
+}
+
+// Convert a code point to a char, using the replacement char for lone surrogates.
+fn to_char(code: u32) -> char {
+    char::from_u32(code).unwrap_or(char::REPLACEMENT_CHARACTER)
+}
+
+// Read the code of a `\uXXXX` or `\u{X}` escape. The scanner must be on the `u`
+// and is left on the last character of the escape.
+fn parse_unicode_code<R: Read>(scanner: &mut Scanner<R>) -> Result<u32, Error> {
+    let mut esc = String::new();
+
+    if scanner.next_is(b"{") {
+        scanner.read()?; // {
+
+        loop {
             scanner.read()?;
 
+            if scanner.cur == b'}' {
+                break;
+            }
             if !scanner.is_hex_digit() {
-                return scanner
-                    .make_generic_err(&format!("Invalid unicode escape '{}'", scanner.cur));
+                return scanner.make_generic_err(&format!(
+                    "Invalid unicode escape '{esc}{}'",
+                    scanner.cur as char
+                ));
             }
-            esc.push(scanner.cur)
+            if esc.len() == 6 {
+                return scanner.make_generic_err(&format!(
+                    "Invalid unicode escape '{esc}{}...', must have 1 to 6 hex digits",
+                    scanner.cur as char
+                ));
+            }
+            esc.push(scanner.cur as char)
         }
 
-        let str = String::from_utf8_lossy(&esc).to_string();
-        match u16::from_str_radix(&str, 16) {
-            Ok(char) => Ok(String::from_utf16_lossy(&[char])),
-            Err(err) => scanner
-                .make_generic_err(&format!("Invalid unicode escape char '{str}', err: {err}")),
+        if esc.is_empty() {
+            return scanner
+                .make_generic_err("Invalid unicode escape '{}', must have 1 to 6 hex digits");
         }
     } else {
-        scanner.make_generic_err(&format!("Invalid escape sequence '{}'", scanner.cur))
+        for _ in 0..4 {
+            scanner.read()?;
+
+            if !scanner.is_hex_digit() {
+                return scanner.make_generic_err(&format!(
+                    "Invalid unicode escape '{}'",
+                    scanner.cur as char
+                ));
+            }
+            esc.push(scanner.cur as char)
+        }
+    }
+
+    match u32::from_str_radix(&esc, 16) {
+        Ok(code) if code <= 0x10FFFF => Ok(code),
+        Ok(_) => scanner.make_generic_err(&format!("Invalid unicode escape '{esc}', out of range")),
+        Err(err) => {
+            scanner.make_generic_err(&format!("Invalid unicode escape '{esc}', err: {err}"))
+        }
     }
 }
 
@@ -149,4 +215,56 @@ mod test {
         let str = parse_str(&mut scanner);
         assert_eq!(str.ok(), Some(Str::make("\x08\x0c")));
     }
+
+    fn parse(input: &str) -> Result<Str, std::io::Error> {
+        let mut input = Cursor::new(input.as_bytes());
+        let mut scanner = super::Scanner::make(&mut input).expect("Scanner");
+        parse_str(&mut scanner)
+    }
+
+    #[test]
+    fn test_zinc_parse_str_unicode_escapes() {
+        assert_eq!(parse(r#""Hell\u00D3""#).ok(), Some(Str::make("HellÓ")));
+        assert_eq!(parse(r#""Hell\u{D3}!""#).ok(), Some(Str::make("HellÓ!")));
+        assert_eq!(parse(r#""a\u{1F600}b""#).ok(), Some(Str::make("a😀b")));
+        assert_eq!(
+            parse(r#""\u{0}\u{10FFFF}""#).ok(),
+            Some(Str::make("\0\u{10FFFF}"))
+        );
+    }
+
+    #[test]
+    fn test_zinc_parse_str_surrogate_pairs() {
+        assert_eq!(parse(r#""a\ud83d\ude00b""#).ok(), Some(Str::make("a😀b")));
+        assert_eq!(
+            parse(r#""a\u{D83D}\u{DE00}b""#).ok(),
+            Some(Str::make("a😀b"))
+        );
+        assert_eq!(parse(r#""a😀b""#).ok(), Some(Str::make("a😀b")));
+
+        // Lone and mismatched surrogates become replacement characters.
+        assert_eq!(parse(r#""\ud83d""#).ok(), Some(Str::make("\u{FFFD}")));
+        assert_eq!(parse(r#""\ud83d\n""#).ok(), Some(Str::make("\u{FFFD}\n")));
+        assert_eq!(
+            parse(r#""\ud83d\u0041""#).ok(),
+            Some(Str::make("\u{FFFD}A"))
+        );
+    }
+
+    #[test]
+    fn test_zinc_parse_str_bad_unicode_escapes() {
+        for bad in [
+            r#""\u{}""#,
+            r#""\u{1234567}""#,
+            r#""\u{110000}""#,
+            r#""\u{12G}""#,
+            r#""\u{12"#,
+            r#""\u12G4""#,
+            r#""\u12"#,
+            r#""\ud83d\"#,
+            r#""\ud83d"#,
+        ] {
+            assert!(parse(bad).is_err(), "{bad} should fail");
+        }
+    }
 }

src/haystack/encoding/zinc/decode/scalar/uri.rs

@@ -31,10 +31,17 @@ pub(crate) fn parse_uri<R: Read>(scanner: &mut Scanner<R>) -> Result<Uri, Error>
                     str.push(next);
                     scanner.read()?;
                 }
-                _ => {
+                b'u' => {
+                    scanner.read()?; // u
                     let unicode = parse_str_unicode_escape(scanner)?;
                     str.extend_from_slice(unicode.as_bytes());
                 }
+                _ => {
+                    return scanner.make_generic_err(&format!(
+                        "Invalid Uri escape sequence '{}'",
+                        next as char
+                    ));
+                }
             };
         } else {
             str.push(scanner.cur);
@@ -78,6 +85,23 @@ mod test {
         }
     }
 
+    fn parse(input: &str) -> Result<Uri, std::io::Error> {
+        let mut input = Cursor::new(input.as_bytes());
+        let mut scanner = super::Scanner::make(&mut input).expect("Scanner");
+        parse_uri(&mut scanner)
+    }
+
+    #[test]
+    fn test_zinc_parse_uri_unicode_escape() {
+        assert_eq!(parse(r#"`/f\u1ff6o`"#).ok(), Some(Uri::make("/fῶo")));
+        assert_eq!(parse(r#"`a\u{1F600}b`"#).ok(), Some(Uri::make("a😀b")));
+        assert_eq!(parse(r#"`a\ud83d\ude00b`"#).ok(), Some(Uri::make("a😀b")));
+        assert_eq!(parse(r#"`a😀b`"#).ok(), Some(Uri::make("a😀b")));
+        assert!(parse(r#"`a\x`"#).is_err());
+        assert!(parse(r#"`a\nb`"#).is_err());
+        assert!(parse(r#"`a\ud83d`"#).is_ok());
+    }
+
     #[test]
     fn test_zinc_parse_uri_bad() {
         let mut input = Cursor::new("`".as_bytes());

src/haystack/encoding/zinc/decode/scanner.rs

@@ -273,6 +273,26 @@ impl<'a, R: Read> Scanner<'a, R> {
         Some(self.buffer_peek(next))
     }
 
+    /// Returns true if the bytes after the current one match `seq`, without
+    /// consuming them. Bytes already stashed by a previous peek are compared
+    /// first, and no further input is read after the first mismatch.
+    pub fn next_is(&mut self, seq: &[u8]) -> bool {
+        for (i, expected) in seq.iter().enumerate() {
+            let byte = match self.next.as_ref().and_then(|next| next.get(i)) {
+                Some(byte) => *byte,
+                None => match self.safe_peek() {
+                    Some(byte) => byte,
+                    None => return false,
+                },
+            };
+
+            if byte != *expected {
+                return false;
+            }
+        }
+        true
+    }
+
     /// Create am error for an missing expect condition
     /// Current position and the current char are captured
     pub fn make_expect_err<T, E: std::fmt::Display>(&self, item: E) -> Result<T, Error> {
@@ -396,6 +416,28 @@ mod test {
         assert!(scanner.peek().is_err());
     }
 
+    #[test]
+    fn test_zinc_scanner_next_is() {
+        let mut input = Cursor::new("abcdef".as_bytes());
+        let mut scanner = super::Scanner::make(&mut input).expect("Scanner");
+
+        assert!(scanner.next_is(b"bc"));
+        assert!(!scanner.next_is(b"bx"));
+
+        // Stashed bytes are compared rather than skipped and are still read in order.
+        assert_eq!(scanner.peek().ok(), Some(b'd'));
+        assert!(scanner.next_is(b"bcd"));
+        assert_eq!(scanner.read().ok(), Some(b'b'));
+        assert!(scanner.next_is(b"cd"));
+        assert_eq!(scanner.read().ok(), Some(b'c'));
+        assert_eq!(scanner.read().ok(), Some(b'd'));
+
+        // Stops at the end of input without matching.
+        assert!(!scanner.next_is(b"efg"));
+        assert_eq!(scanner.read().ok(), Some(b'e'));
+        assert_eq!(scanner.read().ok(), Some(b'f'));
+    }
+
     #[test]
     fn test_zinc_scanner_is_any_of() {
         let mut input = Cursor::new("8".as_bytes());

src/haystack/encoding/zinc/encode.rs

@@ -240,7 +240,12 @@ impl ToZinc for Uri {
                 '`' => writer.write_all(br"\`")?,
                 '\\' => writer.write_all(br"\\")?,
                 '\x20'..='\x7e' => writer.write_all(&[c as u8])?,
-                _ => writer.write_fmt(format_args!("\\u{:04x}", c as u32))?,
+                _ => {
+                    // Write chars above U+FFFF as a surrogate pair.
+                    for unit in c.encode_utf16(&mut [0u16; 2]) {
+                        writer.write_fmt(format_args!("\\u{:04x}", unit))?
+                    }
+                }
             }
         }
         writer.write_all(b"`")?;

src/haystack/filter/lexer.rs

@@ -323,6 +323,15 @@ mod test {
             "Unicode escape"
         );
 
+        let mut input = Cursor::new(r#""a \u{1F600} string""#.as_bytes());
+        let mut lexer = Lexer::make(&mut input).expect("Should create lexer");
+
+        assert_eq!(
+            lexer.read().ok(),
+            Some(&LexerToken::make_value("a 😀 string".into())),
+            "Braced unicode escape"
+        );
+
         let mut input = Cursor::new("\"\\t\\n\\\\\"".as_bytes());
         let mut lexer = Lexer::make(&mut input).expect("Should create lexer");
 
@@ -358,6 +367,15 @@ mod test {
             "Special characters escape"
         );
 
+        let mut input = Cursor::new(r#"`/a\u{1F600}`"#.as_bytes());
+        let mut lexer = Lexer::make(&mut input).expect("Should create lexer");
+
+        assert_eq!(
+            lexer.read().ok(),
+            Some(&LexerToken::make_value(Uri::from("/a😀").into())),
+            "Braced unicode escape"
+        );
+
         let mut input = Cursor::new("`".as_bytes());
         let mut lexer = Lexer::make(&mut input).expect("Should create lexer");
 

tests/zinc/test_string.rs

@@ -43,3 +43,18 @@ fn test_zinc_str_decode() {
 fn test_zinc_str_decode_bad() {
     assert!(from_str(r#""non terminated"#).is_err());
 }
+
+#[test]
+fn test_zinc_str_unicode_escapes() {
+    assert_eq!(from_str(r#""\u{1F600}""#).unwrap(), Value::make_str("😀"));
+    assert_eq!(
+        from_str(r#""\ud83d\ude00""#).unwrap(),
+        Value::make_str("😀")
+    );
+
+    // Non-ASCII is written as UTF-8 and round trips.
+    let value = Value::make_str("a😀b");
+    let zinc = to_zinc_string(&value).unwrap();
+    assert_eq!(zinc, r#""a😀b""#);
+    assert_eq!(from_str(&zinc).unwrap(), value);
+}

tests/zinc/test_uri.rs

@@ -25,3 +25,21 @@ fn test_zinc_uri_decode() {
 fn test_zinc_uri_decode_bad() {
     assert!(from_str(r#"`^"#).is_err());
 }
+
+#[test]
+fn test_zinc_uri_unicode() {
+    let value = Value::make_uri("/fῶo");
+    let zinc = to_zinc_string(&value).unwrap();
+    assert_eq!(zinc, r#"`/f\u1ff6o`"#);
+    assert_eq!(from_str(&zinc).unwrap(), value);
+
+    let value = Value::make_uri("a😀b");
+    let zinc = to_zinc_string(&value).unwrap();
+    assert_eq!(zinc, r#"`a\ud83d\ude00b`"#);
+    assert_eq!(from_str(&zinc).unwrap(), value);
+
+    assert_eq!(
+        from_str(r#"`a\u{1F600}b`"#).unwrap(),
+        Value::make_uri("a😀b")
+    );
+}