Merge pull request #54 from ecency/fix/surrogate-safe-serialization

Surrogate-safe JSON serialization for payloads, responses, and RPC bodies

Author: feruzm

CLAUDE.md

@@ -61,6 +61,8 @@ Handlers are `public static async Task Name(HttpContext ctx)` methods on static
 4. **Upstream requests always carry a `{}` JSON body** (even GETs) with bare `application/json` content type — this matches what upstreams have always received (verified against recorded production traffic). Review bots repeatedly flag this; it is correct.
 5. **`get_accounts` with a missing username must serialize as JSON `[null]`**, never the string `"null"` — `@null` is a real Hive account.
 6. **No hot-path logging.** The service defaults to Warning level and writes ~3 lines at startup; container logs are size-capped but must stay quiet. Don't add `Console.WriteLine` or Information-level logging to request paths.
+7. **Numbers are JavaScript doubles end to end — raw-literal preservation is NOT a goal.** The Node service parsed every request body and upstream response with `JSON.parse` (doubles) and re-emitted with `JSON.stringify`, so integers above 2^53 have always rounded on these paths (e.g. `18446744073709551615` → `18446744073709552000`). `JsJson.Stringify` reproduces this byte-for-byte on purpose. Do not flag double-rounding on payload/response serialization as precision loss, and do not "fix" it by preserving raw literals — either direction breaks parity. The single deliberate exception is the Solana lamports raw-text scan (`PrivateApi.Chain`), which keeps `ToJsonString` because it extracts a u64 from raw text before any parse.
+8. **Lone-surrogate `\u` escapes are valid input and output.** JavaScript strings are arbitrary UTF-16, so `JSON.parse`/`JSON.stringify` accept and re-emit lone surrogates; System.Text.Json throws on them in BOTH directions (string materialization AND the writer, regardless of encoder). All string extraction must go through `JsVal.TryGetStringLenient` and all tree serialization of client/upstream data through `JsJson.Stringify` — never raw `GetValue<string>`/`TryGetValue<string>` or `ToJsonString` on such data. Suggesting a switch back to the strict System.Text.Json APIs reintroduces production 500s.
 
 ## Hive RPC failover
 

dotnet/EcencyApi/Infrastructure/HiveRpcClient.cs

@@ -188,7 +188,9 @@ private List<int> OrderedNodeIndices()
             ["method"] = "call",
             ["params"] = new JsonArray(api, method, @params),
         };
-        var body = request.ToJsonString();
+        // JsJson: a lone-surrogate username from a client token must serialize
+        // (JSON.stringify semantics) instead of throwing in the writer.
+        var body = JsJson.Stringify(request);
 
         Exception? lastError = null;
 

dotnet/EcencyApi/Infrastructure/HttpContextExtensions.cs

@@ -77,19 +77,16 @@ public static async Task SendText(this HttpContext ctx, int status, string text)
         await ctx.Response.WriteAsync(text);
     }
 
-    /// <summary>res.status(code).send(obj) / res.json(obj).</summary>
+    /// <summary>res.status(code).send(obj) / res.json(obj). Serialized with
+    /// JsJson (JSON.stringify parity; tolerates lone surrogates that
+    /// System.Text.Json's writer throws on).</summary>
     public static async Task SendJson(this HttpContext ctx, int status, JsonNode? node)
     {
         ctx.Response.StatusCode = status;
         ctx.Response.ContentType = "application/json; charset=utf-8";
-        await ctx.Response.WriteAsync(node?.ToJsonString(JsonOpts) ?? "null");
+        await ctx.Response.WriteAsync(node is null ? "null" : JsJson.Stringify(node));
     }
 
-    private static readonly JsonSerializerOptions JsonOpts = new()
-    {
-        Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
-    };
-
     /// <summary>Convenience: string body field (undefined -> null).</summary>
     public static string? Str(this JsonObject body, string key) =>
         body.TryGetPropertyValue(key, out var v) && v is JsonValue val && JsVal.TryGetStringLenient(val, out var s)

dotnet/EcencyApi/Infrastructure/Upstream.cs

@@ -92,7 +92,10 @@ public static async Task<UpstreamResponse> BaseApiRequest(
 
         using var req = new HttpRequestMessage(method, finalUrl);
 
-        var bodyJson = payload?.ToJsonString(RawJsonOptions) ?? "{}";
+        // JsJson.Stringify (not ToJsonString): System.Text.Json's writer throws on
+        // lone-surrogate strings that JS handles fine, and JsJson byte-matches the
+        // JSON.stringify output axios sent upstream.
+        var bodyJson = payload is null ? "{}" : JsJson.Stringify(payload);
         req.Content = new StringContent(bodyJson, Encoding.UTF8);
         // axios sends bare "application/json" (no charset); keep upstream
         // requests byte-identical to the Node service.
@@ -155,11 +158,6 @@ public static async Task<UpstreamResponse> BaseApiRequest(
         }
     }
 
-    private static readonly JsonSerializerOptions RawJsonOptions = new()
-    {
-        Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
-    };
-
     public static string AppendQuery(string url, IEnumerable<KeyValuePair<string, string?>>? query)
     {
         if (query == null)
@@ -274,8 +272,9 @@ public static async Task SendLikeExpress(HttpContext ctx, int status, JsonNode?
             }
         }
 
-        // Objects, arrays, booleans -> res.json()
+        // Objects, arrays, booleans -> res.json(); JsJson matches JSON.stringify
+        // (and tolerates lone surrogates that ToJsonString throws on)
         ctx.Response.ContentType = "application/json; charset=utf-8";
-        await ctx.Response.WriteAsync(json!.ToJsonString(RawJsonOptions));
+        await ctx.Response.WriteAsync(JsJson.Stringify(json));
     }
 }