diff --git a/.changeset/ws-backpressure.md b/.changeset/ws-backpressure.md new file mode 100644 index 00000000..7756f39f --- /dev/null +++ b/.changeset/ws-backpressure.md @@ -0,0 +1,29 @@ +--- +'lens': minor +--- + +Bound what a slow websocket client can cost the process. + +`WebSocket#send()` neither blocks nor throws when a peer is slow, it queues. +Nothing inspected `bufferedAmount`, so one subscriber that stopped reading — a +suspended mobile client, a wedged TCP window — accumulated price updates in its +socket buffer without limit, and the message was counted as sent the moment it +was queued. With the feed pushing continuously, one stuck subscriber was enough +to grow the process until it was OOM-killed, taking every other subscriber and +every HTTP client with it. + +`FanOutManager` now applies a per-client policy in +[`src/ws/backpressure.ts`](https://github.com/Divinity-Labs/Lens/blob/main/src/ws/backpressure.ts): +past `WS_SLOW_CLIENT_BUFFER_BYTES` (256 KiB) updates are coalesced to the newest +one, past `WS_MAX_BUFFERED_BYTES` (1 MiB) — or congested for longer than +`WS_MAX_CONGESTED_MS` (30s) — the client is closed with `1013` and counted in +`ws_clients_dropped_total`. Superseded updates count in +`ws_messages_dropped_total`, so a truncated feed is visible rather than silent, +and `ws_buffered_bytes` reports total queued bytes. All the limits are +configurable, and none of them can slow down delivery to any other client. + +`ws_messages_sent_total` now increments when the socket reports the write +completed rather than when `send()` returned, so it counts what was written +rather than what was queued. The websocket metrics also moved from +prom-client's global default registry onto the one `/metrics` renders, which is +where they were supposed to be visible all along. \ No newline at end of file diff --git a/.env.example b/.env.example index 3937aedc..542a512a 100644 --- a/.env.example +++ b/.env.example @@ -126,6 +126,25 @@ AMM_PAGE_SIZE=200 # How often to poll Soroswap pool reserves in milliseconds (default: 60000 = 60s) SOROSWAP_POLL_INTERVAL_MS=60000 +# --- WebSocket backpressure (/ws) --- +# See docs/websocket-backpressure.md for the full policy. +# Max price updates per second per client (default: 100). 0 disables the window. +WS_COALESCE_INTERVAL_MS=100 +# Bytes a client socket may have queued before updates are coalesced to the +# newest one instead of sent (default: 262144 = 256 KiB). Clamped to half of +# WS_MAX_BUFFERED_BYTES if it is set higher. +WS_SLOW_CLIENT_BUFFER_BYTES=262144 +# Bytes a client socket may have queued before it is disconnected with close +# code 1013 (default: 1048576 = 1 MiB). +WS_MAX_BUFFERED_BYTES=1048576 +# How long a client may stay congested before it is disconnected anyway +# (default: 30000). Coalescing bounds the memory a slow client costs; this +# bounds the time. 0 keeps congested clients indefinitely. +WS_MAX_CONGESTED_MS=30000 +# How often the aggregate ws_buffered_bytes gauge is refreshed in milliseconds +# (default: 5000). +WS_BUFFER_SWEEP_INTERVAL_MS=5000 + # --- Auth & Security --- # Secret key required for admin endpoints (e.g., adding/removing pairs) ADMIN_API_KEY=super-secret-admin-key diff --git a/README.md b/README.md index 13334c3b..328d1e02 100644 --- a/README.md +++ b/README.md @@ -137,6 +137,11 @@ histogram_quantile(0.95, See [`docs/http-metrics.md`](docs/http-metrics.md) for the full label reference, the bucket rationale and suggested alerting rules. +The websocket stream carries its own signals — `ws_clients_total`, +`ws_messages_sent_total`, `ws_messages_dropped_total`, `ws_clients_dropped_total` +and `ws_buffered_bytes` — which exist to make slow subscribers visible instead +of silent. See [`docs/websocket-backpressure.md`](docs/websocket-backpressure.md). + The ingestion metrics are labelled by `network` as well, so a dual-network deployment reports each network separately: @@ -227,6 +232,15 @@ spending warning, and worked example before running. Full documentation: [examples/mcp-server/README.md](examples/mcp-server/README.md) +### WebSocket backpressure + +`/ws` counts a message as sent only once the socket reports the write completed, +and bounds what a client that stops reading can cost: past 256 KiB of queued +bytes updates are coalesced to the newest one, past 1 MiB — or congested for 30s +— the client is closed with `1013` and counted. A truncated feed is visible in +`ws_messages_dropped_total` rather than silent, and one slow client cannot affect +any other. See [`docs/websocket-backpressure.md`](docs/websocket-backpressure.md). + ## Usage Examples Lens gates `/price`, `/pools`, and `/candles` behind x402 micropayments on Stellar (testnet by default). The `/status` endpoint is free. @@ -350,6 +364,8 @@ Or interactively at [http://localhost:3002/graphiql](http://localhost:3002/graph Detailed system design and data flow diagrams can be found in the [Architecture Overview](docs/architecture.md). The API specification is available in [OpenAPI 3.0 format](openapi.yaml) and is auto-published to GitHub Pages at https://miracle656.github.io/lens/openapi.json. +Operational docs: [HTTP metrics](docs/http-metrics.md), [ingest metrics](docs/ingest-metrics.md), [websocket backpressure](docs/websocket-backpressure.md), [GraphQL](docs/graphql.md). + ## Examples The [oracle relay example](examples/oracle-relay/README.md) shows a minimal Soroban contract plus a Node relay that reads Lens prices and pushes them on chain. diff --git a/docs/http-metrics.md b/docs/http-metrics.md index 4965f6f4..095573b0 100644 --- a/docs/http-metrics.md +++ b/docs/http-metrics.md @@ -58,10 +58,14 @@ So: need for an exact code appears, add it then — it is easy to add a label and painful to remove one dashboards depend on. - **Websocket pairs use fixed buckets.** `ws_clients_total`, - `ws_messages_sent_total` and `ws_messages_dropped_total` retain their `pair` - label, but pair-specific subscriptions are recorded as `other`; `*` remains - the aggregate across all subscriptions. Arbitrary pair names and issuer - addresses stay in fanout state only and cannot create Prometheus series. + `ws_messages_sent_total`, `ws_messages_dropped_total` and + `ws_clients_dropped_total` retain their `pair` label, but pair-specific + subscriptions are recorded as `other`; `*` remains the aggregate across all + subscriptions. Arbitrary pair names and issuer addresses stay in fanout state + only and cannot create Prometheus series. `ws_buffered_bytes` is aggregate-only + (`*`): one series per connected client would be unbounded cardinality driven by + caller-supplied subscription strings. See + [`websocket-backpressure.md`](websocket-backpressure.md). ## Bucket choice diff --git a/docs/websocket-backpressure.md b/docs/websocket-backpressure.md new file mode 100644 index 00000000..7cf2013d --- /dev/null +++ b/docs/websocket-backpressure.md @@ -0,0 +1,122 @@ +# WebSocket backpressure + +Lens streams live prices over `/ws`. One property of that stream decides how the +server has to behave: **`send()` does not block and does not throw when a peer +is slow.** It queues. A client on a bad mobile connection, or one whose process +has been suspended and whose TCP window has filled, accumulates payloads in its +socket's internal buffer. Nothing inspected `bufferedAmount`, so that buffer had +no upper bound, and the message was counted as sent the moment it was queued. + +With a price feed pushing updates continuously, one stuck subscriber was enough +to grow the process until it was OOM-killed — taking every other subscriber, and +every HTTP client, with it. + +The fix is a policy over the socket's `bufferedAmount`, applied per client. It +lives in [`src/ws/fanout.ts`](../src/ws/fanout.ts) (`FanOutManager.deliver`); +the limits and their defaults are in +[`src/ws/backpressure.ts`](../src/ws/backpressure.ts). + +## The policy + +Checked per client, per update, in this order: + +| Tier | Condition | Action | +|---|---|---| +| 1. Evict (dead peer) | `bufferedAmount >= WS_MAX_BUFFERED_BYTES` | Close with **1013**, unregister, count `ws_clients_dropped_total`. | +| 2. Evict (stuck peer) | congested continuously for `WS_MAX_CONGESTED_MS` | Close with **1013**, unregister, count `ws_clients_dropped_total`. | +| 3. Coalesce | `bufferedAmount >= WS_SLOW_CLIENT_BUFFER_BYTES`, or inside the coalescing window | Keep only the newest update; count every superseded one in `ws_messages_dropped_total`. | +| 4. Send | otherwise | Send, and count `ws_messages_sent_total` when the socket reports the write completed. | + +**Why 1013.** RFC 6455 §7.4.1 reserves 1013 for "Try Again Later": the server is +refusing to keep serving this client right now. The condition here is the +client's rather than the server's, but the meaning to a client is the one it +already handles for a throttled feed — back off and reconnect, do not treat the +close as fatal. The close reason is one of `buffer limit exceeded` or +`congested too long`, kept short because RFC 6455 caps a close reason at 123 +bytes. + +**Why coalescing is the primary tier.** For a price feed, an intermediate tick +that arrives late is worthless: the newest tick has already superseded it, so +truncating the backlog costs the client nothing it could still use — and costs +the server one payload per client instead of one per tick. Tier 3 also covers +the ordinary coalescing window (`WS_COALESCE_INTERVAL_MS`, 10 updates/sec by +default), which existed before and is the same behaviour for the same reason. + +**Why evict at all.** Coalescing bounds the memory a slow client costs. It does +not bound the *time*: a client that has stopped reading entirely would sit +there forever, holding a slot, its buffered bytes and a `ws_clients_total` +series, with no metric saying why. Tier 1 catches a peer wedged past the hard +limit; tier 2 catches one wedged below it. + +**Why the soft limit is clamped.** `WS_SLOW_CLIENT_BUFFER_BYTES` is forced below +`WS_MAX_BUFFERED_BYTES` (at half of it). Set at or above, every client past the +soft limit would be disconnected on its first update over it and tier 3 — the +one that keeps a merely-slow client connected — would never run. + +## Defaults + +| Variable | Default | Meaning | +|---|---|---| +| `WS_COALESCE_INTERVAL_MS` | `100` | Max updates per second per client. `0` disables the window. | +| `WS_SLOW_CLIENT_BUFFER_BYTES` | `262144` (256 KiB) | Queued bytes above which updates are coalesced. Over a thousand ticks of backlog. | +| `WS_MAX_BUFFERED_BYTES` | `1048576` (1 MiB) | Queued bytes above which the client is closed. | +| `WS_MAX_CONGESTED_MS` | `30000` | How long a client may stay congested. `0` keeps them indefinitely. | +| `WS_BUFFER_SWEEP_INTERVAL_MS` | `5000` | How often `ws_buffered_bytes` is refreshed. | + +## Counting, honestly + +- `ws_messages_sent_total` increments **when the socket reports the write + completed**, not when `send()` was called. `send()` returns once the payload is + queued, so counting there reported messages as delivered when they were only in + a buffer — including for a client that had stopped reading and would never see + them. +- `ws_messages_dropped_total` counts updates discarded because a newer one + superseded them, plus the retained update dropped when a client is evicted. +- `ws_clients_dropped_total` counts evicted clients. +- `ws_buffered_bytes` is the aggregate of every client's queued bytes, sampled on + a timer. Aggregate only: one series per client would be unbounded cardinality + driven by caller-supplied subscription strings. + +All four keep the existing bounded `pair` label: pair-specific subscriptions are +recorded as `other`, `*` is the aggregate. See +[`http-metrics.md`](http-metrics.md). + +## Queries + +```promql +# Clients evicted for backpressure, per second +sum(rate(ws_clients_dropped_total[5m])) + +# Updates coalesced away, as a fraction of everything we tried to send +sum(rate(ws_messages_dropped_total[5m])) + / sum(rate(ws_messages_sent_total[5m])) + sum(rate(ws_messages_dropped_total[5m])) + +# Total queued bytes across every client — the thing that used to grow unbounded +ws_buffered_bytes{pair="*"} + +# Connected clients +sum(ws_clients_total{pair="*"}) +``` + +## Suggested alert + +```yaml +groups: + - name: lens-websocket + rules: + - alert: LensWebsocketClientsDropped + expr: sum(rate(ws_clients_dropped_total[10m])) > 0.1 + for: 10m + labels: { severity: warning } + annotations: + summary: "Lens is disconnecting websocket clients for falling behind" + description: >- + ws_clients_dropped_total is rising: clients are not draining their + socket buffers fast enough. Check ws_buffered_bytes and the network + between clients and this instance before raising the thresholds. +``` + +Tune against observed traffic before paging anyone. A steady trickle of +evictions on a mobile-heavy client base is a capacity signal, not an incident — +which is exactly why these clients are counted and dropped rather than allowed +to accumulate. \ No newline at end of file diff --git a/src/__tests__/fanout.test.ts b/src/__tests__/fanout.test.ts index 5740de90..bbc2ff02 100644 --- a/src/__tests__/fanout.test.ts +++ b/src/__tests__/fanout.test.ts @@ -50,6 +50,15 @@ vi.mock('prom-client', () => { } return { register: { metrics: vi.fn().mockResolvedValue('') }, + // src/metrics.ts builds its own Registry; src/ws/fanout.ts registers the + // websocket metrics on it so /metrics actually exports them. + Registry: class { + setDefaultLabels() {} + registerMetric() {} + metrics() { return Promise.resolve('') } + getSingleMetric() { return undefined } + resetMetrics() { return Promise.resolve('') } + }, Gauge: MockMetric, Counter: MockMetric, Histogram: MockMetric, diff --git a/src/__tests__/fanoutBackpressure.test.ts b/src/__tests__/fanoutBackpressure.test.ts new file mode 100644 index 00000000..4c9a6aa8 --- /dev/null +++ b/src/__tests__/fanoutBackpressure.test.ts @@ -0,0 +1,456 @@ +/** + * Backpressure for slow websocket consumers. + * + * The bug this covers: `send()` neither blocks nor throws when a peer stops + * reading, it queues. With nothing inspecting `bufferedAmount`, one subscriber + * on a bad mobile connection could grow the process until it was OOM-killed — + * taking every other subscriber with it. The assertions below drive a socket + * that accepts data and never flushes it, and check that what it holds stays + * bounded, that it eventually gets dropped, and that nothing it does affects + * any other client. + * + * Metrics are read from the real registry (no mock) because the + * counts themselves are the point: a truncated feed that is not counted is + * indistinguishable from a healthy one. + * + * The thresholds are set in the environment *before* the module is imported — + * they are resolved once at import, because they are read on the price-update + * path. They are deliberately tiny here so a handful of updates reaches them; + * `resolveBackpressureLimits` is unit-tested separately against the shipped + * defaults. + */ +import { describe, it, expect, vi, beforeEach, afterAll } from 'vitest' +import { register } from '../metrics' + +const { mockIoredis } = vi.hoisted(() => ({ + mockIoredis: vi.fn(), +})) + +vi.mock('ioredis', () => { + const client = { + connect: vi.fn().mockResolvedValue(undefined), + subscribe: vi.fn().mockResolvedValue(undefined), + unsubscribe: vi.fn().mockResolvedValue(undefined), + disconnect: vi.fn(), + on: vi.fn(), + } + // A `function`, not an arrow: fanout constructs it with `new`. + mockIoredis.mockImplementation(function () { + return client + }) + return { default: mockIoredis } +}) + +vi.mock('../redis', () => ({ + redis: { publish: vi.fn().mockResolvedValue(1), on: vi.fn() }, +})) + +// Small enough that a stalled socket reaches them in a handful of ticks, and a +// coalesce window of 0 so every update attempts a send instead of being held by +// the rate limiter. +const SOFT_LIMIT = 1024 +const HARD_LIMIT = 4096 +const MAX_CONGESTED_MS = 300 + +process.env.WS_COALESCE_INTERVAL_MS = '0' +process.env.WS_SLOW_CLIENT_BUFFER_BYTES = String(SOFT_LIMIT) +process.env.WS_MAX_BUFFERED_BYTES = String(HARD_LIMIT) +process.env.WS_MAX_CONGESTED_MS = String(MAX_CONGESTED_MS) +process.env.WS_BUFFER_SWEEP_INTERVAL_MS = '25' + +const fanout = await import('../ws/fanout') +const { priceEmitter, PRICE_UPDATE } = await import('../events') +const { resolveBackpressureLimits, DEFAULT_BACKPRESSURE_LIMITS } = await import('../ws/backpressure') + +/** Aggregate (`*`) total of a counter. */ +async function counterTotal(name: string): Promise { + const metric = (await register.getSingleMetric(name))!.get() + return metric.then(res => + res.values + .filter(v => v.labels.pair === '*') + .reduce((sum, v) => sum + v.value, 0) + ) +} + +/** Current value of the aggregate buffered-bytes gauge. */ +async function bufferedGauge(): Promise { + const metric = (await register.getSingleMetric('ws_buffered_bytes'))!.get() + return metric.then(res => + res.values + .filter(v => v.labels.pair === '*') + .reduce((max, v) => Math.max(max, v.value), 0) + ) +} + +interface StalledSocket { + id: string + send: (data: string, cb?: (err?: Error | null) => void) => void + close: (code?: number, reason?: string) => void + bufferedAmount: () => number + /** Bytes the fake OS receive window is holding. */ + bytesHeld: number + /** Frames handed to `send`, i.e. queued in the socket's buffer. */ + framesQueued: number + closeCalls: Array<{ code?: number; reason?: string }> + /** Let the peer drain, as a real socket does once a reader reappears. */ + drain: (bytes: number) => void +} + +/** + * A socket that accepts frames and never writes them out — the stalled-peer + * case. `bytesHeld` grows by the frame size on every send and only falls when + * `drain()` is called, which is how `ws`' `bufferedAmount` behaves for a peer + * that has stopped reading. + */ +function stalledSocket(id: string, initialHeld = 0): StalledSocket { + const socket: StalledSocket = { + id, + bytesHeld: initialHeld, + framesQueued: 0, + closeCalls: [], + send(data: string) { + socket.framesQueued++ + socket.bytesHeld += Buffer.byteLength(data) + // Never calls `cb` — the write never completes. + }, + close(code?: number, reason?: string) { + socket.closeCalls.push({ code, reason }) + }, + bufferedAmount: () => socket.bytesHeld, + drain(bytes: number) { + socket.bytesHeld = Math.max(0, socket.bytesHeld - bytes) + }, + } + return socket +} + +/** A socket that writes everything through, as a healthy client does. */ +function healthySocket(id: string) { + const received: string[] = [] + return { + id, + received, + send: vi.fn((data: string, cb?: (err?: Error | null) => void) => { + received.push(data) + cb?.() + }), + close: vi.fn(), + bufferedAmount: () => 0, + } +} + +function priceUpdate(price: number) { + return { + assetA: 'XLM', + assetB: 'USDC', + previousPrice: price, + currentPrice: price + 0.001, + timestamp: new Date(0), + } +} + +function emit(count: number, from = 0): void { + for (let i = 0; i < count; i++) priceEmitter.emit(PRICE_UPDATE, priceUpdate(from + i)) +} + +beforeEach(async () => { + // Each test starts from an empty manager: a client left registered by the + // previous test keeps receiving updates and quietly doubles every count here. + // destroy() also clears the emitter subscription, so the next initialize() + // installs exactly one. + await fanout.fanOutManager.destroy() + priceEmitter.removeAllListeners(PRICE_UPDATE) + await fanout.fanOutManager.initialize() + for (const metric of [ + 'ws_messages_sent_total', + 'ws_messages_dropped_total', + 'ws_clients_dropped_total', + 'ws_buffered_bytes', + ]) { + const found = await register.getSingleMetric(metric) + found?.reset() + } +}) + +afterAll(async () => { + await fanout.fanOutManager.destroy() +}) + +describe('memory stays bounded for a stalled peer', () => { + it('never queues past the soft limit, however many updates arrive', async () => { + const stalled = stalledSocket('stalled-1') + const unsubscribe = fanout.fanOutManager.register('*', stalled) + + // 2 000 updates, each ~190 bytes of JSON, into a peer that never reads. + emit(2000) + + // Before this fix the socket held all of it. Coalescing stops the queue at + // the soft limit instead: the backlog is truncated, not drained. + expect(stalled.bytesHeld).toBeLessThan(SOFT_LIMIT * 2) + expect(stalled.bytesHeld).toBeLessThanOrEqual(HARD_LIMIT) + expect(stalled.framesQueued).toBeLessThan(2000) + + unsubscribe() + }) + + it('holds at the soft limit rather than climbing with every update', async () => { + const stalled = stalledSocket('stalled-2') + fanout.fanOutManager.register('*', stalled) + + emit(50) + const afterFirstBurst = stalled.bytesHeld + + emit(2000, 100) + + // Two thousand more updates changed nothing: the client is congested and + // the newest update replaced the previous newest one each time. + expect(stalled.bytesHeld).toBe(afterFirstBurst) + expect(afterFirstBurst).toBeGreaterThan(0) + expect(afterFirstBurst).toBeLessThanOrEqual(HARD_LIMIT) + }) + + it('releases the retained payload when the client is dropped', async () => { + const stalled = stalledSocket('stalled-3') + fanout.fanOutManager.register('*', stalled) + + emit(10) + expect(stalled.framesQueued).toBeGreaterThan(0) + + // Cross the hard limit so the client is evicted with an update still held. + stalled.bytesHeld = HARD_LIMIT + priceEmitter.emit(PRICE_UPDATE, priceUpdate(11)) + + expect(stalled.closeCalls).toEqual([{ code: 1013, reason: 'buffer limit exceeded' }]) + + // Unregistered: further updates reach nobody and change nothing. + const queuedAfterDrop = stalled.framesQueued + emit(100, 200) + expect(stalled.framesQueued).toBe(queuedAfterDrop) + expect(stalled.bytesHeld).toBe(HARD_LIMIT) + }) + + it('coalesces to the newest update while congested, and counts what it dropped', async () => { + const stalled = stalledSocket('stalled-4', SOFT_LIMIT) + fanout.fanOutManager.register('*', stalled) + + emit(50) + + expect(stalled.framesQueued).toBe(0) + expect(await counterTotal('ws_messages_dropped_total')).toBe(49) + }) + + it('reports queued bytes on the aggregate gauge', async () => { + const stalled = stalledSocket('stalled-5') + fanout.fanOutManager.register('*', stalled) + + emit(20) + await new Promise(resolve => setTimeout(resolve, 120)) + + expect(await bufferedGauge()).toBe(stalled.bytesHeld) + }) +}) + +describe('slow-client policy', () => { + it('closes a client past the hard limit with 1013 and counts it', async () => { + const stalled = stalledSocket('stalled-6', HARD_LIMIT) + fanout.fanOutManager.register('*', stalled) + + priceEmitter.emit(PRICE_UPDATE, priceUpdate(1)) + + expect(stalled.closeCalls).toEqual([{ code: 1013, reason: 'buffer limit exceeded' }]) + expect(await counterTotal('ws_clients_dropped_total')).toBe(1) + }) + + it('closes a client that stays congested past WS_MAX_CONGESTED_MS', async () => { + // Congested but nowhere near the hard limit: coalescing already holds the + // memory, so the only thing left to bound is the time. + const stalled = stalledSocket('stalled-7', SOFT_LIMIT) + fanout.fanOutManager.register('*', stalled) + + priceEmitter.emit(PRICE_UPDATE, priceUpdate(1)) + expect(stalled.closeCalls).toHaveLength(0) + + await new Promise(resolve => setTimeout(resolve, MAX_CONGESTED_MS + 100)) + priceEmitter.emit(PRICE_UPDATE, priceUpdate(2)) + + expect(stalled.closeCalls).toEqual([{ code: 1013, reason: 'congested too long' }]) + expect(await counterTotal('ws_clients_dropped_total')).toBe(1) + }) + + it('keeps a congested client that recovers before the deadline, and delivers to it', async () => { + const stalled = stalledSocket('stalled-8', SOFT_LIMIT) + fanout.fanOutManager.register('*', stalled) + + priceEmitter.emit(PRICE_UPDATE, priceUpdate(1)) + expect(stalled.framesQueued).toBe(0) + + // The peer starts reading again. Coalescing must not have cost it the + // subscription, and the update it missed must still reach it. + stalled.drain(SOFT_LIMIT) + priceEmitter.emit(PRICE_UPDATE, priceUpdate(2)) + + expect(stalled.closeCalls).toHaveLength(0) + expect(stalled.framesQueued).toBe(1) + }) +}) + +describe('threshold configuration', () => { + it('reads every limit from the environment', () => { + const limits = resolveBackpressureLimits({ + WS_COALESCE_INTERVAL_MS: '250', + WS_SLOW_CLIENT_BUFFER_BYTES: '1024', + WS_MAX_BUFFERED_BYTES: '8192', + WS_MAX_CONGESTED_MS: '5000', + } as NodeJS.ProcessEnv) + + expect(limits).toEqual({ + coalesceIntervalMs: 250, + slowClientBufferBytes: 1024, + maxBufferedBytes: 8192, + maxCongestedMs: 5000, + }) + }) + + it('ships defaults that coalesce at 256 KiB and disconnect at 1 MiB', () => { + expect(resolveBackpressureLimits({} as NodeJS.ProcessEnv)).toEqual( + DEFAULT_BACKPRESSURE_LIMITS + ) + expect(DEFAULT_BACKPRESSURE_LIMITS.slowClientBufferBytes).toBeLessThan( + DEFAULT_BACKPRESSURE_LIMITS.maxBufferedBytes + ) + }) + + it('clamps the soft limit below the hard one so coalescing always has a window', () => { + // Otherwise a client past the soft limit is disconnected on its first + // update over it, and the tier that keeps a slow client connected never runs. + const limits = resolveBackpressureLimits({ + WS_SLOW_CLIENT_BUFFER_BYTES: '999999', + WS_MAX_BUFFERED_BYTES: '4096', + } as NodeJS.ProcessEnv) + + expect(limits.slowClientBufferBytes).toBe(2048) + expect(limits.maxBufferedBytes).toBe(4096) + }) + + it('falls back to the defaults for unusable values, and allows disabling via 0', () => { + expect(resolveBackpressureLimits({ WS_MAX_BUFFERED_BYTES: 'lots' } as NodeJS.ProcessEnv)) + .toMatchObject({ maxBufferedBytes: DEFAULT_BACKPRESSURE_LIMITS.maxBufferedBytes }) + expect(resolveBackpressureLimits({ WS_COALESCE_INTERVAL_MS: '0', WS_MAX_CONGESTED_MS: '0' } as NodeJS.ProcessEnv)) + .toMatchObject({ coalesceIntervalMs: 0, maxCongestedMs: 0 }) + }) +}) + +describe('sent accounting', () => { + it('counts a message only once the socket reports the write', async () => { + const healthy = healthySocket('healthy-1') + fanout.fanOutManager.register('*', healthy) + + priceEmitter.emit(PRICE_UPDATE, priceUpdate(1)) + + expect(healthy.send).toHaveBeenCalledTimes(1) + expect(await counterTotal('ws_messages_sent_total')).toBe(1) + }) + + it('does not count a message that was queued but never written', async () => { + const stalled = stalledSocket('stalled-9') + fanout.fanOutManager.register('*', stalled) + + priceEmitter.emit(PRICE_UPDATE, priceUpdate(1)) + + // `send()` returned, so the old code counted this. The write never + // completed, so it was not delivered and must not be counted as sent. + expect(stalled.framesQueued).toBe(1) + expect(await counterTotal('ws_messages_sent_total')).toBe(0) + }) + + it('unregisters a client whose write fails, without counting it as sent', async () => { + const send = vi.fn((_data: string, cb?: (err?: Error | null) => void) => cb?.(new Error('EPIPE'))) + fanout.fanOutManager.register('*', { id: 'failing-1', send, close: vi.fn(), bufferedAmount: () => 0 }) + + priceEmitter.emit(PRICE_UPDATE, priceUpdate(1)) + priceEmitter.emit(PRICE_UPDATE, priceUpdate(2)) + + expect(await counterTotal('ws_messages_sent_total')).toBe(0) + // Unregistered after the failed write, so the second update went nowhere. + expect(send).toHaveBeenCalledTimes(1) + }) + + it('survives a client whose send throws synchronously', async () => { + const client = { + id: 'throwing-1', + send: () => { + throw new Error('socket destroyed') + }, + close: vi.fn(), + bufferedAmount: () => 0, + } + fanout.fanOutManager.register('*', client) + const healthy = healthySocket('healthy-2') + fanout.fanOutManager.register('*', healthy) + + priceEmitter.emit(PRICE_UPDATE, priceUpdate(1)) + + expect(await counterTotal('ws_messages_sent_total')).toBe(1) + expect(healthy.received).toHaveLength(1) + }) + + it('treats a client that cannot report its buffer as unbuffered', () => { + const client = { id: 'opaque-1', send: vi.fn(), close: vi.fn() } + expect(() => fanout.fanOutManager.register('*', client)).not.toThrow() + expect(() => priceEmitter.emit(PRICE_UPDATE, priceUpdate(1))).not.toThrow() + }) +}) + +describe('scrape visibility', () => { + it('exports the backpressure metrics on the registry /metrics renders', async () => { + // They used to be created on prom-client's global default registry, which + // `/metrics` does not render — so a slow client was counted somewhere + // Prometheus never looks. + const healthy = healthySocket('healthy-6') + fanout.fanOutManager.register('*', healthy) + priceEmitter.emit(PRICE_UPDATE, priceUpdate(1)) + + const scrape = await register.metrics() + for (const name of [ + 'ws_clients_total', + 'ws_messages_sent_total', + 'ws_messages_dropped_total', + 'ws_clients_dropped_total', + 'ws_buffered_bytes', + ]) { + expect(scrape, name).toContain(name) + } + expect(scrape).toMatch(/ws_messages_sent_total\{[^}]*pair="\*"[^}]*\} 1/) + }) +}) + +describe('one slow client cannot affect another', () => { + it('keeps delivering to healthy clients while a stalled one is dropped', async () => { + const healthy = healthySocket('healthy-3') + const stalled = stalledSocket('stalled-10', HARD_LIMIT) + const otherHealthy = healthySocket('healthy-4') + fanout.fanOutManager.register('*', healthy) + fanout.fanOutManager.register('*', stalled) + fanout.fanOutManager.register('*', otherHealthy) + + emit(20) + + expect(stalled.closeCalls).toHaveLength(1) + expect(healthy.received).toHaveLength(20) + expect(otherHealthy.received).toHaveLength(20) + expect(await counterTotal('ws_messages_sent_total')).toBe(40) + }) + + it('delivers every update to a healthy client while a congested one is coalescing', async () => { + const healthy = healthySocket('healthy-5') + const stalled = stalledSocket('stalled-11', SOFT_LIMIT) + fanout.fanOutManager.register('*', healthy) + fanout.fanOutManager.register('*', stalled) + + emit(100) + + expect(healthy.received).toHaveLength(100) + expect(stalled.framesQueued).toBe(0) + expect(stalled.closeCalls).toHaveLength(0) + }) +}) \ No newline at end of file diff --git a/src/api/websocket.ts b/src/api/websocket.ts index 0c8b1f5c..acd8dada 100644 --- a/src/api/websocket.ts +++ b/src/api/websocket.ts @@ -89,14 +89,25 @@ export async function registerWebSocket(app: FastifyInstance) { const clientId = uuid() const unsubscribe = fanOutManager.register('*', { id: clientId, - send: (data: string) => { - if (socket.readyState === 1) { // OPEN - socket.send(data) + send: (data: string, cb?: (err?: Error | null) => void) => { + if (socket.readyState !== 1) { // OPEN + // Tell the fan-out rather than silently swallowing it: a socket that + // cannot be written to is a client that must be unregistered, not one + // whose messages quietly stop being counted. + cb?.(new Error(`socket not open (readyState ${socket.readyState})`)) + return } + // The callback fires once the frame has actually been written, which is + // what FanOutManager counts as "sent" — see src/ws/fanout.ts. + socket.send(data, (err?: Error) => cb?.(err)) }, - close: () => { - socket.close() + close: (code?: number, reason?: string) => { + socket.close(code, reason) }, + // Bytes queued for this socket and not yet written to the peer. This is + // the only backpressure signal available, and every threshold in + // FanOutManager is measured against it. + bufferedAmount: () => socket.bufferedAmount, }) socket.on('close', () => { diff --git a/src/ws/backpressure.ts b/src/ws/backpressure.ts new file mode 100644 index 00000000..8a95691c --- /dev/null +++ b/src/ws/backpressure.ts @@ -0,0 +1,114 @@ +/** + * Websocket backpressure policy. + * + * `WebSocket#send()` neither blocks nor throws when the peer is merely slow — + * it queues. A client on a bad mobile connection, or one whose process has been + * suspended, accumulates price updates in its socket's internal buffer, and + * nothing inspects `bufferedAmount`, so that buffer has no upper bound. With a + * price feed running continuously, one stuck subscriber is enough to grow the + * process until it is OOM-killed, taking every other subscriber with it. + * + * The policy, in three tiers, is implemented in `FanOutManager.deliver`: + * + * 1. **Coalesce** — past {@link BackpressureLimits.slowClientBufferBytes}, or + * inside the coalescing window, keep only the newest update and drop the + * ones it supersedes. For a price feed an intermediate tick that arrives + * late is worthless: the newest tick has already superseded it, so the + * backlog is truncated rather than drained. This is the tier that actually + * saves a healthy-but-slow client, and the one most clients hit. + * 2. **Disconnect** — past {@link BackpressureLimits.maxBufferedBytes}, close + * the socket with {@link CLOSE_CODE_SLOW_CONSUMER}. No amount of coalescing + * fixes a peer that has stopped reading entirely; what bounds it is not + * letting its buffer grow. + * 3. **Evict** — congested for longer than + * {@link BackpressureLimits.maxCongestedMs}, close it too. Tier 1 bounds the + * memory such a client costs; this bounds the time. Without it a client that + * stopped reading holds its slot, its buffered bytes and a + * `ws_clients_total` series forever, and nothing would say why. + * + * A stalled client can never affect another one: the thresholds are read per + * client, dropping one only touches that client's own socket, and every send is + * individually guarded. + * + * Resolved once at import (see `FanOutManager`), because this is read on the + * price-update path. + */ + +/** + * RFC 6455 §7.4.1 code 1013, "Try Again Later". The condition is the client's + * rather than the server's, but the meaning to a client is the one it already + * handles for a throttled feed: reconnect, do not treat the close as fatal. + */ +export const CLOSE_CODE_SLOW_CONSUMER = 1013 + +export interface BackpressureLimits { + /** Max updates per second per client, independent of backpressure. 0 disables the window. */ + coalesceIntervalMs: number + /** Queued bytes above which updates are coalesced instead of sent. */ + slowClientBufferBytes: number + /** Queued bytes above which the client is closed. */ + maxBufferedBytes: number + /** How long a client may stay congested before it is closed anyway. 0 disables. */ + maxCongestedMs: number +} + +/** Defaults, all overridable per deployment. */ +export const DEFAULT_BACKPRESSURE_LIMITS: BackpressureLimits = { + coalesceIntervalMs: 100, + // 256 KiB of price updates is over a thousand ticks of backlog. A client that + // far behind wants the newest value, not the thousand before it. + slowClientBufferBytes: 256 * 1024, + maxBufferedBytes: 1024 * 1024, + maxCongestedMs: 30_000, +} + +function intFromEnv(env: NodeJS.ProcessEnv, name: string, fallback: number, allowZero: boolean): number { + const raw = Number(env[name]) + if (!Number.isFinite(raw)) return fallback + if (raw === 0) return allowZero ? 0 : fallback + if (raw < 0) return fallback + return Math.floor(raw) +} + +/** + * Resolves the limits from the environment, clamped so they stay a policy + * rather than two knobs that can cancel each other. + * + * `WS_SLOW_CLIENT_BUFFER_BYTES` at or above `WS_MAX_BUFFERED_BYTES` would mean + * every client past the soft limit is disconnected on its first update over it + * and the coalescing tier — the one that keeps a merely-slow client connected — + * never runs. So the soft limit is clamped to half the hard one. + */ +export function resolveBackpressureLimits(env: NodeJS.ProcessEnv = process.env): BackpressureLimits { + const coalesceIntervalMs = intFromEnv( + env, + 'WS_COALESCE_INTERVAL_MS', + DEFAULT_BACKPRESSURE_LIMITS.coalesceIntervalMs, + true + ) + const maxCongestedMs = intFromEnv( + env, + 'WS_MAX_CONGESTED_MS', + DEFAULT_BACKPRESSURE_LIMITS.maxCongestedMs, + true + ) + const maxBufferedBytes = intFromEnv( + env, + 'WS_MAX_BUFFERED_BYTES', + DEFAULT_BACKPRESSURE_LIMITS.maxBufferedBytes, + false + ) + const slowClientBufferBytes = intFromEnv( + env, + 'WS_SLOW_CLIENT_BUFFER_BYTES', + DEFAULT_BACKPRESSURE_LIMITS.slowClientBufferBytes, + false + ) + + return { + coalesceIntervalMs, + maxBufferedBytes, + maxCongestedMs, + slowClientBufferBytes: Math.min(slowClientBufferBytes, Math.floor(maxBufferedBytes / 2)), + } +} \ No newline at end of file diff --git a/src/ws/fanout.ts b/src/ws/fanout.ts index 85c414ed..d8430b49 100644 --- a/src/ws/fanout.ts +++ b/src/ws/fanout.ts @@ -1,13 +1,18 @@ -import { EventEmitter } from 'events' import { redis } from '../redis' import { activeNetwork } from '../config' import { priceEmitter, PRICE_UPDATE, PriceUpdateEvent } from '../events' -import { register, Gauge, Counter } from 'prom-client' +import { Gauge, Counter } from 'prom-client' +import { register } from '../metrics' import { createLogger } from '../logger' +import { resolveBackpressureLimits, CLOSE_CODE_SLOW_CONSUMER } from './backpressure' const log = createLogger('ws:fanout') // ─── Metrics ────────────────────────────────────────────────────────────────── +// Registered on the app's own registry (src/metrics.ts), not prom-client's global +// default. `/metrics` renders that registry, so a slow client counted on the +// default one would be counted somewhere Prometheus never looks — visible in the +// source, invisible in the dashboard. export const ws_clients_total = new Gauge({ name: 'ws_clients_total', help: 'Current number of connected WebSocket clients', @@ -17,7 +22,7 @@ export const ws_clients_total = new Gauge({ export const ws_messages_sent_total = new Counter({ name: 'ws_messages_sent_total', - help: 'Total number of price updates sent to WebSocket clients', + help: 'Total number of price updates written to WebSocket clients', labelNames: ['pair'], registers: [register], }) @@ -29,9 +34,37 @@ export const ws_messages_dropped_total = new Counter({ registers: [register], }) +export const ws_clients_dropped_total = new Counter({ + name: 'ws_clients_dropped_total', + help: 'Total number of WebSocket clients disconnected for failing to keep up with the price feed', + labelNames: ['pair'], + registers: [register], +}) + +export const ws_buffered_bytes = new Gauge({ + name: 'ws_buffered_bytes', + help: 'Bytes handed to client sockets and not yet written out (all clients, aggregate)', + labelNames: ['pair'], + registers: [register], +}) + // ─── Configuration ───────────────────────────────────────────────────────────── const REDIS_CHANNEL = `lens:${activeNetwork}:price:updates` -const DEFAULT_COALESCE_INTERVAL_MS = 100 // max 10 updates/sec per client + +function envInt(name: string, fallback: number): number { + const raw = Number(process.env[name]) + return Number.isFinite(raw) && raw > 0 ? Math.floor(raw) : fallback +} + +/** How often the aggregate buffered-bytes gauge is refreshed. */ +const BUFFER_SWEEP_INTERVAL_MS = envInt('WS_BUFFER_SWEEP_INTERVAL_MS', 5000) + +/** + * Backpressure limits, resolved once at import — this is read on the + * price-update path. The policy, the tiers and the reasoning behind each + * default live in src/ws/backpressure.ts. + */ +const LIMITS = resolveBackpressureLimits() /** * Pair names can contain arbitrary asset codes and issuer identifiers supplied @@ -45,8 +78,27 @@ function pairMetricLabel(pairKey: string): 'other' | null { // ─── Client wrapper with backpressure & coalescing ──────────────────────────── export interface FanOutClient { id: string - send: (data: string) => void - close: () => void + /** + * Queues `data` on the socket. + * + * `cb` is invoked once the data has actually been written out, or with an + * error. The fan-out counts a message as *sent* only from that callback: + * `send()` returns as soon as the payload is queued, so counting at call time + * reported messages as delivered when they were only in a buffer — including + * for a client that had stopped reading entirely and would never see them. + */ + send: (data: string, cb?: (err?: Error | null) => void) => void + /** Closes the underlying socket, ideally with a status code and reason. */ + close: (code?: number, reason?: string) => void + /** + * Bytes queued on this socket and not yet written to the peer. + * + * This is the only backpressure signal the WebSocket API offers, and it is + * what every threshold below is compared against. Omit it only for a client + * that cannot report a buffer (in-process fakes); a wrapper backed by a real + * socket must expose it, or nothing bounds that client. + */ + bufferedAmount?: () => number } interface InternalClient { @@ -57,6 +109,8 @@ interface InternalClient { pendingUpdate: string | null timer: ReturnType | null destroyed: boolean + /** When this client first crossed the soft buffer limit, if it is congested. */ + congestedSince: number | null } // ─── Fan-Out Manager ────────────────────────────────────────────────────────── @@ -69,12 +123,20 @@ class FanOutManager { private publisher: typeof redis = redis private redisConnected = false + private bufferSweep: ReturnType | null = null + async initialize(): Promise { // Subscribe to local price emitter (always works) priceEmitter.on(PRICE_UPDATE, (event: PriceUpdateEvent) => { this.onPriceUpdate(event) }) + // Aggregate queued-bytes gauge. Sampled on a timer rather than per broadcast + // because the number of clients is unbounded and a per-send set() would + // turn every tick into a metric write per subscriber. + this.bufferSweep = setInterval(() => this.reportBufferedBytes(), BUFFER_SWEEP_INTERVAL_MS) + this.bufferSweep.unref?.() + // Subscribe to Redis channel for cross-instance updates (best-effort) try { this.subscriber = new (await import('ioredis')).default( @@ -113,6 +175,7 @@ class FanOutManager { pendingUpdate: null, timer: null, destroyed: false, + congestedSince: null, } this.clients.set(id, client) @@ -135,7 +198,11 @@ class FanOutManager { const client = this.clients.get(id) if (!client) return if (client.timer) clearTimeout(client.timer) + client.timer = null client.destroyed = true + // Drop the retained payload. An unregistered client must hold no memory: + // the whole point of bounding the buffer is that the bytes go away with it. + client.pendingUpdate = null this.clients.delete(id) const clients = this.pairClients.get(client.pairKey) @@ -153,9 +220,15 @@ class FanOutManager { /** * Called when a price update is received from the local emitter. - * Publishes to Redis for other instances and broadcasts locally. + * Publishes to Redis for cross-instance broadcasting. */ private async onPriceUpdate(event: PriceUpdateEvent): Promise { + // Local clients first. Publishing to Redis is a network round trip, and + // waiting on it before delivering locally meant a slow (or stalled) Redis + // delayed every subscriber on this instance — the one dependency that can + // be added to the critical path without improving the result. + this.broadcastToLocalClients(event) + // Publish to Redis for cross-instance fan-out if (this.redisConnected) { try { @@ -164,14 +237,11 @@ class FanOutManager { // non-fatal } } - - // Broadcast to local clients - this.broadcastToLocalClients(event) } /** * Broadcast a price update to local clients subscribed to this pair. - * Applies per-client coalescing for slow consumers. + * Applies per-client backpressure and coalescing for slow consumers. * Also sends to wildcard ('*') subscribers that receive all pairs. */ private broadcastToLocalClients(event: PriceUpdateEvent): void { @@ -199,40 +269,177 @@ class FanOutManager { const client = this.clients.get(clientId) if (!client || client.destroyed) continue - const elapsed = now - client.lastSent - - if (elapsed >= DEFAULT_COALESCE_INTERVAL_MS) { - // Client is ready to receive - this.sendToClient(client, payload) - client.lastSent = now - client.pendingUpdate = null - } else { - // Client is too fast — coalesce: buffer the latest update - client.pendingUpdate = payload - if (!client.timer) { - const timeout = DEFAULT_COALESCE_INTERVAL_MS - elapsed - client.timer = setTimeout(() => { - client.timer = null - if (client.destroyed) return - if (client.pendingUpdate) { - this.sendToClient(client, client.pendingUpdate) - client.lastSent = Date.now() - client.pendingUpdate = null - } - }, timeout) - } + // Each client is handled independently: a dropped, congested or throwing + // client must never stop the next one on this list from being served. + this.deliver(client, payload, now) + } + } + + /** + * Hand one payload to one client, respecting what its socket can absorb. + * + * The policy, in order: + * + * 1. **Over the hard limit** (`WS_MAX_BUFFERED_BYTES`) — the peer is not + * draining at all. Close it with 1013 and forget it. There is no amount + * of coalescing that fixes a peer which has stopped reading, and an + * unbounded socket buffer is how one subscriber OOM-kills the process + * for everyone else. + * 2. **Over the soft limit** (`WS_SLOW_CLIENT_BUFFER_BYTES`) — congested but + * alive: keep only the newest update. For a price feed an intermediate + * tick that arrives late is worthless, because the newest tick has + * already superseded it, so the backlog is truncated rather than drained. + * 3. **Congested for `WS_MAX_CONGESTED_MS`** — close with 1013. Tier 2 + * bounds the memory a stuck client costs; this bounds the time. Without + * it a client that stopped reading holds its slot, its buffered bytes and + * a `ws_clients_total` series forever, and no metric would ever say why. + * 4. **Otherwise** — send, and count it as sent only once the socket reports + * the write completed. + * + * Tier 2 also subsumes the fixed coalescing window: a client inside that + * window gets the same newest-only treatment, for the same reason. + */ + private deliver(client: InternalClient, payload: string, now: number): void { + const buffered = this.bufferedBytesOf(client) + + if (buffered >= LIMITS.maxBufferedBytes) { + this.dropSlowClient(client, 'buffer limit exceeded', buffered) + return + } + + const congested = buffered >= LIMITS.slowClientBufferBytes + if (congested) { + if (client.congestedSince === null) client.congestedSince = now + if (LIMITS.maxCongestedMs > 0 && now - client.congestedSince >= LIMITS.maxCongestedMs) { + this.dropSlowClient(client, 'congested too long', buffered) + return } + } else { + client.congestedSince = null } + + const elapsed = now - client.lastSent + if (!congested && elapsed >= LIMITS.coalesceIntervalMs) { + client.pendingUpdate = null + client.lastSent = now + this.sendToClient(client, payload) + return + } + + // Only the newest update is worth keeping. Anything already buffered for + // this client is superseded by `payload`, so it is counted as dropped — + // silently, and without a metric, a truncated feed is indistinguishable + // from a healthy one. + if (client.pendingUpdate !== null) this.countDroppedUpdate(client) + client.pendingUpdate = payload + + // A congested client is re-checked on the fixed interval rather than + // immediately: its buffer can only shrink on its own, and a zero-delay + // reschedule would spin the event loop for as long as the peer stays stuck. + this.scheduleFlush(client, congested ? LIMITS.coalesceIntervalMs : LIMITS.coalesceIntervalMs - elapsed) } - private sendToClient(client: InternalClient, payload: string): void { + /** + * Re-attempt a coalesced update once the client's window has elapsed. + * + * Goes back through {@link deliver}, so the buffer is re-inspected before + * anything is written: by the time this fires the client may have recovered, + * or crossed the hard limit, or been unregistered. + */ + private scheduleFlush(client: InternalClient, delayMs: number): void { + if (client.timer || client.destroyed) return + + client.timer = setTimeout(() => { + client.timer = null + if (client.destroyed) return + const pending = client.pendingUpdate + client.pendingUpdate = null + if (pending === null) return + this.deliver(client, pending, Date.now()) + }, Math.max(delayMs, 1)) + // A pending flush must not be a reason to keep the process alive. + client.timer.unref?.() + } + + /** Bytes queued on a client's socket, 0 when it cannot report any. */ + private bufferedBytesOf(client: InternalClient): number { try { - client.socket.send(payload) + const buffered = client.socket.bufferedAmount?.() + if (typeof buffered !== 'number' || !Number.isFinite(buffered) || buffered <= 0) return 0 + return buffered + } catch { + // A wrapper that throws while reporting its buffer gets the benefit of + // the doubt: worse to drop a healthy client than to serve a wedged one. + return 0 + } + } + + /** Aggregate queued bytes across every connected client, for `ws_buffered_bytes`. */ + private reportBufferedBytes(): void { + let total = 0 + for (const client of this.clients.values()) total += this.bufferedBytesOf(client) + // Aggregate only. One series per client pair would be unbounded cardinality + // driven by caller-supplied subscription strings. + ws_buffered_bytes.labels('*').set(total) + } + + /** + * Disconnect a client that cannot keep up, and say so out loud. + * + * The close reason is kept short on purpose: RFC 6455 caps a close reason at + * 123 bytes and `ws` throws past that, which would turn a backpressure + * eviction into an unhandled exception on the broadcast path. + */ + private dropSlowClient(client: InternalClient, reason: string, bufferedBytes: number): void { + if (client.pendingUpdate !== null) this.countDroppedUpdate(client) + + const metricPair = pairMetricLabel(client.pairKey) + if (metricPair) ws_clients_dropped_total.labels(metricPair).inc() + ws_clients_dropped_total.labels('*').inc() + + log.warn( + { + clientId: client.id, + pair: client.pairKey, + reason, + bufferedBytes, + congestedMs: client.congestedSince === null ? 0 : Date.now() - client.congestedSince, + }, + 'dropping websocket client that cannot keep up with the price feed' + ) + + try { + client.socket.close(CLOSE_CODE_SLOW_CONSUMER, reason) + } catch { + // The socket may already be gone; unregistering below is what matters. + } + this.unregister(client.id) + } + + private countDroppedUpdate(client: InternalClient): void { + const metricPair = pairMetricLabel(client.pairKey) + if (metricPair) ws_messages_dropped_total.labels(metricPair).inc() + ws_messages_dropped_total.labels('*').inc() + } + + private sendToClient(client: InternalClient, payload: string): void { + let settled = false + const onWritten = (err?: Error | null): void => { + if (settled) return + settled = true + if (err) { + // Client likely disconnected, or the write failed underneath us. + this.unregister(client.id) + return + } const metricPair = pairMetricLabel(client.pairKey) if (metricPair) ws_messages_sent_total.labels(metricPair).inc() ws_messages_sent_total.labels('*').inc() + } + + try { + client.socket.send(payload, onWritten) } catch { - // Client likely disconnected this.unregister(client.id) } } @@ -241,6 +448,10 @@ class FanOutManager { * Dispose all clients and clean up. */ async destroy(): Promise { + if (this.bufferSweep) { + clearInterval(this.bufferSweep) + this.bufferSweep = null + } if (this.subscriber) { try { await this.subscriber.unsubscribe(REDIS_CHANNEL)