From e0b3d05dc31081511b2b1c775f09975564a8a6f7 Mon Sep 17 00:00:00 2001 From: jamby77 Date: Mon, 12 Oct 2026 07:55:30 +0300 Subject: [PATCH 01/31] Track KV cache evictions per node and clean up removed connections - Compute the evictions delta per cluster node so an unreachable primary no longer reads as a counter reset - Stop scraping engines whose connection is gone and drop results that land after removal - Delete connection data only after pending sample writes settle --- .../__tests__/footprint-estimate.spec.ts | 16 +++++++----- .../kv-cache-engine-registry.spec.ts | 3 ++- .../kv-cache-footprint.service.spec.ts | 16 +++++++++++- .../__tests__/kv-cache-scrape.service.spec.ts | 25 +++++++++++++++--- proprietary/kv-cache/footprint-estimate.ts | 19 ++++++++++---- .../kv-cache/kv-cache-engine-registry.ts | 1 + .../kv-cache/kv-cache-footprint.service.ts | 26 ++++++++++--------- .../kv-cache/kv-cache-samples.service.ts | 4 +++ .../kv-cache/kv-cache-scrape.service.ts | 12 +++++++-- 9 files changed, 91 insertions(+), 31 deletions(-) diff --git a/proprietary/kv-cache/__tests__/footprint-estimate.spec.ts b/proprietary/kv-cache/__tests__/footprint-estimate.spec.ts index 4d5809c2e..efc9cc81f 100644 --- a/proprietary/kv-cache/__tests__/footprint-estimate.spec.ts +++ b/proprietary/kv-cache/__tests__/footprint-estimate.spec.ts @@ -1,4 +1,4 @@ -import { buildSnapshot, layoutOf, pickSample, type KeySample, type NodeObservation } from '../footprint-estimate'; +import { buildSnapshot, evictedKeysDelta, layoutOf, pickSample, type KeySample, type NodeObservation } from '../footprint-estimate'; import { classifyKey } from '../key-classifier'; const M = 'Qwen/Qwen2.5-0.5B-Instruct'; @@ -12,7 +12,7 @@ const sample = (k: string, o: Partial = {}): KeySample => ({ ...o, }); const memory = { usedMemory: 100_000_000, maxmemory: 0, maxmemoryPolicy: 'noeviction', evictedKeys: 0 }; -const base = { connectionId: 'c', timestamp: 1, memory, previousEvictedKeys: null, otherDbs: [], clientDetected: false }; +const base = { connectionId: 'c', timestamp: 1, memory, evictedKeysDelta: null, otherDbs: [], clientDetected: false }; describe('pickSample', () => { it('returns everything under the size and strides above it', () => { @@ -117,11 +117,13 @@ describe('buildSnapshot', () => { ]); }); - it('computes the eviction delta and treats a lower counter as a restart', () => { - const node: NodeObservation = { scannedKeys: 0, matchedKeys: [], scanComplete: true, dbSize: 0, samples: [] }; - expect(buildSnapshot({ ...base, nodes: [node] }).evictedKeysDelta).toBeNull(); - expect(buildSnapshot({ ...base, nodes: [node], memory: { ...memory, evictedKeys: 50 }, previousEvictedKeys: 20 }).evictedKeysDelta).toBe(30); - expect(buildSnapshot({ ...base, nodes: [node], memory: { ...memory, evictedKeys: 5 }, previousEvictedKeys: 20 }).evictedKeysDelta).toBe(5); + it('computes the eviction delta per node and treats a lower counter as a restart', () => { + expect(evictedKeysDelta(undefined, new Map([['a', 5]]))).toBeNull(); + expect(evictedKeysDelta(new Map([['a', 20]]), new Map([['a', 50]]))).toBe(30); + expect(evictedKeysDelta(new Map([['a', 20]]), new Map([['a', 5]]))).toBe(5); + expect(evictedKeysDelta(new Map([['a', 20], ['b', 1000]]), new Map([['a', 25]]))).toBe(5); + expect(evictedKeysDelta(new Map([['a', 20]]), new Map([['a', 20], ['b', 1000]]))).toBe(0); + expect(evictedKeysDelta(new Map([['a', 20]]), new Map([['b', 1000]]))).toBeNull(); }); it('detects through the GLIDE client even with no keys', () => { diff --git a/proprietary/kv-cache/__tests__/kv-cache-engine-registry.spec.ts b/proprietary/kv-cache/__tests__/kv-cache-engine-registry.spec.ts index a4d34c56b..a4724e323 100644 --- a/proprietary/kv-cache/__tests__/kv-cache-engine-registry.spec.ts +++ b/proprietary/kv-cache/__tests__/kv-cache-engine-registry.spec.ts @@ -13,7 +13,7 @@ function setup(initial = [engine('a', 'c1', 'lmc-a'), engine('b', 'c2')]) { } as any; let removal: (id: string) => void = () => undefined; const footprint = { onConnectionRemoval: jest.fn((l) => (removal = l)) } as any; - const samples = { forgetEngine: jest.fn() } as any; + const samples = { forgetEngine: jest.fn(), settled: jest.fn().mockResolvedValue(undefined) } as any; const registry = new KvCacheEngineRegistry(storage, footprint, samples); return { registry, storage, samples, removal: (id: string) => removal(id) }; } @@ -100,5 +100,6 @@ describe('KvCacheEngineRegistry', () => { expect(registry.list().map((e) => e.id)).toEqual(['b']); expect(samples.forgetEngine).toHaveBeenCalledTimes(1); expect(samples.forgetEngine).toHaveBeenCalledWith('a'); + expect(samples.settled).toHaveBeenCalled(); }); }); diff --git a/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts b/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts index f61a1c9c9..bc3299dc9 100644 --- a/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts +++ b/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts @@ -82,14 +82,28 @@ describe('KvCacheFootprintService', () => { await (service as any).tick(); registry.list.mockReturnValue([]); await (service as any).tick(); + await new Promise((resolve) => setImmediate(resolve)); expect(storage.deleteKvCacheConnectionData).toHaveBeenCalledWith('c1'); expect(storage.saveKvCacheFootprintSnapshot).not.toHaveBeenCalled(); expect(listener).toHaveBeenCalledWith('c1'); }); - it('deletes stored data when a connection is removed', () => { + it('deletes stored data when a connection is removed', async () => { const { service, storage } = setup(); (service as any).onConnectionRemoved('c1'); + await new Promise((resolve) => setImmediate(resolve)); + expect(storage.deleteKvCacheConnectionData).toHaveBeenCalledWith('c1'); + }); + + it('deletes stored data only after async removal listeners settle', async () => { + const { service, storage } = setup(); + let release: () => void = () => undefined; + service.onConnectionRemoval(() => new Promise((resolve) => (release = resolve))); + (service as any).onConnectionRemoved('c1'); + await new Promise((resolve) => setImmediate(resolve)); + expect(storage.deleteKvCacheConnectionData).not.toHaveBeenCalled(); + release(); + await new Promise((resolve) => setImmediate(resolve)); expect(storage.deleteKvCacheConnectionData).toHaveBeenCalledWith('c1'); }); diff --git a/proprietary/kv-cache/__tests__/kv-cache-scrape.service.spec.ts b/proprietary/kv-cache/__tests__/kv-cache-scrape.service.spec.ts index a1417b203..5b5bbf40c 100644 --- a/proprietary/kv-cache/__tests__/kv-cache-scrape.service.spec.ts +++ b/proprietary/kv-cache/__tests__/kv-cache-scrape.service.spec.ts @@ -19,7 +19,7 @@ const engine = (over: Record = {}) => ({ const body = 'lmcache:num_hit_tokens_total{model_name="m",worker_id="0",role="worker"} 42\n'; -function setup(engines: unknown[], licensed = true) { +function setup(engines: unknown[], licensed = true, connectionIds = ['c1']) { const registry = { list: jest.fn().mockReturnValue(engines), get: jest.fn((id: string) => (engines as { id: string }[]).find((e) => e.id === id) ?? null), @@ -28,8 +28,9 @@ function setup(engines: unknown[], licensed = true) { const samples = { observe: jest.fn() }; const engineService = { authHeaderFor: jest.fn().mockReturnValue(null) }; const license = { hasFeature: jest.fn().mockReturnValue(licensed) }; - const service = new KvCacheScrapeService(registry as any, samples as any, engineService as any, license as any); - return { registry, samples, engineService, license, service }; + const connections = { getConfig: jest.fn((id: string) => (connectionIds.includes(id) ? { id } : null)) }; + const service = new KvCacheScrapeService(registry as any, samples as any, engineService as any, license as any, connections as any); + return { registry, samples, engineService, license, connections, service }; } describe('KvCacheScrapeService', () => { @@ -47,6 +48,24 @@ describe('KvCacheScrapeService', () => { expect(fetchMock).not.toHaveBeenCalled(); }); + it('skips engines whose connection was removed', async () => { + const { service } = setup([engine()], true, []); + await service.tick(1000); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('drops a result when the connection is removed mid-fetch', async () => { + fetchMock.mockResolvedValue(body); + const { service, samples, registry, connections } = setup([engine()]); + fetchMock.mockImplementationOnce(async () => { + connections.getConfig.mockReturnValue(null); + return body; + }); + await service.tick(1000); + expect(samples.observe).not.toHaveBeenCalled(); + expect(registry.recordResult).not.toHaveBeenCalled(); + }); + it('observes cumulative samples and records success', async () => { fetchMock.mockResolvedValue(body); const { service, samples, registry } = setup([engine()]); diff --git a/proprietary/kv-cache/footprint-estimate.ts b/proprietary/kv-cache/footprint-estimate.ts index 286e60050..773b27923 100644 --- a/proprietary/kv-cache/footprint-estimate.ts +++ b/proprietary/kv-cache/footprint-estimate.ts @@ -29,13 +29,24 @@ export interface SnapshotInput { timestamp: number; nodes: NodeObservation[]; memory: MemoryStats; - previousEvictedKeys: number | null; + evictedKeysDelta: number | null; otherDbs: number[]; clientDetected: boolean; } export const DETECTION_MIN_MATCHES = 5; +export function evictedKeysDelta(previous: Map | undefined, current: Map): number | null { + if (!previous) return null; + let delta: number | null = null; + for (const [node, evicted] of current) { + const before = previous.get(node); + if (before === undefined) continue; + delta = (delta ?? 0) + (evicted >= before ? evicted - before : evicted); + } + return delta; +} + const ratio = (value: number) => Math.round(value * 1e4) / 1e4; export function pickSample(items: T[], size: number): T[] { @@ -114,8 +125,7 @@ export function buildSnapshot(input: SnapshotInput): KvCacheFootprintSnapshot { live.push(...node.samples.filter(isLive)); } const kvBytes = live.filter((s) => s.parsed.suffix === 'kv_bytes'); - const { usedMemory, maxmemory, maxmemoryPolicy, evictedKeys } = input.memory; - const previous = input.previousEvictedKeys; + const { usedMemory, maxmemory, maxmemoryPolicy } = input.memory; return { connectionId: input.connectionId, timestamp: input.timestamp, @@ -136,8 +146,7 @@ export function buildSnapshot(input: SnapshotInput): KvCacheFootprintSnapshot { kvBytes.length === 0 ? null : ratio(kvBytes.filter((s) => s.siblingExists === false).length / kvBytes.length), - evictedKeysDelta: - previous === null ? null : evictedKeys >= previous ? evictedKeys - previous : evictedKeys, + evictedKeysDelta: input.evictedKeysDelta, otherDbs: input.otherDbs, perModel: splitPerModel(live, chunks, bytes), }; diff --git a/proprietary/kv-cache/kv-cache-engine-registry.ts b/proprietary/kv-cache/kv-cache-engine-registry.ts index bf28b8f14..cf6f8275a 100644 --- a/proprietary/kv-cache/kv-cache-engine-registry.ts +++ b/proprietary/kv-cache/kv-cache-engine-registry.ts @@ -23,6 +23,7 @@ export class KvCacheEngineRegistry implements OnModuleInit { this.engines.delete(id); this.samples.forgetEngine(id); } + return this.samples.settled(); }); } diff --git a/proprietary/kv-cache/kv-cache-footprint.service.ts b/proprietary/kv-cache/kv-cache-footprint.service.ts index de8256d82..3c8a7e179 100644 --- a/proprietary/kv-cache/kv-cache-footprint.service.ts +++ b/proprietary/kv-cache/kv-cache-footprint.service.ts @@ -6,11 +6,12 @@ import { ConnectionRegistry } from '@app/connections/connection-registry.service import { LicenseService } from '@proprietary/licenses/license.service'; import { collectionPlan, type CollectionPlan } from '../key-analytics/key-analytics-plan'; import { clusterScanNodes, primaryScanNodes, type ScanNode } from '../key-analytics/scan-nodes'; -import { buildSnapshot, pickSample, type MemoryStats, type NodeObservation } from './footprint-estimate'; +import { buildSnapshot, evictedKeysDelta, pickSample, type MemoryStats, type NodeObservation } from './footprint-estimate'; import { hasLmcacheGlideClient, otherNonEmptyDbs, readMemoryStats, sampleLmcacheKeys, scanLmcacheKeys } from './footprint-scanner'; import { classifyKey, maskKey } from './key-classifier'; export type SnapshotListener = (snapshot: KvCacheFootprintSnapshot) => void | Promise; +export type RemovalListener = (connectionId: string) => void | Promise; interface NodeReading { observation: NodeObservation; @@ -42,10 +43,10 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On }; private readonly sampleSize = resolvePositiveInt(process.env.KV_CACHE_SAMPLE_KEYS, 500, 1); private readonly running = new Set(); - private readonly lastEvicted = new Map(); + private readonly lastEvicted = new Map>(); private readonly sampleKeys = new Map(); private readonly listeners: SnapshotListener[] = []; - private readonly removalListeners: Array<(connectionId: string) => void> = []; + private readonly removalListeners: RemovalListener[] = []; constructor( connectionRegistry: ConnectionRegistry, @@ -72,19 +73,19 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On this.running.delete(connectionId); this.lastEvicted.delete(connectionId); this.sampleKeys.delete(connectionId); - void this.storage - .deleteKvCacheConnectionData(connectionId) - .catch((error) => this.logger.warn(`Could not delete KV cache data for ${connectionId}: ${errorMessage(error)}`)); - for (const listener of this.removalListeners) { + const pending = this.removalListeners.map(async (listener) => { try { - listener(connectionId); + await listener(connectionId); } catch (error) { this.logger.warn(`KV cache removal listener failed: ${errorMessage(error)}`); } - } + }); + void Promise.all(pending) + .then(() => this.storage.deleteKvCacheConnectionData(connectionId)) + .catch((error) => this.logger.warn(`Could not delete KV cache data for ${connectionId}: ${errorMessage(error)}`)); } - onConnectionRemoval(listener: (connectionId: string) => void): void { + onConnectionRemoval(listener: RemovalListener): void { this.removalListeners.push(listener); } @@ -117,17 +118,18 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On const ownDb = this.connectionRegistry.getConfig(ctx.connectionId)?.dbIndex ?? 0; const readings = await Promise.all(nodes.map((node) => this.read(node, ownDb))); const memory = this.sumMemory(readings.map((reading) => reading.memory)); + const evicted = new Map(nodes.map((node, index) => [node.name, readings[index].memory.evictedKeys])); const snapshot = buildSnapshot({ connectionId: ctx.connectionId, timestamp: Date.now(), nodes: readings.map((reading) => reading.observation), memory, - previousEvictedKeys: this.lastEvicted.get(ctx.connectionId) ?? null, + evictedKeysDelta: evictedKeysDelta(this.lastEvicted.get(ctx.connectionId), evicted), otherDbs: readings[0]?.otherDbs ?? [], clientDetected: readings.some((reading) => reading.glide), }); await this.storage.saveKvCacheFootprintSnapshot(snapshot); - this.lastEvicted.set(ctx.connectionId, memory.evictedKeys); + this.lastEvicted.set(ctx.connectionId, new Map([...(this.lastEvicted.get(ctx.connectionId) ?? []), ...evicted])); this.rememberSampleKey(ctx.connectionId, readings); await this.announce(snapshot); return snapshot; diff --git a/proprietary/kv-cache/kv-cache-samples.service.ts b/proprietary/kv-cache/kv-cache-samples.service.ts index 248cb6afb..03d65dfae 100644 --- a/proprietary/kv-cache/kv-cache-samples.service.ts +++ b/proprietary/kv-cache/kv-cache-samples.service.ts @@ -67,6 +67,10 @@ export class KvCacheSamplesService implements OnModuleInit, OnModuleDestroy { this.tracker.forgetEngine(engineId); } + settled(): Promise { + return this.flushing; + } + async deleteEngine(engineId: string): Promise { this.forgetEngine(engineId); await this.flushing; diff --git a/proprietary/kv-cache/kv-cache-scrape.service.ts b/proprietary/kv-cache/kv-cache-scrape.service.ts index 7c8e17a5d..1bd3dcee7 100644 --- a/proprietary/kv-cache/kv-cache-scrape.service.ts +++ b/proprietary/kv-cache/kv-cache-scrape.service.ts @@ -1,6 +1,7 @@ import { Injectable, Logger, OnModuleDestroy, OnModuleInit } from '@nestjs/common'; import { Feature } from '@betterdb/shared'; import type { StoredKvCacheEngine } from '@app/common/interfaces/storage-port.interface'; +import { ConnectionRegistry } from '@app/connections/connection-registry.service'; import { LicenseService } from '@proprietary/licenses/license.service'; import { KvCacheEngineRegistry } from './kv-cache-engine-registry'; import { KvCacheEnginesService } from './kv-cache-engines.service'; @@ -28,6 +29,7 @@ export class KvCacheScrapeService implements OnModuleInit, OnModuleDestroy { private readonly samples: KvCacheSamplesService, private readonly engines: KvCacheEnginesService, private readonly license: LicenseService, + private readonly connections: ConnectionRegistry, ) {} onModuleInit(): void { @@ -46,7 +48,9 @@ export class KvCacheScrapeService implements OnModuleInit, OnModuleDestroy { if (this.running || !this.license.hasFeature(Feature.KV_CACHE_MONITORING)) return; this.running = true; try { - const targets = this.registry.list().filter((e) => e.enabled && e.source === 'scrape' && e.scrapeUrl); + const targets = this.registry + .list() + .filter((e) => e.enabled && e.source === 'scrape' && e.scrapeUrl && this.connectionExists(e.connectionId)); await Promise.allSettled(targets.map((engine) => this.scrape(engine, now))); } finally { this.running = false; @@ -89,6 +93,10 @@ export class KvCacheScrapeService implements OnModuleInit, OnModuleDestroy { private stillCurrent(engine: StoredKvCacheEngine): boolean { const current = this.registry.get(engine.id); - return !!current?.enabled && current.scrapeUrl === engine.scrapeUrl; + return !!current?.enabled && current.scrapeUrl === engine.scrapeUrl && this.connectionExists(engine.connectionId); + } + + private connectionExists(connectionId: string): boolean { + return this.connections.getConfig(connectionId) !== null; } } From b2c3353ccaa1457e9d6696ca7c0fb0575481a692 Mon Sep 17 00:00:00 2001 From: jamby77 Date: Mon, 12 Oct 2026 07:56:59 +0300 Subject: [PATCH 02/31] Flag KV cache sample ranges that hit the row limit - Return a truncated flag when a samples query reaches 50,000 rows - Tell the user on the KV Cache page and in the docs --- apps/web/src/pages/KvCache.tsx | 6 ++++++ docs/kv-cache-monitoring.md | 2 ++ packages/shared/src/types/kv-cache.types.ts | 1 + .../kv-cache/__tests__/kv-cache-samples.service.spec.ts | 8 +++++++- proprietary/kv-cache/kv-cache-samples.service.ts | 2 +- 5 files changed, 17 insertions(+), 2 deletions(-) diff --git a/apps/web/src/pages/KvCache.tsx b/apps/web/src/pages/KvCache.tsx index bc3b4ce44..c2b51b0b3 100644 --- a/apps/web/src/pages/KvCache.tsx +++ b/apps/web/src/pages/KvCache.tsx @@ -103,6 +103,12 @@ export function KvCache() { recentBuckets={recent.data?.buckets ?? []} /> + {samples.data?.truncated && ( +

+ This range holds more samples than can be shown, so the charts and table cover only its most recent + part. Pick a shorter range to see all of it. +

+ )} diff --git a/docs/kv-cache-monitoring.md b/docs/kv-cache-monitoring.md index dbf8fc0be..65bc9b229 100644 --- a/docs/kv-cache-monitoring.md +++ b/docs/kv-cache-monitoring.md @@ -41,6 +41,8 @@ that hold keys. Page controls: - A date range picker for the charts, defaulting to the last 6 hours. + A range covers at most 31 days and 50,000 stored samples. When a range holds more + samples, the page shows only the most recent ones and says so. - **Refresh**, which re-collects the footprint now. - The alert settings sheet (see [Per-connection switches](#per-connection-switches)). - **Rescan now** on the not-detected view, which collects the footprint again. diff --git a/packages/shared/src/types/kv-cache.types.ts b/packages/shared/src/types/kv-cache.types.ts index 7c3c95303..9d7209467 100644 --- a/packages/shared/src/types/kv-cache.types.ts +++ b/packages/shared/src/types/kv-cache.types.ts @@ -87,6 +87,7 @@ export interface KvCacheSampleBucket extends KvCacheEngineSample { export interface KvCacheSamplesResponse { buckets: KvCacheSampleBucket[]; rangeHitRate: number | null; + truncated: boolean; } export interface KvCacheStatus { diff --git a/proprietary/kv-cache/__tests__/kv-cache-samples.service.spec.ts b/proprietary/kv-cache/__tests__/kv-cache-samples.service.spec.ts index 62d895606..4fc55e3ef 100644 --- a/proprietary/kv-cache/__tests__/kv-cache-samples.service.spec.ts +++ b/proprietary/kv-cache/__tests__/kv-cache-samples.service.spec.ts @@ -160,9 +160,15 @@ describe('KvCacheSamplesService', () => { expect(result.rangeHitRate).toBe(0.75); }); + it('flags a result that hit the row limit', async () => { + const { service, storage } = setup(); + storage.getKvCacheEngineSamples.mockResolvedValue(Array.from({ length: SAMPLES_LIMIT }, (_, i) => sample({ timestamp: i }))); + expect((await service.getSamples('c1', { from: 0, to: 1000 })).truncated).toBe(true); + }); + it('returns a null range hit rate when nothing was requested', async () => { const { service } = setup(); - expect(await service.getSamples('c1', { from: 0, to: 1000 })).toEqual({ buckets: [], rangeHitRate: null }); + expect(await service.getSamples('c1', { from: 0, to: 1000 })).toEqual({ buckets: [], rangeHitRate: null, truncated: false }); }); it('flushes the open bucket on module destroy and stops the timer', async () => { diff --git a/proprietary/kv-cache/kv-cache-samples.service.ts b/proprietary/kv-cache/kv-cache-samples.service.ts index 03d65dfae..02daca933 100644 --- a/proprietary/kv-cache/kv-cache-samples.service.ts +++ b/proprietary/kv-cache/kv-cache-samples.service.ts @@ -108,6 +108,6 @@ export class KvCacheSamplesService implements OnModuleInit, OnModuleDestroy { }); const requested = rows.reduce((sum, row) => sum + row.requestedTokens, 0); const hit = rows.reduce((sum, row) => sum + row.hitTokens, 0); - return { buckets: mergeBuckets(rows), rangeHitRate: hitRate(requested, hit) }; + return { buckets: mergeBuckets(rows), rangeHitRate: hitRate(requested, hit), truncated: rows.length >= SAMPLES_LIMIT }; } } From 5d99ae693695590097ac2192c97c8fbd6d76096e Mon Sep 17 00:00:00 2001 From: jamby77 Date: Mon, 12 Oct 2026 07:59:13 +0300 Subject: [PATCH 03/31] Pin KV cache scrapes to DNS-validated addresses - Resolve scrape hosts through a guarded lookup at connect time so a changed DNS answer cannot reach a refused address - Map connect-time refusals to the blocked address scrape error --- docs/kv-cache-monitoring.md | 5 ++ .../kv-cache/__tests__/metrics-fetch.spec.ts | 6 ++ .../kv-cache/__tests__/pinned-fetch.spec.ts | 83 +++++++++++++++++++ proprietary/kv-cache/metrics-fetch.ts | 8 +- proprietary/kv-cache/pinned-fetch.ts | 63 ++++++++++++++ 5 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 proprietary/kv-cache/__tests__/pinned-fetch.spec.ts create mode 100644 proprietary/kv-cache/pinned-fetch.ts diff --git a/docs/kv-cache-monitoring.md b/docs/kv-cache-monitoring.md index 65bc9b229..8586b269e 100644 --- a/docs/kv-cache-monitoring.md +++ b/docs/kv-cache-monitoring.md @@ -182,6 +182,11 @@ addresses as well. Link-local addresses (`169.254.0.0/16`, which includes cloud metadata endpoints, and `fe80::/10`) are always refused, whatever the setting. +The guard checks every address a hostname resolves to, and the scrape then +connects only to an address that passed the check. A DNS answer that changes +between the check and the connection cannot redirect a scrape to a refused +address. + When you change the metrics URL of a linked engine to a different origin (scheme, host or port) and a stored auth header exists, BetterDB refuses the update with a `400` unless you also supply an auth header. Send an empty diff --git a/proprietary/kv-cache/__tests__/metrics-fetch.spec.ts b/proprietary/kv-cache/__tests__/metrics-fetch.spec.ts index 88a19f859..43265bdcc 100644 --- a/proprietary/kv-cache/__tests__/metrics-fetch.spec.ts +++ b/proprietary/kv-cache/__tests__/metrics-fetch.spec.ts @@ -1,5 +1,6 @@ import { assertSafeOutboundUrl } from '@app/common/utils/outbound-url-guard'; import { fetchMetricsText, ScrapeError } from '../metrics-fetch'; +import { BlockedAddressError } from '../pinned-fetch'; jest.mock('@app/common/utils/outbound-url-guard', () => ({ assertSafeOutboundUrl: jest.fn().mockResolvedValue(new URL('http://x')), @@ -42,6 +43,11 @@ describe('fetchMetricsText', () => { } }); + it('maps a connect-time blocked address to blocked address', async () => { + const blocked = jest.fn().mockRejectedValue(new BlockedAddressError('blocked address 169.254.169.254')); + await expect(fetchMetricsText('http://x', null, blocked)).rejects.toThrow('blocked address'); + }); + it('maps fetch failures to timeout and connection failed', async () => { const timeout = jest.fn().mockRejectedValue(Object.assign(new Error('t'), { name: 'TimeoutError' })); await expect(fetchMetricsText('http://x', null, timeout)).rejects.toThrow('timeout'); diff --git a/proprietary/kv-cache/__tests__/pinned-fetch.spec.ts b/proprietary/kv-cache/__tests__/pinned-fetch.spec.ts new file mode 100644 index 000000000..a58e2a13d --- /dev/null +++ b/proprietary/kv-cache/__tests__/pinned-fetch.spec.ts @@ -0,0 +1,83 @@ +import { createServer, type Server } from 'http'; +import type { AddressInfo } from 'net'; +import { BlockedAddressError, guardedLookup, pinnedFetch } from '../pinned-fetch'; + +const resolver = (addresses: Array<{ address: string; family: number }>) => + jest.fn((_host: string, _options: unknown, callback: (error: null, addresses: Array<{ address: string; family: number }>) => void) => + callback(null, addresses), + ); + +const lookupOnce = (lookup: ReturnType, host: string, all = false) => + new Promise<{ error: Error | null; address: unknown; family?: number }>((resolve) => + lookup(host, { all } as any, (error, address, family) => resolve({ error, address, family })), + ); + +describe('guardedLookup', () => { + const env = process.env.NODE_ENV; + afterEach(() => { + process.env.NODE_ENV = env; + }); + + it('returns a public address in single and all modes', async () => { + const lookup = guardedLookup(false, resolver([{ address: '8.8.8.8', family: 4 }])); + expect(await lookupOnce(lookup, 'example.com')).toEqual({ error: null, address: '8.8.8.8', family: 4 }); + expect((await lookupOnce(lookup, 'example.com', true)).address).toEqual([{ address: '8.8.8.8', family: 4 }]); + }); + + it('refuses a host when any answer is blocked', async () => { + const lookup = guardedLookup(true, resolver([{ address: '8.8.8.8', family: 4 }, { address: '169.254.169.254', family: 4 }])); + expect((await lookupOnce(lookup, 'rebind.example')).error).toBeInstanceOf(BlockedAddressError); + }); + + it('applies the private network setting', async () => { + const answers = [{ address: '10.0.0.5', family: 4 }]; + expect((await lookupOnce(guardedLookup(false, resolver(answers)), 'vllm.internal')).error).toBeInstanceOf(BlockedAddressError); + expect((await lookupOnce(guardedLookup(true, resolver(answers)), 'vllm.internal')).address).toBe('10.0.0.5'); + }); + + it('allows localhost outside production only', async () => { + const answers = [{ address: '127.0.0.1', family: 4 }]; + process.env.NODE_ENV = 'development'; + expect((await lookupOnce(guardedLookup(true, resolver(answers)), 'localhost')).address).toBe('127.0.0.1'); + expect((await lookupOnce(guardedLookup(true, resolver(answers)), 'evil.example')).error).toBeInstanceOf(BlockedAddressError); + process.env.NODE_ENV = 'production'; + expect((await lookupOnce(guardedLookup(true, resolver(answers)), 'localhost')).error).toBeInstanceOf(BlockedAddressError); + }); +}); + +describe('pinnedFetch', () => { + let server: Server; + let port: number; + + beforeAll(async () => { + server = createServer((request, response) => { + if (request.url === '/redirect') { + response.writeHead(302, { location: '/metrics' }); + response.end(); + return; + } + response.writeHead(200, { 'content-type': 'text/plain', 'x-auth': request.headers.authorization ?? '' }); + response.end('lmcache:a_total 1'); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + port = (server.address() as AddressInfo).port; + }); + + afterAll(() => new Promise((resolve) => server.close(() => resolve()))); + + it('fetches through the validated address and does not follow redirects', async () => { + const fetchImpl = pinnedFetch(true, resolver([{ address: '127.0.0.1', family: 4 }])); + const ok = await fetchImpl(`http://localhost:${port}/metrics`, { headers: { Authorization: 'Bearer t' } }); + expect(ok.status).toBe(200); + expect(ok.headers.get('x-auth')).toBe('Bearer t'); + expect(await ok.text()).toBe('lmcache:a_total 1'); + const redirect = await fetchImpl(`http://localhost:${port}/redirect`); + expect(redirect.status).toBe(302); + await redirect.body?.cancel(); + }); + + it('rejects when the host resolves to a blocked address at connect time', async () => { + const fetchImpl = pinnedFetch(true, resolver([{ address: '169.254.169.254', family: 4 }])); + await expect(fetchImpl(`http://metadata.example:${port}/`)).rejects.toBeInstanceOf(BlockedAddressError); + }); +}); diff --git a/proprietary/kv-cache/metrics-fetch.ts b/proprietary/kv-cache/metrics-fetch.ts index 116274f86..670a7d980 100644 --- a/proprietary/kv-cache/metrics-fetch.ts +++ b/proprietary/kv-cache/metrics-fetch.ts @@ -1,4 +1,5 @@ import { assertSafeOutboundUrl } from '@app/common/utils/outbound-url-guard'; +import { BlockedAddressError, pinnedFetch } from './pinned-fetch'; export const SCRAPE_TIMEOUT_MS = 5_000; export const SCRAPE_MAX_BYTES = 5 * 1024 * 1024; @@ -33,7 +34,11 @@ async function readCapped(response: Response): Promise { return Buffer.concat(chunks).toString('utf8'); } -export async function fetchMetricsText(url: string, authHeader: string | null, fetchImpl: typeof fetch = fetch): Promise { +export async function fetchMetricsText( + url: string, + authHeader: string | null, + fetchImpl: typeof fetch = pinnedFetch(scrapeAllowsPrivateNetworks()), +): Promise { try { await assertSafeOutboundUrl(url, { label: 'metrics URL', allowPrivateNetworks: scrapeAllowsPrivateNetworks() }); } catch { @@ -47,6 +52,7 @@ export async function fetchMetricsText(url: string, authHeader: string | null, f headers: { Accept: 'text/plain', ...(authHeader ? { Authorization: authHeader } : {}) }, }); } catch (error) { + if (error instanceof BlockedAddressError) throw new ScrapeError('blocked address'); const name = (error as Error)?.name; throw new ScrapeError(name === 'TimeoutError' || name === 'AbortError' ? 'timeout' : 'connection failed'); } diff --git a/proprietary/kv-cache/pinned-fetch.ts b/proprietary/kv-cache/pinned-fetch.ts new file mode 100644 index 000000000..ab4ac43d4 --- /dev/null +++ b/proprietary/kv-cache/pinned-fetch.ts @@ -0,0 +1,63 @@ +import { lookup as dnsLookup, type LookupAddress, type LookupAllOptions, type LookupOptions } from 'dns'; +import http from 'http'; +import https from 'https'; +import { Readable } from 'stream'; +import { isBlockedIp } from '@app/common/utils/outbound-url-guard'; + +type LookupCallback = (error: NodeJS.ErrnoException | null, address: string | LookupAddress[], family?: number) => void; +type Resolver = ( + hostname: string, + options: LookupAllOptions, + callback: (error: NodeJS.ErrnoException | null, addresses: LookupAddress[]) => void, +) => void; + +export class BlockedAddressError extends Error { + readonly code = 'EBLOCKED'; +} + +export function guardedLookup(allowPrivateNetworks: boolean, resolve: Resolver = dnsLookup as Resolver) { + return (hostname: string, options: LookupOptions, callback: LookupCallback): void => { + resolve(hostname, { ...options, all: true }, (error, addresses) => { + if (error) return callback(error, ''); + const trustedLocalhost = hostname === 'localhost' && process.env.NODE_ENV !== 'production'; + const blocked = trustedLocalhost ? undefined : addresses.find((entry) => isBlockedIp(entry.address, allowPrivateNetworks)); + if (blocked) return callback(new BlockedAddressError(`blocked address ${blocked.address}`), ''); + if (addresses.length === 0) return callback(Object.assign(new Error(`no address for ${hostname}`), { code: 'ENOTFOUND' }), ''); + if (options.all) return callback(null, addresses); + callback(null, addresses[0].address, addresses[0].family); + }); + }; +} + +function responseHeaders(raw: http.IncomingHttpHeaders): Headers { + const headers = new Headers(); + for (const [name, value] of Object.entries(raw)) { + if (value !== undefined) headers.set(name, Array.isArray(value) ? value.join(', ') : value); + } + return headers; +} + +export function pinnedFetch(allowPrivateNetworks: boolean, resolve?: Resolver): typeof fetch { + const lookup = guardedLookup(allowPrivateNetworks, resolve); + return ((input: string | URL, init?: RequestInit) => + new Promise((resolveResponse, rejectResponse) => { + const url = new URL(String(input)); + const client = url.protocol === 'https:' ? https : http; + const request = client.request( + url, + { + method: 'GET', + headers: init?.headers as Record | undefined, + signal: init?.signal ?? undefined, + lookup: lookup as unknown as http.RequestOptions['lookup'], + }, + (response) => { + const status = response.statusCode ?? 502; + const body = status === 204 || status === 304 ? null : (Readable.toWeb(response) as unknown as ReadableStream); + resolveResponse(new Response(body, { status, headers: responseHeaders(response.headers) })); + }, + ); + request.on('error', rejectResponse); + request.end(); + })) as typeof fetch; +} From 557a5493457f837223772241bfa01d06c451e94b Mon Sep 17 00:00:00 2001 From: jamby77 Date: Mon, 12 Oct 2026 08:31:55 +0300 Subject: [PATCH 04/31] Address review on KV cache follow-ups - Fetch one row past the samples limit so truncated means rows were cut - Discard an in-flight footprint collection when its connection is removed and delete data only after it settles - Allow only loopback answers for the dev localhost exception - Return a null body for HTTP 205 and reject response build failures --- .../kv-cache-footprint.service.spec.ts | 19 ++++++ .../kv-cache-samples.service.spec.ts | 11 +++- .../kv-cache/__tests__/pinned-fetch.spec.ts | 14 ++++ .../kv-cache/kv-cache-footprint.service.ts | 66 ++++++++++++------- .../kv-cache/kv-cache-samples.service.ts | 8 ++- proprietary/kv-cache/pinned-fetch.ts | 20 ++++-- 6 files changed, 105 insertions(+), 33 deletions(-) diff --git a/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts b/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts index bc3299dc9..673e55ac4 100644 --- a/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts +++ b/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts @@ -107,6 +107,25 @@ describe('KvCacheFootprintService', () => { expect(storage.deleteKvCacheConnectionData).toHaveBeenCalledWith('c1'); }); + it('discards an in-flight collection and deletes data after it settles when the connection is removed', async () => { + const { service, storage, client } = setup(); + let finishRead: (size: number) => void = () => undefined; + client.getDbSize.mockImplementationOnce(() => new Promise((resolve) => (finishRead = resolve))); + const listener = jest.fn(); + service.onSnapshot(listener); + const collecting = service.triggerCollection('c1'); + await new Promise((resolve) => setImmediate(resolve)); + (service as any).onConnectionRemoved('c1'); + await new Promise((resolve) => setImmediate(resolve)); + expect(storage.deleteKvCacheConnectionData).not.toHaveBeenCalled(); + finishRead(6); + await expect(collecting).resolves.toBeNull(); + await new Promise((resolve) => setImmediate(resolve)); + expect(storage.saveKvCacheFootprintSnapshot).not.toHaveBeenCalled(); + expect(listener).not.toHaveBeenCalled(); + expect(storage.deleteKvCacheConnectionData).toHaveBeenCalledWith('c1'); + }); + it('notifies removal listeners with the connection id', () => { const { service } = setup(); const listener = jest.fn(); diff --git a/proprietary/kv-cache/__tests__/kv-cache-samples.service.spec.ts b/proprietary/kv-cache/__tests__/kv-cache-samples.service.spec.ts index 4fc55e3ef..e815e5154 100644 --- a/proprietary/kv-cache/__tests__/kv-cache-samples.service.spec.ts +++ b/proprietary/kv-cache/__tests__/kv-cache-samples.service.spec.ts @@ -152,7 +152,7 @@ describe('KvCacheSamplesService', () => { connectionId: 'c1', from: to - SAMPLES_MAX_RANGE_MS, to, - limit: SAMPLES_LIMIT, + limit: SAMPLES_LIMIT + 1, engineId: 'e1', modelName: 'm', }); @@ -160,10 +160,15 @@ describe('KvCacheSamplesService', () => { expect(result.rangeHitRate).toBe(0.75); }); - it('flags a result that hit the row limit', async () => { + it('flags a result only when rows beyond the limit exist', async () => { const { service, storage } = setup(); storage.getKvCacheEngineSamples.mockResolvedValue(Array.from({ length: SAMPLES_LIMIT }, (_, i) => sample({ timestamp: i }))); - expect((await service.getSamples('c1', { from: 0, to: 1000 })).truncated).toBe(true); + expect((await service.getSamples('c1', { from: 0, to: 1000 })).truncated).toBe(false); + storage.getKvCacheEngineSamples.mockResolvedValue(Array.from({ length: SAMPLES_LIMIT + 1 }, (_, i) => sample({ timestamp: i }))); + const result = await service.getSamples('c1', { from: 0, to: 1000 }); + expect(result.truncated).toBe(true); + expect(result.buckets).toHaveLength(SAMPLES_LIMIT); + expect(result.buckets[0].timestamp).toBe(1); }); it('returns a null range hit rate when nothing was requested', async () => { diff --git a/proprietary/kv-cache/__tests__/pinned-fetch.spec.ts b/proprietary/kv-cache/__tests__/pinned-fetch.spec.ts index a58e2a13d..8e2c49b92 100644 --- a/proprietary/kv-cache/__tests__/pinned-fetch.spec.ts +++ b/proprietary/kv-cache/__tests__/pinned-fetch.spec.ts @@ -40,6 +40,8 @@ describe('guardedLookup', () => { process.env.NODE_ENV = 'development'; expect((await lookupOnce(guardedLookup(true, resolver(answers)), 'localhost')).address).toBe('127.0.0.1'); expect((await lookupOnce(guardedLookup(true, resolver(answers)), 'evil.example')).error).toBeInstanceOf(BlockedAddressError); + const linkLocal = [{ address: '169.254.169.254', family: 4 }]; + expect((await lookupOnce(guardedLookup(true, resolver(linkLocal)), 'localhost')).error).toBeInstanceOf(BlockedAddressError); process.env.NODE_ENV = 'production'; expect((await lookupOnce(guardedLookup(true, resolver(answers)), 'localhost')).error).toBeInstanceOf(BlockedAddressError); }); @@ -51,6 +53,11 @@ describe('pinnedFetch', () => { beforeAll(async () => { server = createServer((request, response) => { + if (request.url === '/reset') { + response.writeHead(205); + response.end('ignored'); + return; + } if (request.url === '/redirect') { response.writeHead(302, { location: '/metrics' }); response.end(); @@ -76,6 +83,13 @@ describe('pinnedFetch', () => { await redirect.body?.cancel(); }); + it('returns a null body for 205', async () => { + const fetchImpl = pinnedFetch(true, resolver([{ address: '127.0.0.1', family: 4 }])); + const reset = await fetchImpl(`http://localhost:${port}/reset`); + expect(reset.status).toBe(205); + expect(reset.body).toBeNull(); + }); + it('rejects when the host resolves to a blocked address at connect time', async () => { const fetchImpl = pinnedFetch(true, resolver([{ address: '169.254.169.254', family: 4 }])); await expect(fetchImpl(`http://metadata.example:${port}/`)).rejects.toBeInstanceOf(BlockedAddressError); diff --git a/proprietary/kv-cache/kv-cache-footprint.service.ts b/proprietary/kv-cache/kv-cache-footprint.service.ts index 3c8a7e179..23fff7746 100644 --- a/proprietary/kv-cache/kv-cache-footprint.service.ts +++ b/proprietary/kv-cache/kv-cache-footprint.service.ts @@ -42,7 +42,8 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On maxMatched: resolvePositiveInt(process.env.KV_CACHE_MATCH_MAX_KEYS, 2000, 1), }; private readonly sampleSize = resolvePositiveInt(process.env.KV_CACHE_SAMPLE_KEYS, 500, 1); - private readonly running = new Set(); + private readonly running = new Map>(); + private readonly generations = new Map(); private readonly lastEvicted = new Map>(); private readonly sampleKeys = new Map(); private readonly listeners: SnapshotListener[] = []; @@ -70,9 +71,15 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On } protected onConnectionRemoved(connectionId: string): void { + const active = this.running.get(connectionId); this.running.delete(connectionId); + this.generations.set(connectionId, this.generationOf(connectionId) + 1); this.lastEvicted.delete(connectionId); this.sampleKeys.delete(connectionId); + const settled = active?.then( + () => undefined, + () => undefined, + ); const pending = this.removalListeners.map(async (listener) => { try { await listener(connectionId); @@ -80,7 +87,7 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On this.logger.warn(`KV cache removal listener failed: ${errorMessage(error)}`); } }); - void Promise.all(pending) + void Promise.all([...pending, settled]) .then(() => this.storage.deleteKvCacheConnectionData(connectionId)) .catch((error) => this.logger.warn(`Could not delete KV cache data for ${connectionId}: ${errorMessage(error)}`)); } @@ -112,32 +119,45 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On async collect(ctx: ConnectionContext): Promise { const plan = this.planFor(ctx.connectionId); if (plan.kind === 'skip' || this.running.has(ctx.connectionId)) return null; - this.running.add(ctx.connectionId); + const run = this.runCollection(ctx, plan); + this.running.set(ctx.connectionId, run); try { - const nodes = await this.nodesFor(ctx, plan); - const ownDb = this.connectionRegistry.getConfig(ctx.connectionId)?.dbIndex ?? 0; - const readings = await Promise.all(nodes.map((node) => this.read(node, ownDb))); - const memory = this.sumMemory(readings.map((reading) => reading.memory)); - const evicted = new Map(nodes.map((node, index) => [node.name, readings[index].memory.evictedKeys])); - const snapshot = buildSnapshot({ - connectionId: ctx.connectionId, - timestamp: Date.now(), - nodes: readings.map((reading) => reading.observation), - memory, - evictedKeysDelta: evictedKeysDelta(this.lastEvicted.get(ctx.connectionId), evicted), - otherDbs: readings[0]?.otherDbs ?? [], - clientDetected: readings.some((reading) => reading.glide), - }); - await this.storage.saveKvCacheFootprintSnapshot(snapshot); - this.lastEvicted.set(ctx.connectionId, new Map([...(this.lastEvicted.get(ctx.connectionId) ?? []), ...evicted])); - this.rememberSampleKey(ctx.connectionId, readings); - await this.announce(snapshot); - return snapshot; + return await run; } finally { - this.running.delete(ctx.connectionId); + if (this.running.get(ctx.connectionId) === run) this.running.delete(ctx.connectionId); } } + private generationOf(connectionId: string): number { + return this.generations.get(connectionId) ?? 0; + } + + private async runCollection(ctx: ConnectionContext, plan: CollectionPlan): Promise { + const generation = this.generationOf(ctx.connectionId); + const current = () => this.generationOf(ctx.connectionId) === generation; + const nodes = await this.nodesFor(ctx, plan); + const ownDb = this.connectionRegistry.getConfig(ctx.connectionId)?.dbIndex ?? 0; + const readings = await Promise.all(nodes.map((node) => this.read(node, ownDb))); + const memory = this.sumMemory(readings.map((reading) => reading.memory)); + const evicted = new Map(nodes.map((node, index) => [node.name, readings[index].memory.evictedKeys])); + const snapshot = buildSnapshot({ + connectionId: ctx.connectionId, + timestamp: Date.now(), + nodes: readings.map((reading) => reading.observation), + memory, + evictedKeysDelta: evictedKeysDelta(this.lastEvicted.get(ctx.connectionId), evicted), + otherDbs: readings[0]?.otherDbs ?? [], + clientDetected: readings.some((reading) => reading.glide), + }); + if (!current()) return null; + await this.storage.saveKvCacheFootprintSnapshot(snapshot); + if (!current()) return null; + this.lastEvicted.set(ctx.connectionId, new Map([...(this.lastEvicted.get(ctx.connectionId) ?? []), ...evicted])); + this.rememberSampleKey(ctx.connectionId, readings); + await this.announce(snapshot); + return snapshot; + } + private planFor(connectionId: string): CollectionPlan { const connections = this.connectionRegistry.list(); const connection = connections.find((candidate) => candidate.id === connectionId); diff --git a/proprietary/kv-cache/kv-cache-samples.service.ts b/proprietary/kv-cache/kv-cache-samples.service.ts index 02daca933..038a270c9 100644 --- a/proprietary/kv-cache/kv-cache-samples.service.ts +++ b/proprietary/kv-cache/kv-cache-samples.service.ts @@ -98,16 +98,18 @@ export class KvCacheSamplesService implements OnModuleInit, OnModuleDestroy { connectionId: string, query: { from: number; to: number; engineId?: string; model?: string }, ): Promise { - const rows = await this.storage.getKvCacheEngineSamples({ + const fetched = await this.storage.getKvCacheEngineSamples({ connectionId, from: Math.max(query.from, query.to - SAMPLES_MAX_RANGE_MS), to: query.to, - limit: SAMPLES_LIMIT, + limit: SAMPLES_LIMIT + 1, engineId: query.engineId, modelName: query.model, }); + const truncated = fetched.length > SAMPLES_LIMIT; + const rows = truncated ? fetched.slice(fetched.length - SAMPLES_LIMIT) : fetched; const requested = rows.reduce((sum, row) => sum + row.requestedTokens, 0); const hit = rows.reduce((sum, row) => sum + row.hitTokens, 0); - return { buckets: mergeBuckets(rows), rangeHitRate: hitRate(requested, hit), truncated: rows.length >= SAMPLES_LIMIT }; + return { buckets: mergeBuckets(rows), rangeHitRate: hitRate(requested, hit), truncated }; } } diff --git a/proprietary/kv-cache/pinned-fetch.ts b/proprietary/kv-cache/pinned-fetch.ts index ab4ac43d4..a4894c8f6 100644 --- a/proprietary/kv-cache/pinned-fetch.ts +++ b/proprietary/kv-cache/pinned-fetch.ts @@ -11,6 +11,10 @@ type Resolver = ( callback: (error: NodeJS.ErrnoException | null, addresses: LookupAddress[]) => void, ) => void; +const LOOPBACK_ANSWER = /^(127\.|::1$|::ffff:127\.)/i; + +const NULL_BODY_STATUSES = new Set([204, 205, 304]); + export class BlockedAddressError extends Error { readonly code = 'EBLOCKED'; } @@ -20,7 +24,9 @@ export function guardedLookup(allowPrivateNetworks: boolean, resolve: Resolver = resolve(hostname, { ...options, all: true }, (error, addresses) => { if (error) return callback(error, ''); const trustedLocalhost = hostname === 'localhost' && process.env.NODE_ENV !== 'production'; - const blocked = trustedLocalhost ? undefined : addresses.find((entry) => isBlockedIp(entry.address, allowPrivateNetworks)); + const blocked = addresses.find( + (entry) => !(trustedLocalhost && LOOPBACK_ANSWER.test(entry.address)) && isBlockedIp(entry.address, allowPrivateNetworks), + ); if (blocked) return callback(new BlockedAddressError(`blocked address ${blocked.address}`), ''); if (addresses.length === 0) return callback(Object.assign(new Error(`no address for ${hostname}`), { code: 'ENOTFOUND' }), ''); if (options.all) return callback(null, addresses); @@ -52,9 +58,15 @@ export function pinnedFetch(allowPrivateNetworks: boolean, resolve?: Resolver): lookup: lookup as unknown as http.RequestOptions['lookup'], }, (response) => { - const status = response.statusCode ?? 502; - const body = status === 204 || status === 304 ? null : (Readable.toWeb(response) as unknown as ReadableStream); - resolveResponse(new Response(body, { status, headers: responseHeaders(response.headers) })); + try { + const status = response.statusCode ?? 502; + const body = NULL_BODY_STATUSES.has(status) ? null : (Readable.toWeb(response) as unknown as ReadableStream); + if (!body) response.resume(); + resolveResponse(new Response(body, { status, headers: responseHeaders(response.headers) })); + } catch (error) { + response.destroy(); + rejectResponse(error); + } }, ); request.on('error', rejectResponse); From a3df866c7f9588a544f1a707a4ba4c9d9457e7be Mon Sep 17 00:00:00 2001 From: jamby77 Date: Tue, 13 Oct 2026 13:20:02 +0300 Subject: [PATCH 05/31] Guard deferred KV cache delete against re-registration - Skip the parked deleteKvCacheConnectionData when the removal generation has moved or the id is back in the registry, so a re-registered connection keeps its fresh data - Drop the removal generation once the delete fires with no run in flight, bounding the generations map - Drop the oldest timestamp when samples hit the row limit so the oldest returned bucket is never partially counted --- .../kv-cache-footprint.service.spec.ts | 51 +++++++++++++++++-- .../kv-cache-samples.service.spec.ts | 12 +++++ .../kv-cache/kv-cache-footprint.service.ts | 15 +++++- .../kv-cache/kv-cache-samples.service.ts | 7 ++- 4 files changed, 79 insertions(+), 6 deletions(-) diff --git a/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts b/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts index 673e55ac4..b43d9aacb 100644 --- a/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts +++ b/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts @@ -89,14 +89,16 @@ describe('KvCacheFootprintService', () => { }); it('deletes stored data when a connection is removed', async () => { - const { service, storage } = setup(); + const { service, storage, registry } = setup(); + registry.list.mockReturnValue([]); (service as any).onConnectionRemoved('c1'); await new Promise((resolve) => setImmediate(resolve)); expect(storage.deleteKvCacheConnectionData).toHaveBeenCalledWith('c1'); }); it('deletes stored data only after async removal listeners settle', async () => { - const { service, storage } = setup(); + const { service, storage, registry } = setup(); + registry.list.mockReturnValue([]); let release: () => void = () => undefined; service.onConnectionRemoval(() => new Promise((resolve) => (release = resolve))); (service as any).onConnectionRemoved('c1'); @@ -108,13 +110,14 @@ describe('KvCacheFootprintService', () => { }); it('discards an in-flight collection and deletes data after it settles when the connection is removed', async () => { - const { service, storage, client } = setup(); + const { service, storage, client, registry } = setup(); let finishRead: (size: number) => void = () => undefined; client.getDbSize.mockImplementationOnce(() => new Promise((resolve) => (finishRead = resolve))); const listener = jest.fn(); service.onSnapshot(listener); const collecting = service.triggerCollection('c1'); await new Promise((resolve) => setImmediate(resolve)); + registry.list.mockReturnValue([]); (service as any).onConnectionRemoved('c1'); await new Promise((resolve) => setImmediate(resolve)); expect(storage.deleteKvCacheConnectionData).not.toHaveBeenCalled(); @@ -126,6 +129,48 @@ describe('KvCacheFootprintService', () => { expect(storage.deleteKvCacheConnectionData).toHaveBeenCalledWith('c1'); }); + it('keeps fresh data when the connection is re-registered before an in-flight run settles', async () => { + const { service, storage, client, registry } = setup(); + const listed = registry.list(); + let finishRead: (size: number) => void = () => undefined; + client.getDbSize.mockImplementationOnce(() => new Promise((resolve) => (finishRead = resolve))); + const stale = service.triggerCollection('c1'); + await new Promise((resolve) => setImmediate(resolve)); + registry.list.mockReturnValue([]); + (service as any).onConnectionRemoved('c1'); + registry.list.mockReturnValue(listed); + const fresh = await service.triggerCollection('c1'); + expect(fresh).not.toBeNull(); + expect(storage.saveKvCacheFootprintSnapshot).toHaveBeenCalledTimes(1); + finishRead(6); + await expect(stale).resolves.toBeNull(); + await new Promise((resolve) => setImmediate(resolve)); + expect(storage.saveKvCacheFootprintSnapshot).toHaveBeenCalledTimes(1); + expect(storage.deleteKvCacheConnectionData).not.toHaveBeenCalled(); + expect(await service.triggerCollection('c1')).not.toBeNull(); + }); + + it('deletes once for the latest removal when a connection is removed again before data is deleted', async () => { + const { service, storage, registry } = setup(); + registry.list.mockReturnValue([]); + const releases: Array<() => void> = []; + service.onConnectionRemoval(() => new Promise((resolve) => releases.push(resolve))); + (service as any).onConnectionRemoved('c1'); + (service as any).onConnectionRemoved('c1'); + releases.forEach((release) => release()); + await new Promise((resolve) => setImmediate(resolve)); + expect(storage.deleteKvCacheConnectionData).toHaveBeenCalledTimes(1); + }); + + it('forgets the removal generation once the data is deleted', async () => { + const { service, storage, registry } = setup(); + registry.list.mockReturnValue([]); + (service as any).onConnectionRemoved('c1'); + await new Promise((resolve) => setImmediate(resolve)); + expect(storage.deleteKvCacheConnectionData).toHaveBeenCalledWith('c1'); + expect((service as any).generations.has('c1')).toBe(false); + }); + it('notifies removal listeners with the connection id', () => { const { service } = setup(); const listener = jest.fn(); diff --git a/proprietary/kv-cache/__tests__/kv-cache-samples.service.spec.ts b/proprietary/kv-cache/__tests__/kv-cache-samples.service.spec.ts index e815e5154..c25903ae8 100644 --- a/proprietary/kv-cache/__tests__/kv-cache-samples.service.spec.ts +++ b/proprietary/kv-cache/__tests__/kv-cache-samples.service.spec.ts @@ -171,6 +171,18 @@ describe('KvCacheSamplesService', () => { expect(result.buckets[0].timestamp).toBe(1); }); + it('drops the oldest bucket when the row limit splits it', async () => { + const { service, storage } = setup(); + const boundary = [sample({ timestamp: 0, requestedTokens: 1, hitTokens: 1 }), sample({ timestamp: 0, engineId: 'e2', requestedTokens: 1 })]; + const newer = Array.from({ length: SAMPLES_LIMIT - 1 }, (_, i) => sample({ timestamp: Math.floor(i / 2) + 1, engineId: i % 2 ? 'e2' : 'e1', requestedTokens: 2, hitTokens: 1 })); + storage.getKvCacheEngineSamples.mockResolvedValue([...boundary, ...newer]); + const result = await service.getSamples('c1', { from: 0, to: 1_000_000 }); + expect(result.truncated).toBe(true); + expect(result.buckets.some((bucket) => bucket.timestamp === 0)).toBe(false); + expect(result.buckets[0]).toMatchObject({ timestamp: 1, requestedTokens: 2 }); + expect(result.rangeHitRate).toBe(0.5); + }); + it('returns a null range hit rate when nothing was requested', async () => { const { service } = setup(); expect(await service.getSamples('c1', { from: 0, to: 1000 })).toEqual({ buckets: [], rangeHitRate: null, truncated: false }); diff --git a/proprietary/kv-cache/kv-cache-footprint.service.ts b/proprietary/kv-cache/kv-cache-footprint.service.ts index 23fff7746..d192c44ce 100644 --- a/proprietary/kv-cache/kv-cache-footprint.service.ts +++ b/proprietary/kv-cache/kv-cache-footprint.service.ts @@ -73,7 +73,8 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On protected onConnectionRemoved(connectionId: string): void { const active = this.running.get(connectionId); this.running.delete(connectionId); - this.generations.set(connectionId, this.generationOf(connectionId) + 1); + const generation = this.generationOf(connectionId) + 1; + this.generations.set(connectionId, generation); this.lastEvicted.delete(connectionId); this.sampleKeys.delete(connectionId); const settled = active?.then( @@ -88,10 +89,20 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On } }); void Promise.all([...pending, settled]) - .then(() => this.storage.deleteKvCacheConnectionData(connectionId)) + .then(() => this.deleteRemovedData(connectionId, generation)) .catch((error) => this.logger.warn(`Could not delete KV cache data for ${connectionId}: ${errorMessage(error)}`)); } + private async deleteRemovedData(connectionId: string, generation: number): Promise { + if (this.generationOf(connectionId) !== generation || this.isRegistered(connectionId)) return; + if (!this.running.has(connectionId)) this.generations.delete(connectionId); + await this.storage.deleteKvCacheConnectionData(connectionId); + } + + private isRegistered(connectionId: string): boolean { + return this.connectionRegistry.list().some((connection) => connection.id === connectionId); + } + onConnectionRemoval(listener: RemovalListener): void { this.removalListeners.push(listener); } diff --git a/proprietary/kv-cache/kv-cache-samples.service.ts b/proprietary/kv-cache/kv-cache-samples.service.ts index 038a270c9..ea3df4db4 100644 --- a/proprietary/kv-cache/kv-cache-samples.service.ts +++ b/proprietary/kv-cache/kv-cache-samples.service.ts @@ -35,6 +35,11 @@ export function mergeBuckets(samples: KvCacheEngineSample[]): KvCacheSampleBucke .map((row) => ({ ...row, hitRate: hitRate(row.requestedTokens, row.hitTokens) })); } +function dropOldestTimestamp(samples: KvCacheEngineSample[]): KvCacheEngineSample[] { + const oldest = samples.reduce((min, row) => Math.min(min, row.timestamp), Infinity); + return samples.filter((row) => row.timestamp > oldest); +} + @Injectable() export class KvCacheSamplesService implements OnModuleInit, OnModuleDestroy { private readonly logger = new Logger(KvCacheSamplesService.name); @@ -107,7 +112,7 @@ export class KvCacheSamplesService implements OnModuleInit, OnModuleDestroy { modelName: query.model, }); const truncated = fetched.length > SAMPLES_LIMIT; - const rows = truncated ? fetched.slice(fetched.length - SAMPLES_LIMIT) : fetched; + const rows = truncated ? dropOldestTimestamp(fetched) : fetched; const requested = rows.reduce((sum, row) => sum + row.requestedTokens, 0); const hit = rows.reduce((sum, row) => sum + row.hitTokens, 0); return { buckets: mergeBuckets(rows), rangeHitRate: hitRate(requested, hit), truncated }; From b548dd959b21ebafc7c12a9358960b5864a1a0bc Mon Sep 17 00:00:00 2001 From: jamby77 Date: Wed, 14 Oct 2026 10:26:25 +0300 Subject: [PATCH 06/31] Fix KV cache removal delete race and empty truncation - Hold collections for a connection until an in-flight removal delete completes, both before starting and before saving a snapshot - Keep the newest SAMPLES_LIMIT rows when every fetched sample shares the oldest timestamp instead of returning no buckets --- .../kv-cache-footprint.service.spec.ts | 51 +++++++++++++++++++ .../kv-cache-samples.service.spec.ts | 11 ++++ .../kv-cache/kv-cache-footprint.service.ts | 15 +++++- .../kv-cache/kv-cache-samples.service.ts | 3 +- 4 files changed, 78 insertions(+), 2 deletions(-) diff --git a/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts b/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts index 5b956e88f..cc5e41a68 100644 --- a/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts +++ b/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts @@ -191,6 +191,57 @@ describe('KvCacheFootprintService', () => { expect(await service.triggerCollection('c1')).not.toBeNull(); }); + it('waits for an in-flight removal delete before saving for a re-registered connection', async () => { + const { service, storage, registry } = setup(); + const listed = registry.list(); + const order: string[] = []; + let finishDelete: () => void = () => undefined; + storage.deleteKvCacheConnectionData.mockImplementationOnce( + () => new Promise((resolve) => (finishDelete = () => { order.push('delete'); resolve(); })), + ); + storage.saveKvCacheFootprintSnapshot.mockImplementation(async () => { order.push('save'); }); + registry.list.mockReturnValue([]); + (service as any).onConnectionRemoved('c1'); + await new Promise((resolve) => setImmediate(resolve)); + expect(storage.deleteKvCacheConnectionData).toHaveBeenCalledWith('c1'); + registry.list.mockReturnValue(listed); + const collecting = service.triggerCollection('c1'); + await new Promise((resolve) => setImmediate(resolve)); + expect(storage.saveKvCacheFootprintSnapshot).not.toHaveBeenCalled(); + finishDelete(); + await expect(collecting).resolves.not.toBeNull(); + expect(order).toEqual(['delete', 'save']); + }); + + it('holds a save that races an in-flight removal delete until the delete completes', async () => { + const { service, storage, client, registry } = setup(); + const listed = registry.list(); + const order: string[] = []; + let release: () => void = () => undefined; + let finishRead: (size: number) => void = () => undefined; + let finishDelete: () => void = () => undefined; + service.onConnectionRemoval(() => new Promise((resolve) => (release = resolve))); + client.getDbSize.mockImplementationOnce(() => new Promise((resolve) => (finishRead = resolve))); + storage.deleteKvCacheConnectionData.mockImplementationOnce( + () => new Promise((resolve) => (finishDelete = () => { order.push('delete'); resolve(); })), + ); + storage.saveKvCacheFootprintSnapshot.mockImplementation(async () => { order.push('save'); }); + registry.list.mockReturnValue([]); + (service as any).onConnectionRemoved('c1'); + const collecting = service.collect({ connectionId: 'c1', connectionName: 'one', client, host: 'h', port: 6379 }); + await new Promise((resolve) => setImmediate(resolve)); + release(); + await new Promise((resolve) => setImmediate(resolve)); + expect(storage.deleteKvCacheConnectionData).toHaveBeenCalledWith('c1'); + registry.list.mockReturnValue(listed); + finishRead(6); + await new Promise((resolve) => setImmediate(resolve)); + expect(storage.saveKvCacheFootprintSnapshot).not.toHaveBeenCalled(); + finishDelete(); + await expect(collecting).resolves.not.toBeNull(); + expect(order).toEqual(['delete', 'save']); + }); + it('deletes once for the latest removal when a connection is removed again before data is deleted', async () => { const { service, storage, registry } = setup(); registry.list.mockReturnValue([]); diff --git a/proprietary/kv-cache/__tests__/kv-cache-samples.service.spec.ts b/proprietary/kv-cache/__tests__/kv-cache-samples.service.spec.ts index 6b3b8af43..9fa6cd7fd 100644 --- a/proprietary/kv-cache/__tests__/kv-cache-samples.service.spec.ts +++ b/proprietary/kv-cache/__tests__/kv-cache-samples.service.spec.ts @@ -188,6 +188,17 @@ describe('KvCacheSamplesService', () => { expect(result.rangeHitRate).toBe(0.5); }); + it('keeps the newest rows when every fetched row shares the oldest timestamp', async () => { + const { service, storage } = setup(); + const rows = Array.from({ length: SAMPLES_LIMIT + 1 }, (_, i) => sample({ timestamp: 5, engineId: `e${i}`, requestedTokens: 2, hitTokens: 1 })); + storage.getKvCacheEngineSamples.mockResolvedValue(rows); + const result = await service.getSamples('c1', { from: 0, to: 1000 }); + expect(result.truncated).toBe(true); + expect(result.buckets).toHaveLength(SAMPLES_LIMIT); + expect(result.buckets.some((bucket) => bucket.engineId === 'e0')).toBe(false); + expect(result.rangeHitRate).toBe(0.5); + }); + it('returns a null range hit rate when nothing was requested', async () => { const { service } = setup(); expect(await service.getSamples('c1', { from: 0, to: 1000 })).toEqual({ buckets: [], rangeHitRate: null, truncated: false }); diff --git a/proprietary/kv-cache/kv-cache-footprint.service.ts b/proprietary/kv-cache/kv-cache-footprint.service.ts index 502bdec75..2e10e4007 100644 --- a/proprietary/kv-cache/kv-cache-footprint.service.ts +++ b/proprietary/kv-cache/kv-cache-footprint.service.ts @@ -49,6 +49,7 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On ); private readonly running = new Map>(); private readonly generations = new Map(); + private readonly deleting = new Map>(); private readonly lastEvicted = new Map>(); private readonly sampleKeys = new Map(); private readonly undetectedUntil = new Map(); @@ -104,7 +105,17 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On private async deleteRemovedData(connectionId: string, generation: number): Promise { if (this.generationOf(connectionId) !== generation || this.isRegistered(connectionId)) return; if (!this.running.has(connectionId)) this.generations.delete(connectionId); - await this.storage.deleteKvCacheConnectionData(connectionId); + const deletion = this.storage.deleteKvCacheConnectionData(connectionId); + const settled = deletion.then( + () => undefined, + () => undefined, + ); + this.deleting.set(connectionId, settled); + try { + await deletion; + } finally { + if (this.deleting.get(connectionId) === settled) this.deleting.delete(connectionId); + } } private isRegistered(connectionId: string): boolean { @@ -137,6 +148,7 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On } async collect(ctx: ConnectionContext): Promise { + await this.deleting.get(ctx.connectionId); const plan = this.planFor(ctx.connectionId); if (plan.kind === 'skip' || this.running.has(ctx.connectionId)) return null; const run = this.runCollection(ctx, plan); @@ -169,6 +181,7 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On otherDbs: readings[0]?.otherDbs ?? [], clientDetected: readings.some((reading) => reading.glide), }); + await this.deleting.get(ctx.connectionId); if (!current()) return null; await this.storage.saveKvCacheFootprintSnapshot(snapshot); if (!current()) return null; diff --git a/proprietary/kv-cache/kv-cache-samples.service.ts b/proprietary/kv-cache/kv-cache-samples.service.ts index c21fc80ed..57ad3b19f 100644 --- a/proprietary/kv-cache/kv-cache-samples.service.ts +++ b/proprietary/kv-cache/kv-cache-samples.service.ts @@ -38,7 +38,8 @@ export function mergeBuckets(samples: KvCacheEngineSample[]): KvCacheSampleBucke function dropOldestTimestamp(samples: KvCacheEngineSample[]): KvCacheEngineSample[] { const oldest = samples.reduce((min, row) => Math.min(min, row.timestamp), Infinity); - return samples.filter((row) => row.timestamp > oldest); + const newer = samples.filter((row) => row.timestamp > oldest); + return newer.length > 0 ? newer : samples.slice(-SAMPLES_LIMIT); } @Injectable() From 6a869846a1281fa02bb6df87f9a3808695778431 Mon Sep 17 00:00:00 2001 From: jamby77 Date: Wed, 14 Oct 2026 11:11:26 +0300 Subject: [PATCH 07/31] Restore KV engines when a connection re-registers When a removed connection is registered again before cleanup runs, announce a restore instead of skipping silently, and reload that connection's engines from storage so scrapes resume. --- .../kv-cache-engine-registry.spec.ts | 33 +++++++++++++++++-- .../kv-cache-footprint.service.spec.ts | 21 ++++++++++++ .../kv-cache/kv-cache-engine-registry.ts | 13 ++++++++ .../kv-cache/kv-cache-footprint.service.ts | 18 +++++++++- 4 files changed, 82 insertions(+), 3 deletions(-) diff --git a/proprietary/kv-cache/__tests__/kv-cache-engine-registry.spec.ts b/proprietary/kv-cache/__tests__/kv-cache-engine-registry.spec.ts index 9655b6a73..42cdf5487 100644 --- a/proprietary/kv-cache/__tests__/kv-cache-engine-registry.spec.ts +++ b/proprietary/kv-cache/__tests__/kv-cache-engine-registry.spec.ts @@ -12,10 +12,14 @@ function setup(initial = [engine('a', 'c1', 'lmc-a'), engine('b', 'c2')]) { deleteKvCacheEngine: jest.fn().mockResolvedValue(true), } as any; let removal: (id: string) => unknown = () => undefined; - const footprint = { onConnectionRemoval: jest.fn((l) => (removal = l)) } as any; + let restore: (id: string) => unknown = () => undefined; + const footprint = { + onConnectionRemoval: jest.fn((l) => (removal = l)), + onConnectionRestore: jest.fn((l) => (restore = l)), + } as any; const samples = { forgetEngine: jest.fn(), settled: jest.fn().mockResolvedValue(undefined) } as any; const registry = new KvCacheEngineRegistry(storage, footprint, samples); - return { registry, storage, samples, removal: (id: string) => removal(id) }; + return { registry, storage, samples, removal: (id: string) => removal(id), restore: (id: string) => restore(id) }; } describe('KvCacheEngineRegistry', () => { @@ -140,4 +144,29 @@ describe('KvCacheEngineRegistry', () => { await removing; expect(samples.settled).toHaveBeenCalled(); }); + + it('reloads the engines of a connection that was registered again before cleanup', async () => { + const { registry, storage, removal, restore } = setup(); + await registry.onModuleInit(); + await removal('c1'); + expect(registry.get('a')).toBeNull(); + await restore('c1'); + expect(registry.list('c1').map((e) => e.id)).toEqual(['a']); + registry.recordResult('a', { lastSeenAt: 9, lastError: null }); + await new Promise((resolve) => setImmediate(resolve)); + expect(storage.saveKvCacheEngine).toHaveBeenCalledWith(expect.objectContaining({ id: 'a', lastSeenAt: 9 })); + }); + + it('does not restore engines when the connection is removed again during the reload', async () => { + const { registry, storage, removal, restore } = setup(); + await registry.onModuleInit(); + await removal('c1'); + let finishLoad: (engines: unknown) => void = () => undefined; + storage.getKvCacheEngines.mockImplementationOnce(() => new Promise((resolve) => (finishLoad = resolve))); + const restoring = restore('c1'); + await removal('c1'); + finishLoad([engine('a', 'c1', 'lmc-a')]); + await restoring; + expect(registry.get('a')).toBeNull(); + }); }); diff --git a/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts b/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts index cc5e41a68..badd922db 100644 --- a/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts +++ b/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts @@ -191,6 +191,27 @@ describe('KvCacheFootprintService', () => { expect(await service.triggerCollection('c1')).not.toBeNull(); }); + it('announces a restore instead of deleting when the connection is registered again', async () => { + const { service, storage } = setup(); + const restored = jest.fn(); + service.onConnectionRestore(restored); + (service as any).onConnectionRemoved('c1'); + await new Promise((resolve) => setImmediate(resolve)); + expect(storage.deleteKvCacheConnectionData).not.toHaveBeenCalled(); + expect(restored).toHaveBeenCalledWith('c1'); + }); + + it('does not announce a restore when the connection stays removed', async () => { + const { service, storage, registry } = setup(); + const restored = jest.fn(); + service.onConnectionRestore(restored); + registry.list.mockReturnValue([]); + (service as any).onConnectionRemoved('c1'); + await new Promise((resolve) => setImmediate(resolve)); + expect(storage.deleteKvCacheConnectionData).toHaveBeenCalledWith('c1'); + expect(restored).not.toHaveBeenCalled(); + }); + it('waits for an in-flight removal delete before saving for a re-registered connection', async () => { const { service, storage, registry } = setup(); const listed = registry.list(); diff --git a/proprietary/kv-cache/kv-cache-engine-registry.ts b/proprietary/kv-cache/kv-cache-engine-registry.ts index dfc9999ba..d8e6343f8 100644 --- a/proprietary/kv-cache/kv-cache-engine-registry.ts +++ b/proprietary/kv-cache/kv-cache-engine-registry.ts @@ -10,6 +10,7 @@ export class KvCacheEngineRegistry implements OnModuleInit { private readonly writes = new Map>(); private readonly statusVersions = new Map(); private readonly removed = new Set(); + private readonly removals = new Map(); constructor( @Inject('STORAGE_CLIENT') private readonly storage: StoragePort, @@ -20,6 +21,7 @@ export class KvCacheEngineRegistry implements OnModuleInit { async onModuleInit(): Promise { for (const engine of await this.storage.getKvCacheEngines()) this.engines.set(engine.id, engine); this.footprint.onConnectionRemoval(async (connectionId) => { + this.removals.set(connectionId, (this.removals.get(connectionId) ?? 0) + 1); const ids = this.list(connectionId).map((engine) => engine.id); for (const id of ids) { this.removed.add(id); @@ -30,6 +32,17 @@ export class KvCacheEngineRegistry implements OnModuleInit { await Promise.all(ids.map((id) => this.writes.get(id))); return this.samples.settled(); }); + this.footprint.onConnectionRestore((connectionId) => this.restore(connectionId)); + } + + private async restore(connectionId: string): Promise { + const removals = this.removals.get(connectionId); + const stored = (await this.storage.getKvCacheEngines()).filter((engine) => engine.connectionId === connectionId); + if (this.removals.get(connectionId) !== removals) return; + for (const engine of stored) { + this.removed.delete(engine.id); + if (!this.engines.has(engine.id)) this.engines.set(engine.id, engine); + } } list(connectionId?: string): StoredKvCacheEngine[] { diff --git a/proprietary/kv-cache/kv-cache-footprint.service.ts b/proprietary/kv-cache/kv-cache-footprint.service.ts index 2e10e4007..e59f9145c 100644 --- a/proprietary/kv-cache/kv-cache-footprint.service.ts +++ b/proprietary/kv-cache/kv-cache-footprint.service.ts @@ -55,6 +55,7 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On private readonly undetectedUntil = new Map(); private readonly listeners: SnapshotListener[] = []; private readonly removalListeners: RemovalListener[] = []; + private readonly restoreListeners: RemovalListener[] = []; constructor( connectionRegistry: ConnectionRegistry, @@ -103,7 +104,8 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On } private async deleteRemovedData(connectionId: string, generation: number): Promise { - if (this.generationOf(connectionId) !== generation || this.isRegistered(connectionId)) return; + if (this.generationOf(connectionId) !== generation) return; + if (this.isRegistered(connectionId)) return this.announceRestore(connectionId); if (!this.running.has(connectionId)) this.generations.delete(connectionId); const deletion = this.storage.deleteKvCacheConnectionData(connectionId); const settled = deletion.then( @@ -118,6 +120,16 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On } } + private async announceRestore(connectionId: string): Promise { + for (const listener of this.restoreListeners) { + try { + await listener(connectionId); + } catch (error) { + this.logger.warn(`KV cache restore listener failed: ${errorMessage(error)}`); + } + } + } + private isRegistered(connectionId: string): boolean { return this.connectionRegistry.list().some((connection) => connection.id === connectionId); } @@ -126,6 +138,10 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On this.removalListeners.push(listener); } + onConnectionRestore(listener: RemovalListener): void { + this.restoreListeners.push(listener); + } + onSnapshot(listener: SnapshotListener): void { this.listeners.push(listener); } From 268678e9ab07dd6bb6ea0e6045f3f39490635918 Mon Sep 17 00:00:00 2001 From: jamby77 Date: Wed, 14 Oct 2026 11:40:29 +0300 Subject: [PATCH 08/31] Skip restored KV engines that clash with new links On restore, keep an engine linked during cleanup and delete the stored engine it supersedes by name or OTLP id instead of loading both. --- .../__tests__/kv-cache-engine-registry.spec.ts | 12 ++++++++++++ .../kv-cache/kv-cache-engine-registry.ts | 17 ++++++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/proprietary/kv-cache/__tests__/kv-cache-engine-registry.spec.ts b/proprietary/kv-cache/__tests__/kv-cache-engine-registry.spec.ts index 42cdf5487..0b3c914a4 100644 --- a/proprietary/kv-cache/__tests__/kv-cache-engine-registry.spec.ts +++ b/proprietary/kv-cache/__tests__/kv-cache-engine-registry.spec.ts @@ -169,4 +169,16 @@ describe('KvCacheEngineRegistry', () => { await restoring; expect(registry.get('a')).toBeNull(); }); + + it('keeps a link created during cleanup and deletes the stored engine it supersedes', async () => { + const { registry, storage, removal, restore } = setup([engine('a', 'c1', 'lmc-a'), engine('old', 'c1')]); + await registry.onModuleInit(); + await removal('c1'); + await registry.save({ ...engine('new', 'c1', 'lmc-a'), name: 'fresh' }); + await registry.save({ ...engine('dup', 'c1'), name: 'old' }); + await restore('c1'); + expect(registry.list('c1').map((e) => e.id).sort()).toEqual(['dup', 'new']); + expect(storage.deleteKvCacheEngine).toHaveBeenCalledWith('a'); + expect(storage.deleteKvCacheEngine).toHaveBeenCalledWith('old'); + }); }); diff --git a/proprietary/kv-cache/kv-cache-engine-registry.ts b/proprietary/kv-cache/kv-cache-engine-registry.ts index d8e6343f8..e5201b5f1 100644 --- a/proprietary/kv-cache/kv-cache-engine-registry.ts +++ b/proprietary/kv-cache/kv-cache-engine-registry.ts @@ -40,11 +40,26 @@ export class KvCacheEngineRegistry implements OnModuleInit { const stored = (await this.storage.getKvCacheEngines()).filter((engine) => engine.connectionId === connectionId); if (this.removals.get(connectionId) !== removals) return; for (const engine of stored) { + if (this.engines.has(engine.id)) continue; + if (this.clashes(engine)) { + await this.serialize(engine.id, () => this.storage.deleteKvCacheEngine(engine.id)).catch((error) => + this.logger.warn(`Could not delete superseded KV cache engine ${engine.id}: ${error instanceof Error ? error.message : String(error)}`), + ); + continue; + } this.removed.delete(engine.id); - if (!this.engines.has(engine.id)) this.engines.set(engine.id, engine); + this.engines.set(engine.id, engine); } } + private clashes(engine: StoredKvCacheEngine): boolean { + return [...this.engines.values()].some( + (other) => + (other.connectionId === engine.connectionId && other.name === engine.name) || + (engine.otlpEngineId !== null && other.otlpEngineId === engine.otlpEngineId), + ); + } + list(connectionId?: string): StoredKvCacheEngine[] { const all = [...this.engines.values()]; return connectionId === undefined ? all : all.filter((engine) => engine.connectionId === connectionId); From 287bc635bd520b24a7b26b2faf6dfc0895452cbe Mon Sep 17 00:00:00 2001 From: jamby77 Date: Wed, 14 Oct 2026 12:10:30 +0300 Subject: [PATCH 09/31] Apply KV engine restores before deleting superseded rows - Decide which engines to reload synchronously, so a removal that lands during a delete cannot be undone by the rest of the restore - Skip a superseded delete once the connection is removed again or the engine is back in memory --- .../kv-cache-engine-registry.spec.ts | 32 +++++++++++++++++++ .../kv-cache/kv-cache-engine-registry.ts | 15 +++++++-- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/proprietary/kv-cache/__tests__/kv-cache-engine-registry.spec.ts b/proprietary/kv-cache/__tests__/kv-cache-engine-registry.spec.ts index 0b3c914a4..3fa21a0f5 100644 --- a/proprietary/kv-cache/__tests__/kv-cache-engine-registry.spec.ts +++ b/proprietary/kv-cache/__tests__/kv-cache-engine-registry.spec.ts @@ -181,4 +181,36 @@ describe('KvCacheEngineRegistry', () => { expect(storage.deleteKvCacheEngine).toHaveBeenCalledWith('a'); expect(storage.deleteKvCacheEngine).toHaveBeenCalledWith('old'); }); + + it('does not reload engines after a removal that lands while a superseded engine is deleted', async () => { + const { registry, storage, removal, restore } = setup([engine('a', 'c1', 'lmc-a'), engine('b', 'c1')]); + await registry.onModuleInit(); + await removal('c1'); + await registry.save(engine('new', 'c1', 'lmc-a')); + let finishDelete: () => void = () => undefined; + storage.deleteKvCacheEngine.mockImplementationOnce(() => new Promise((resolve) => (finishDelete = () => resolve(true)))); + const restoring = restore('c1'); + await new Promise((resolve) => setImmediate(resolve)); + await removal('c1'); + finishDelete(); + await restoring; + expect(registry.list('c1')).toEqual([]); + }); + + it('skips deleting a superseded engine once the connection is removed again', async () => { + const { registry, storage, removal, restore } = setup([engine('a', 'c1', 'lmc-a')]); + await registry.onModuleInit(); + await removal('c1'); + await registry.save(engine('new', 'c1', 'lmc-a')); + let finishPending: () => void = () => undefined; + storage.deleteKvCacheEngine.mockImplementationOnce(() => new Promise((resolve) => (finishPending = () => resolve(true)))); + const pending = registry.remove('a'); + const restoring = restore('c1'); + await new Promise((resolve) => setImmediate(resolve)); + await removal('c1'); + finishPending(); + await pending; + await restoring; + expect(storage.deleteKvCacheEngine).toHaveBeenCalledTimes(1); + }); }); diff --git a/proprietary/kv-cache/kv-cache-engine-registry.ts b/proprietary/kv-cache/kv-cache-engine-registry.ts index e5201b5f1..963d2ca98 100644 --- a/proprietary/kv-cache/kv-cache-engine-registry.ts +++ b/proprietary/kv-cache/kv-cache-engine-registry.ts @@ -39,17 +39,26 @@ export class KvCacheEngineRegistry implements OnModuleInit { const removals = this.removals.get(connectionId); const stored = (await this.storage.getKvCacheEngines()).filter((engine) => engine.connectionId === connectionId); if (this.removals.get(connectionId) !== removals) return; + const superseded: string[] = []; for (const engine of stored) { if (this.engines.has(engine.id)) continue; if (this.clashes(engine)) { - await this.serialize(engine.id, () => this.storage.deleteKvCacheEngine(engine.id)).catch((error) => - this.logger.warn(`Could not delete superseded KV cache engine ${engine.id}: ${error instanceof Error ? error.message : String(error)}`), - ); + superseded.push(engine.id); continue; } this.removed.delete(engine.id); this.engines.set(engine.id, engine); } + await Promise.all( + superseded.map((id) => + this.serialize(id, async () => { + if (this.removals.get(connectionId) !== removals || this.engines.has(id)) return; + await this.storage.deleteKvCacheEngine(id); + }).catch((error) => + this.logger.warn(`Could not delete superseded KV cache engine ${id}: ${error instanceof Error ? error.message : String(error)}`), + ), + ), + ); } private clashes(engine: StoredKvCacheEngine): boolean { From 0898c44320cc40cf1fdffc8650523d595a631a98 Mon Sep 17 00:00:00 2001 From: jamby77 Date: Tue, 20 Oct 2026 11:14:14 +0300 Subject: [PATCH 10/31] Weight per-model KV shares by the keys each sample represents Samples from a node with few matched keys no longer skew the per-model split as much as samples from a node holding most of the cache. --- .../__tests__/footprint-estimate.spec.ts | 12 ++++++++++ proprietary/kv-cache/footprint-estimate.ts | 24 ++++++++++++------- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/proprietary/kv-cache/__tests__/footprint-estimate.spec.ts b/proprietary/kv-cache/__tests__/footprint-estimate.spec.ts index f0f49bbf8..74ae6fb65 100644 --- a/proprietary/kv-cache/__tests__/footprint-estimate.spec.ts +++ b/proprietary/kv-cache/__tests__/footprint-estimate.spec.ts @@ -169,4 +169,16 @@ describe('buildSnapshot', () => { const s = buildSnapshot({ ...base, nodes: [n1, n2] }); expect(s).toMatchObject({ scannedKeys: 10, matchedKeys: 3, chunksEst: 3, bytesEst: 50 }); }); + + it('weights each node sample by the keys it stands for when splitting per model', () => { + const a = Array.from({ length: 100 }, (_, i) => key(i)); + const b = key(0, '', 'other', 'float16'); + const n1: NodeObservation = { scannedKeys: 100, matchedKeys: a, scanComplete: true, dbSize: 100, samples: [sample(a[0], { memoryBytes: 100 })] }; + const n2: NodeObservation = { scannedKeys: 1, matchedKeys: [b], scanComplete: true, dbSize: 1, samples: [sample(b, { memoryBytes: 100 })] }; + const s = buildSnapshot({ ...base, nodes: [n1, n2] }); + expect(s.perModel).toEqual([ + { model: M, dtype: 'bfloat16', chunksEst: 100, bytesEst: 10_000 }, + { model: 'other', dtype: 'float16', chunksEst: 1, bytesEst: 100 }, + ]); + }); }); diff --git a/proprietary/kv-cache/footprint-estimate.ts b/proprietary/kv-cache/footprint-estimate.ts index d0d06704a..510546281 100644 --- a/proprietary/kv-cache/footprint-estimate.ts +++ b/proprietary/kv-cache/footprint-estimate.ts @@ -70,18 +70,23 @@ function isLive(sample: KeySample): boolean { function nodeTotals( node: NodeObservation, parsed: ParsedLmcacheKey[], -): { chunks: number; bytes: number } { +): { chunks: number; bytes: number; keys: number } { const chunkKeys = parsed.filter(isChunkKey).length; const scale = node.scanComplete || node.scannedKeys === 0 ? 1 : node.dbSize / node.scannedKeys; const live = node.samples.filter(isLive); const mean = live.length === 0 ? 0 : live.reduce((sum, s) => sum + (s.memoryBytes ?? 0), 0) / live.length; - return { chunks: chunkKeys * scale, bytes: parsed.length * scale * mean }; + return { chunks: chunkKeys * scale, bytes: parsed.length * scale * mean, keys: parsed.length * scale }; } -function splitPerModel(samples: KeySample[], chunks: number, bytes: number): KvCacheModelFootprint[] { +interface WeightedSample { + sample: KeySample; + weight: number; +} + +function splitPerModel(samples: WeightedSample[], chunks: number, bytes: number): KvCacheModelFootprint[] { const groups = new Map(); - for (const s of samples) { + for (const { sample: s, weight } of samples) { const id = `${s.parsed.model}|${s.parsed.dtype}`; const group = groups.get(id) ?? { model: s.parsed.model, @@ -89,8 +94,8 @@ function splitPerModel(samples: KeySample[], chunks: number, bytes: number): KvC chunkSamples: 0, bytes: 0, }; - if (isChunkKey(s.parsed)) group.chunkSamples += 1; - group.bytes += s.memoryBytes ?? 0; + if (isChunkKey(s.parsed)) group.chunkSamples += weight; + group.bytes += (s.memoryBytes ?? 0) * weight; groups.set(id, group); } const allChunkSamples = [...groups.values()].reduce((sum, g) => sum + g.chunkSamples, 0); @@ -112,6 +117,7 @@ export function buildSnapshot(input: SnapshotInput): KvCacheFootprintSnapshot { let matched = 0; const allParsed: ParsedLmcacheKey[] = []; const live: KeySample[] = []; + const weighted: WeightedSample[] = []; for (const node of input.nodes) { const parsed = node.matchedKeys .map(classifyKey) @@ -122,7 +128,9 @@ export function buildSnapshot(input: SnapshotInput): KvCacheFootprintSnapshot { scanned += node.scannedKeys; matched += parsed.length; allParsed.push(...parsed); - live.push(...node.samples.filter(isLive)); + const nodeLive = node.samples.filter(isLive); + live.push(...nodeLive); + weighted.push(...nodeLive.map((sample) => ({ sample, weight: totals.keys / nodeLive.length }))); } const probed = live.filter((s) => s.parsed.suffix === 'kv_bytes' && s.siblingExists !== null); const { usedMemory, maxmemory, maxmemoryPolicy } = input.memory; @@ -148,6 +156,6 @@ export function buildSnapshot(input: SnapshotInput): KvCacheFootprintSnapshot { : ratio(probed.filter((s) => s.siblingExists === false).length / probed.length), evictedKeysDelta: input.evictedKeysDelta, otherDbs: input.otherDbs, - perModel: splitPerModel(live, chunks, bytes), + perModel: splitPerModel(weighted, chunks, bytes), }; } From dbda1a1679be7c4a422021142eba6501e7e50a99 Mon Sep 17 00:00:00 2001 From: jamby77 Date: Tue, 20 Oct 2026 11:14:23 +0300 Subject: [PATCH 11/31] Harden KV footprint collection for agents and partial clusters - Sample through call() in batches so agent connections work - Count glob hits toward the match budget; stop probing undetected connections after two empty pages - Ask CLIENT LIST only when too few keys matched; match the GLIDE lib-name field exactly - Keep readable nodes when one fails and mark the scan incomplete - Warn once per node when MEMORY USAGE returns nothing - Treat retired members as registered so retiring never wipes data - Reject a manual refresh while one runs, and for sentinel or disconnected connections - Floor KV_CACHE_MATCH_MAX_KEYS at the detection threshold --- .../__tests__/footprint-scanner.spec.ts | 92 +++++++++++++++---- .../kv-cache-footprint.service.spec.ts | 80 ++++++++++++++-- proprietary/kv-cache/footprint-scanner.ts | 79 +++++++++------- .../kv-cache/kv-cache-footprint.service.ts | 80 +++++++++++++--- 4 files changed, 260 insertions(+), 71 deletions(-) diff --git a/proprietary/kv-cache/__tests__/footprint-scanner.spec.ts b/proprietary/kv-cache/__tests__/footprint-scanner.spec.ts index f1eceddfd..b9876c77c 100644 --- a/proprietary/kv-cache/__tests__/footprint-scanner.spec.ts +++ b/proprietary/kv-cache/__tests__/footprint-scanner.spec.ts @@ -35,19 +35,54 @@ describe('scanLmcacheKeys', () => { expect(client.call).toHaveBeenCalledTimes(2); expect(result.scanComplete).toBe(false); }); + + it('counts glob hits that fail the classifier toward the matched budget', async () => { + const lookalikes = (n: number) => Array.from({ length: n }, (_, i) => `user@tenant@region@shard@${i}`); + const client = scanClient([['7', lookalikes(3)], ['8', lookalikes(3)], ['9', lookalikes(3)]]); + const result = await scanLmcacheKeys(client, { maxScanned: 200_000, maxMatched: 5 }, 10_000); + expect(client.call).toHaveBeenCalledTimes(2); + expect(result).toEqual({ scannedKeys: 2000, matchedKeys: [], scanComplete: false }); + }); + + it('gives up after the probe pages when nothing matches the glob', async () => { + const pages: Array<[string, string[]]> = [['7', []], ['8', []], ['0', [k(0)]]]; + const probing = scanClient(pages); + await scanLmcacheKeys(probing, { maxScanned: 200_000, maxMatched: 2000 }, 10_000, true); + expect(probing.call).toHaveBeenCalledTimes(2); + + const full = scanClient(pages); + const result = await scanLmcacheKeys(full, { maxScanned: 200_000, maxMatched: 2000 }, 10_000); + expect(result.matchedKeys).toEqual([k(0)]); + }); + + it('keeps scanning past the probe pages once the glob has hits', async () => { + const client = scanClient([['7', ['a@b@c@d@e']], ['8', []], ['0', [k(0)]]]); + const result = await scanLmcacheKeys(client, { maxScanned: 200_000, maxMatched: 2000 }, 3000, true); + expect(result.matchedKeys).toEqual([k(0)]); + }); }); +function sampleClient(replies: Record) { + const call = jest.fn(async (command: string, args: string[]) => { + const reply = replies[`${command} ${args.join(' ')}`]; + if (reply instanceof Error) throw reply; + return reply; + }); + return { call, getClient: () => { throw new Error('Raw client access not available through agent connections'); } } as any; +} + describe('sampleLmcacheKeys', () => { - it('reads memory, ttl and sibling existence in one pipeline', async () => { - const pipeline = { call: jest.fn().mockReturnThis(), ttl: jest.fn().mockReturnThis(), exists: jest.fn().mockReturnThis(), exec: jest.fn() }; - pipeline.exec.mockResolvedValue([ - [null, 3670112], [null, -1], [null, 0], - [null, 120], [null, 50], - [null, null], [null, -2], - ]); - const client = { getClient: () => ({ pipeline: () => pipeline }) } as any; + it('reads memory, ttl and sibling existence through call only', async () => { + const client = sampleClient({ + [`MEMORY USAGE ${k(0, 'kv_bytes')}`]: 3670112, + [`TTL ${k(0, 'kv_bytes')}`]: -1, + [`EXISTS ${k(0, 'metadata')}`]: 0, + [`MEMORY USAGE ${k(1)}`]: 120, + [`TTL ${k(1)}`]: 50, + [`MEMORY USAGE ${k(2)}`]: null, + [`TTL ${k(2)}`]: -2, + }); const samples = await sampleLmcacheKeys(client, [k(0, 'kv_bytes'), k(1), k(2)]); - expect(pipeline.exists).toHaveBeenCalledWith(k(0, 'metadata')); expect(samples.map(({ memoryBytes, ttl, siblingExists }) => ({ memoryBytes, ttl, siblingExists }))).toEqual([ { memoryBytes: 3670112, ttl: -1, siblingExists: false }, { memoryBytes: 120, ttl: 50, siblingExists: null }, @@ -56,19 +91,31 @@ describe('sampleLmcacheKeys', () => { }); it('treats a cross-slot sibling probe rejected with MOVED as unknown, not orphaned', async () => { - const pipeline = { call: jest.fn().mockReturnThis(), ttl: jest.fn().mockReturnThis(), exists: jest.fn().mockReturnThis(), exec: jest.fn() }; - pipeline.exec.mockResolvedValue([ - [null, 3670112], [null, -1], [new Error('MOVED 1234 10.0.0.2:6379'), null], - [null, 3670112], [null, -1], [null, 1], - [null, 3670112], [null, -1], [null, 0], - ]); - const client = { getClient: () => ({ pipeline: () => pipeline }) } as any; - const samples = await sampleLmcacheKeys(client, [k(0, 'kv_bytes'), k(1, 'kv_bytes'), k(2, 'kv_bytes')]); - expect(samples.map((s) => s.siblingExists)).toEqual([null, true, false]); + const client = sampleClient({ + [`MEMORY USAGE ${k(0, 'kv_bytes')}`]: 3670112, + [`TTL ${k(0, 'kv_bytes')}`]: -1, + [`EXISTS ${k(0, 'metadata')}`]: new Error('MOVED 1234 10.0.0.2:6379'), + [`MEMORY USAGE ${k(1, 'kv_bytes')}`]: 3670112, + [`TTL ${k(1, 'kv_bytes')}`]: -1, + [`EXISTS ${k(1, 'metadata')}`]: 1, + }); + const samples = await sampleLmcacheKeys(client, [k(0, 'kv_bytes'), k(1, 'kv_bytes')]); + expect(samples.map((s) => s.siblingExists)).toEqual([null, true]); }); - it('skips the pipeline for no keys', async () => { - expect(await sampleLmcacheKeys({ getClient: () => { throw new Error('unused'); } } as any, [])).toEqual([]); + it('reports a key as unsized when MEMORY is unavailable', async () => { + const client = sampleClient({ + [`MEMORY USAGE ${k(0)}`]: new Error("ERR unknown command 'MEMORY'"), + [`TTL ${k(0)}`]: -1, + }); + const [sample] = await sampleLmcacheKeys(client, [k(0)]); + expect(sample.memoryBytes).toBeNull(); + }); + + it('issues no commands for no keys', async () => { + const client = sampleClient({}); + expect(await sampleLmcacheKeys(client, [])).toEqual([]); + expect(client.call).not.toHaveBeenCalled(); }); }); @@ -79,6 +126,11 @@ describe('helpers', () => { expect(await hasLmcacheGlideClient({ call: jest.fn().mockRejectedValue(new Error('NOPERM')) } as any)).toBe(false); }); + it('ignores a client name that imitates the GLIDE library field', async () => { + const list = 'id=1 name=lib-name=GlidePySync(lmcache:0.5.5) lib-name=redis-py lib-ver=5.0\n'; + expect(await hasLmcacheGlideClient({ call: jest.fn().mockResolvedValue(list) } as any)).toBe(false); + }); + it('reads memory stats and other databases', () => { const info = { memory: { used_memory: '100', maxmemory: '200', maxmemory_policy: 'volatile-lru' }, diff --git a/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts b/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts index badd922db..dcd54d45f 100644 --- a/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts +++ b/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts @@ -1,16 +1,18 @@ -import { NotFoundException } from '@nestjs/common'; +import { ConflictException, NotFoundException } from '@nestjs/common'; import { Feature } from '@betterdb/shared'; import { KvCacheFootprintService, resolvePositiveInt } from '../kv-cache-footprint.service'; const KEY = (i: number) => `m/x@1@0@${(i + 1).toString(16)}@bfloat16`; +const REPLIES: Record = { MEMORY: 100, TTL: -1, EXISTS: 0, CLIENT: 'id=1 lib-name=redis-py\n' }; + function fakeClient(keys: string[]) { - const pipeline = { call: jest.fn().mockReturnThis(), ttl: jest.fn().mockReturnThis(), exists: jest.fn().mockReturnThis(), exec: jest.fn() }; - pipeline.exec.mockImplementation(async () => keys.flatMap(() => [[null, 100], [null, -1]])); return { getDbSize: jest.fn().mockResolvedValue(keys.length), - call: jest.fn(async (cmd: string) => (cmd === 'SCAN' ? ['0', keys] : 'id=1 lib-name=redis-py\n')), - getClient: () => ({ pipeline: () => pipeline }), + call: jest.fn(async (cmd: string) => (cmd === 'SCAN' ? ['0', keys] : REPLIES[cmd])), + getClient: () => { + throw new Error('Raw client access not available through agent connections'); + }, getInfoParsed: jest.fn().mockResolvedValue({ memory: { used_memory: '1000', maxmemory: '0', maxmemory_policy: 'noeviction' }, stats: { evicted_keys: '3' }, @@ -51,6 +53,58 @@ describe('KvCacheFootprintService', () => { expect((await service.triggerCollection('c1'))?.evictedKeysDelta).toBe(0); }); + it('rejects a manual trigger while a collection is already running', async () => { + const { service, client } = setup(); + let finishRead: (size: number) => void = () => undefined; + client.getDbSize.mockImplementationOnce(() => new Promise((resolve) => (finishRead = resolve))); + const first = service.triggerCollection('c1'); + await new Promise((resolve) => setImmediate(resolve)); + await expect(service.triggerCollection('c1')).rejects.toBeInstanceOf(ConflictException); + finishRead(6); + await expect(first).resolves.not.toBeNull(); + }); + + it('keeps the readable nodes and marks the scan incomplete when one node fails', async () => { + const { service, client } = setup(); + const broken = fakeClient([]); + broken.getDbSize.mockRejectedValue(new Error('LOADING')); + jest.spyOn(service as any, 'nodesFor').mockResolvedValue([ + { name: 'a', client }, + { name: 'b', client: broken }, + ]); + const snapshot = await service.triggerCollection('c1'); + expect(snapshot).toMatchObject({ matchedKeys: 6, scanComplete: false }); + }); + + it('fails the collection when every node fails', async () => { + const { service, client } = setup(); + client.getDbSize.mockRejectedValue(new Error('LOADING')); + await expect(service.triggerCollection('c1')).rejects.toThrow('LOADING'); + }); + + it('only asks CLIENT LIST for the GLIDE client when too few keys matched', async () => { + const many = setup(); + await many.service.triggerCollection('c1'); + expect(many.client.call).not.toHaveBeenCalledWith('CLIENT', ['LIST']); + + const few = setup([KEY(0)]); + await few.service.triggerCollection('c1'); + expect(few.client.call).toHaveBeenCalledWith('CLIENT', ['LIST']); + }); + + it('probes briefly until a connection is detected and scans fully afterwards', async () => { + const { service, client } = setup(); + client.call.mockImplementation(async (cmd: string) => (cmd === 'SCAN' ? ['7', []] : REPLIES[cmd])); + client.getDbSize.mockResolvedValue(1_000_000); + await service.triggerCollection('c1'); + expect(client.call.mock.calls.filter(([cmd]: [string]) => cmd === 'SCAN')).toHaveLength(2); + + (service as any).detected.add('c1'); + client.call.mockClear(); + await service.triggerCollection('c1'); + expect(client.call.mock.calls.filter(([cmd]: [string]) => cmd === 'SCAN')).toHaveLength(200); + }); + it('keeps going when a listener throws', async () => { const { service, storage } = setup(); service.onSnapshot(() => { throw new Error('boom'); }); @@ -115,8 +169,7 @@ describe('KvCacheFootprintService', () => { await (service as any).tick(); expect(storage.saveKvCacheFootprintSnapshot).toHaveBeenCalledTimes(1); client.getDbSize.mockResolvedValue(keys.length); - client.call.mockImplementation(async (cmd: string) => (cmd === 'SCAN' ? ['0', keys] : '')); - client.getClient().pipeline().exec.mockImplementation(async () => keys.flatMap(() => [[null, 100], [null, -1]])); + client.call.mockImplementation(async (cmd: string) => (cmd === 'SCAN' ? ['0', keys] : REPLIES[cmd])); expect((await service.triggerCollection('c1'))?.detected).toBe(true); await (service as any).tick(); expect(storage.saveKvCacheFootprintSnapshot).toHaveBeenCalledTimes(3); @@ -201,6 +254,19 @@ describe('KvCacheFootprintService', () => { expect(restored).toHaveBeenCalledWith('c1'); }); + it('announces a restore instead of deleting when the connection was only retired', async () => { + const { service, storage, registry } = setup(); + const restored = jest.fn(); + service.onConnectionRestore(restored); + registry.list.mockImplementation((options?: { includeRetired?: boolean }) => + options?.includeRetired ? [{ id: 'c1', name: 'one', isConnected: true, connectionType: 'direct' }] : [], + ); + (service as any).onConnectionRemoved('c1'); + await new Promise((resolve) => setImmediate(resolve)); + expect(storage.deleteKvCacheConnectionData).not.toHaveBeenCalled(); + expect(restored).toHaveBeenCalledWith('c1'); + }); + it('does not announce a restore when the connection stays removed', async () => { const { service, storage, registry } = setup(); const restored = jest.fn(); diff --git a/proprietary/kv-cache/footprint-scanner.ts b/proprietary/kv-cache/footprint-scanner.ts index aa8e469b6..39dd3f046 100644 --- a/proprietary/kv-cache/footprint-scanner.ts +++ b/proprietary/kv-cache/footprint-scanner.ts @@ -15,20 +15,30 @@ export interface ScanResult { } const SCAN_COUNT = 1000; -const GLIDE_LMCACHE = /lib-name=GlidePySync\(lmcache:/; +const PROBE_PAGES = 2; +const SAMPLE_BATCH = 100; +const GLIDE_LMCACHE_PREFIX = 'lib-name=GlidePySync(lmcache:'; -export async function scanLmcacheKeys(client: DatabasePort, budgets: ScanBudgets, dbSize: number): Promise { +export async function scanLmcacheKeys( + client: DatabasePort, + budgets: ScanBudgets, + dbSize: number, + probe = false, +): Promise { let cursor = '0'; let iterations = 0; + let globHits = 0; const matched: string[] = []; do { const [next, keys] = (await client.call('SCAN', [cursor, 'MATCH', LMCACHE_SCAN_PATTERN, 'COUNT', String(SCAN_COUNT)])) as [string, string[]]; cursor = String(next); iterations += 1; + globHits += keys.length; for (const key of keys) { if (classifyKey(key)) matched.push(key); } - } while (cursor !== '0' && iterations * SCAN_COUNT < budgets.maxScanned && matched.length < budgets.maxMatched); + if (probe && iterations >= PROBE_PAGES && globHits === 0) break; + } while (cursor !== '0' && iterations * SCAN_COUNT < budgets.maxScanned && globHits < budgets.maxMatched); const walked = cursor === '0' ? dbSize : Math.min(iterations * SCAN_COUNT, dbSize); const kept = matched.slice(0, budgets.maxMatched); const scannedKeys = kept.length < matched.length ? Math.max(1, Math.floor((walked * kept.length) / matched.length)) : walked; @@ -39,40 +49,45 @@ export async function scanLmcacheKeys(client: DatabasePort, budgets: ScanBudgets }; } +type Settled = { ok: true; value: unknown } | { ok: false }; + +const settle = (promise: Promise): Promise => + promise.then( + (value) => ({ ok: true as const, value }), + () => ({ ok: false as const }), + ); + +async function sampleKey(client: DatabasePort, key: string): Promise { + const parsed = classifyKey(key)!; + const sibling = siblingKey(key, parsed); + const [memory, ttl, exists] = await Promise.all([ + settle(client.call('MEMORY', ['USAGE', key])), + settle(client.call('TTL', [key])), + sibling ? settle(client.call('EXISTS', [sibling])) : Promise.resolve(null), + ]); + return { + key, + parsed, + memoryBytes: memory.ok && memory.value !== null ? Number(memory.value) : null, + ttl: ttl.ok ? Number(ttl.value) : -2, + siblingExists: exists?.ok ? Number(exists.value) === 1 : null, + }; +} + export async function sampleLmcacheKeys(client: DatabasePort, keys: string[]): Promise { - if (keys.length === 0) return []; - const pipeline = client.getClient().pipeline(); - const plan = keys.map((key) => { - const parsed = classifyKey(key)!; - const sibling = siblingKey(key, parsed); - pipeline.call('MEMORY', 'USAGE', key); - pipeline.ttl(key); - if (sibling) pipeline.exists(sibling); - return { key, parsed, sibling }; - }); - const replies = ((await pipeline.exec()) ?? []) as Array<[Error | null, unknown]>; - let index = 0; - return plan.map(({ key, parsed, sibling }) => { - const [memoryErr, memory] = replies[index++] ?? [new Error('missing'), null]; - const [ttlErr, ttl] = replies[index++] ?? [new Error('missing'), null]; - let siblingExists: boolean | null = null; - if (sibling) { - const [existsErr, exists] = replies[index++] ?? [new Error('missing'), null]; - siblingExists = existsErr ? null : Number(exists) === 1; - } - return { - key, - parsed, - memoryBytes: memoryErr || memory === null ? null : Number(memory), - ttl: ttlErr ? -2 : Number(ttl), - siblingExists, - }; - }); + const samples: KeySample[] = []; + for (let start = 0; start < keys.length; start += SAMPLE_BATCH) { + const batch = keys.slice(start, start + SAMPLE_BATCH); + samples.push(...(await Promise.all(batch.map((key) => sampleKey(client, key))))); + } + return samples; } export async function hasLmcacheGlideClient(client: DatabasePort): Promise { try { - return GLIDE_LMCACHE.test(String(await client.call('CLIENT', ['LIST']))); + return String(await client.call('CLIENT', ['LIST'])) + .split('\n') + .some((line) => line.split(' ').some((field) => field.startsWith(GLIDE_LMCACHE_PREFIX))); } catch { return false; } diff --git a/proprietary/kv-cache/kv-cache-footprint.service.ts b/proprietary/kv-cache/kv-cache-footprint.service.ts index e59f9145c..c88edc1b6 100644 --- a/proprietary/kv-cache/kv-cache-footprint.service.ts +++ b/proprietary/kv-cache/kv-cache-footprint.service.ts @@ -1,4 +1,13 @@ -import { Inject, Injectable, Logger, NotFoundException, OnModuleInit } from '@nestjs/common'; +import { + BadRequestException, + ConflictException, + Inject, + Injectable, + Logger, + NotFoundException, + OnModuleInit, + ServiceUnavailableException, +} from '@nestjs/common'; import { Feature, type KvCacheFootprintSnapshot } from '@betterdb/shared'; import { MultiConnectionPoller, type ConnectionContext } from '@app/common/services/multi-connection-poller'; import type { StoragePort } from '@app/common/interfaces/storage-port.interface'; @@ -6,7 +15,14 @@ import { ConnectionRegistry } from '@app/connections/connection-registry.service import { LicenseService } from '@proprietary/licenses/license.service'; import { collectionPlan, type CollectionPlan } from '../key-analytics/key-analytics-plan'; import { clusterScanNodes, primaryScanNodes, type ScanNode } from '../key-analytics/scan-nodes'; -import { buildSnapshot, evictedKeysDelta, pickSample, type MemoryStats, type NodeObservation } from './footprint-estimate'; +import { + buildSnapshot, + DETECTION_MIN_MATCHES, + evictedKeysDelta, + pickSample, + type MemoryStats, + type NodeObservation, +} from './footprint-estimate'; import { hasLmcacheGlideClient, otherNonEmptyDbs, readMemoryStats, sampleLmcacheKeys, scanLmcacheKeys } from './footprint-scanner'; import { classifyKey, maskKey } from './key-classifier'; @@ -39,7 +55,7 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On ); private readonly budgets = { maxScanned: resolvePositiveInt(process.env.KV_CACHE_SCAN_MAX_KEYS, 200000, 1), - maxMatched: resolvePositiveInt(process.env.KV_CACHE_MATCH_MAX_KEYS, 2000, 1), + maxMatched: resolvePositiveInt(process.env.KV_CACHE_MATCH_MAX_KEYS, 2000, DETECTION_MIN_MATCHES), }; private readonly sampleSize = resolvePositiveInt(process.env.KV_CACHE_SAMPLE_KEYS, 500, 1); private readonly undetectedRecheckMs = resolvePositiveInt( @@ -53,6 +69,8 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On private readonly lastEvicted = new Map>(); private readonly sampleKeys = new Map(); private readonly undetectedUntil = new Map(); + private readonly detected = new Set(); + private readonly memoryBlocked = new Set(); private readonly listeners: SnapshotListener[] = []; private readonly removalListeners: RemovalListener[] = []; private readonly restoreListeners: RemovalListener[] = []; @@ -87,6 +105,7 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On this.lastEvicted.delete(connectionId); this.sampleKeys.delete(connectionId); this.undetectedUntil.delete(connectionId); + this.detected.delete(connectionId); const settled = active?.then( () => undefined, () => undefined, @@ -131,7 +150,7 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On } private isRegistered(connectionId: string): boolean { - return this.connectionRegistry.list().some((connection) => connection.id === connectionId); + return this.connectionRegistry.list({ includeRetired: true }).some((connection) => connection.id === connectionId); } onConnectionRemoval(listener: RemovalListener): void { @@ -153,6 +172,14 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On async triggerCollection(connectionId: string): Promise { const config = this.connectionRegistry.getConfig(connectionId); if (!config) throw new NotFoundException(`Connection '${connectionId}' not found`); + if (this.isSentinelConnection(connectionId)) { + throw new BadRequestException('Sentinel connections have no keyspace to scan'); + } + const status = this.connectionRegistry.list().find((connection) => connection.id === connectionId); + if (status && !status.isConnected) throw new ServiceUnavailableException(`Connection '${connectionId}' is not connected`); + if (this.running.has(connectionId)) { + throw new ConflictException(`A KV cache collection is already running for '${connectionId}'`); + } const client = this.connectionRegistry.get(connectionId); return this.collect({ connectionId, @@ -185,10 +212,22 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On const current = () => this.generationOf(ctx.connectionId) === generation; const nodes = await this.nodesFor(ctx, plan); const ownDb = this.connectionRegistry.getConfig(ctx.connectionId)?.dbIndex ?? 0; - const readings = await Promise.all(nodes.map((node) => this.read(node, ownDb))); + const probe = !this.detected.has(ctx.connectionId); + const settled = await Promise.allSettled(nodes.map((node) => this.read(node, ownDb, probe))); + const readings: NodeReading[] = []; + const evicted = new Map(); + settled.forEach((result, index) => { + if (result.status === 'fulfilled') { + readings.push(result.value); + evicted.set(nodes[index].name, result.value.memory.evictedKeys); + } else { + this.logger.warn(`KV cache read failed on ${nodes[index].name}: ${errorMessage(result.reason)}`); + } + }); + const failed = settled.find((result): result is PromiseRejectedResult => result.status === 'rejected'); + if (readings.length === 0 && failed) throw failed.reason; const memory = this.sumMemory(readings.map((reading) => reading.memory)); - const evicted = new Map(nodes.map((node, index) => [node.name, readings[index].memory.evictedKeys])); - const snapshot = buildSnapshot({ + const built = buildSnapshot({ connectionId: ctx.connectionId, timestamp: Date.now(), nodes: readings.map((reading) => reading.observation), @@ -197,13 +236,19 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On otherDbs: readings[0]?.otherDbs ?? [], clientDetected: readings.some((reading) => reading.glide), }); + const snapshot = failed ? { ...built, scanComplete: false } : built; await this.deleting.get(ctx.connectionId); if (!current()) return null; await this.storage.saveKvCacheFootprintSnapshot(snapshot); if (!current()) return null; this.lastEvicted.set(ctx.connectionId, new Map([...(this.lastEvicted.get(ctx.connectionId) ?? []), ...evicted])); - if (snapshot.detected) this.undetectedUntil.delete(ctx.connectionId); - else this.undetectedUntil.set(ctx.connectionId, snapshot.timestamp + this.undetectedRecheckMs); + if (snapshot.detected) { + this.undetectedUntil.delete(ctx.connectionId); + this.detected.add(ctx.connectionId); + } else { + this.undetectedUntil.set(ctx.connectionId, snapshot.timestamp + this.undetectedRecheckMs); + this.detected.delete(ctx.connectionId); + } this.rememberSampleKey(ctx.connectionId, readings); await this.announce(snapshot); return snapshot; @@ -227,19 +272,30 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On return nodes; } - private async read(node: ScanNode, ownDb: number): Promise { + private async read(node: ScanNode, ownDb: number, probe: boolean): Promise { const dbSize = await node.client.getDbSize(); - const scan = await scanLmcacheKeys(node.client, this.budgets, dbSize); + const scan = await scanLmcacheKeys(node.client, this.budgets, dbSize, probe); const samples = await sampleLmcacheKeys(node.client, pickSample(scan.matchedKeys, this.sampleSize)); + this.noteMemoryAccess(node.name, samples.length > 0 && samples.every((sample) => sample.memoryBytes === null)); const info = await node.client.getInfoParsed(['memory', 'stats', 'keyspace']); return { observation: { ...scan, dbSize, samples }, memory: readMemoryStats(info), otherDbs: otherNonEmptyDbs(info, ownDb), - glide: await hasLmcacheGlideClient(node.client), + glide: scan.matchedKeys.length < DETECTION_MIN_MATCHES && (await hasLmcacheGlideClient(node.client)), }; } + private noteMemoryAccess(nodeName: string, blocked: boolean): void { + if (!blocked) { + this.memoryBlocked.delete(nodeName); + return; + } + if (this.memoryBlocked.has(nodeName)) return; + this.memoryBlocked.add(nodeName); + this.logger.warn(`MEMORY USAGE returned nothing on ${nodeName}; KV cache byte estimates will read as zero`); + } + private sumMemory(stats: MemoryStats[]): MemoryStats { return { usedMemory: stats.reduce((sum, s) => sum + s.usedMemory, 0), From 0b417644f89f4c1cb3445a99a152206c6b8fa51f Mon Sep 17 00:00:00 2001 From: jamby77 Date: Tue, 20 Oct 2026 11:14:27 +0300 Subject: [PATCH 12/31] Delete KV cache data when a connection is removed A retired cluster member keeps its KV data; this clears it once the connection itself is deleted. --- .../connections/__tests__/connection-registry.external.spec.ts | 1 + .../__tests__/connection-registry.membership.spec.ts | 1 + .../connections/__tests__/connection-registry.service.spec.ts | 3 +++ apps/api/src/connections/connection-registry.service.ts | 3 +++ 4 files changed, 8 insertions(+) diff --git a/apps/api/src/connections/__tests__/connection-registry.external.spec.ts b/apps/api/src/connections/__tests__/connection-registry.external.spec.ts index 90c8b65f1..4860a4574 100644 --- a/apps/api/src/connections/__tests__/connection-registry.external.spec.ts +++ b/apps/api/src/connections/__tests__/connection-registry.external.spec.ts @@ -11,6 +11,7 @@ function build() { const storage = { saveConnection: jest.fn().mockResolvedValue(undefined), deleteConnection: jest.fn().mockResolvedValue(undefined), + deleteKvCacheConnectionData: jest.fn().mockResolvedValue(undefined), updateConnection: jest.fn().mockResolvedValue(undefined), getConnections: jest.fn().mockResolvedValue([]), }; diff --git a/apps/api/src/connections/__tests__/connection-registry.membership.spec.ts b/apps/api/src/connections/__tests__/connection-registry.membership.spec.ts index 606e467ea..45185522c 100644 --- a/apps/api/src/connections/__tests__/connection-registry.membership.spec.ts +++ b/apps/api/src/connections/__tests__/connection-registry.membership.spec.ts @@ -10,6 +10,7 @@ function build(env: Record = {}) { const storage = { saveConnection: jest.fn().mockResolvedValue(undefined), deleteConnection: jest.fn().mockResolvedValue(undefined), + deleteKvCacheConnectionData: jest.fn().mockResolvedValue(undefined), updateConnection: jest.fn().mockResolvedValue(undefined), getConnections: jest.fn().mockResolvedValue([]), }; diff --git a/apps/api/src/connections/__tests__/connection-registry.service.spec.ts b/apps/api/src/connections/__tests__/connection-registry.service.spec.ts index 3368d1c3b..fe4a8a124 100644 --- a/apps/api/src/connections/__tests__/connection-registry.service.spec.ts +++ b/apps/api/src/connections/__tests__/connection-registry.service.spec.ts @@ -45,6 +45,7 @@ describe('ConnectionRegistry', () => { saveConnection: jest.fn().mockResolvedValue(undefined), getConnection: jest.fn().mockResolvedValue(null), deleteConnection: jest.fn().mockResolvedValue(undefined), + deleteKvCacheConnectionData: jest.fn().mockResolvedValue(undefined), updateConnection: jest.fn().mockResolvedValue(undefined), // Other methods not used in these tests saveAclEntries: jest.fn(), @@ -311,6 +312,7 @@ describe('ConnectionRegistry', () => { await registry.removeConnection(id); expect(mockStorage.deleteConnection).toHaveBeenCalledWith(id); + expect(mockStorage.deleteKvCacheConnectionData).toHaveBeenCalledWith(id); expect(() => registry.get(id)).toThrow(NotFoundException); }); @@ -517,6 +519,7 @@ describe('ConnectionRegistry with encryption', () => { saveConnection: jest.fn().mockResolvedValue(undefined), getConnection: jest.fn().mockResolvedValue(null), deleteConnection: jest.fn().mockResolvedValue(undefined), + deleteKvCacheConnectionData: jest.fn().mockResolvedValue(undefined), updateConnection: jest.fn().mockResolvedValue(undefined), saveAclEntries: jest.fn(), getAclEntries: jest.fn(), diff --git a/apps/api/src/connections/connection-registry.service.ts b/apps/api/src/connections/connection-registry.service.ts index d54b60a9c..5c982a2ec 100644 --- a/apps/api/src/connections/connection-registry.service.ts +++ b/apps/api/src/connections/connection-registry.service.ts @@ -739,6 +739,9 @@ export class ConnectionRegistry implements OnModuleInit, OnModuleDestroy { this.runtimeCapabilityTracker.removeConnection(id); this.externalMetricsStore.clear(id); await this.storage.deleteConnection(id); + await this.storage.deleteKvCacheConnectionData(id).catch((err) => { + this.logger.warn(`Failed to delete KV cache data for ${id}: ${err instanceof Error ? err.message : err}`); + }); if (removedConfig) { this.usageTelemetry?.trackDbRemove({ From 5f7ba278635a2516fcaf7bfe47f8c045031dab03 Mon Sep 17 00:00:00 2001 From: jamby77 Date: Tue, 20 Oct 2026 11:29:02 +0300 Subject: [PATCH 13/31] Route retired-member deletion through the KV removal path - Revert the immediate KV wipe in removeOne (75d5ddb2); it raced in-flight collections and engine writes - Sweep retired members each poll and run the guarded removal path once one is deleted from the registry - Probe only connections whose last snapshot was undetected, so a restart or retire/restore starts with a full scan --- .../connection-registry.external.spec.ts | 1 - .../connection-registry.membership.spec.ts | 1 - .../connection-registry.service.spec.ts | 3 -- .../connection-registry.service.ts | 3 -- .../kv-cache-footprint.service.spec.ts | 29 ++++++++++++++++--- .../kv-cache/kv-cache-footprint.service.ts | 21 ++++++++++---- 6 files changed, 41 insertions(+), 17 deletions(-) diff --git a/apps/api/src/connections/__tests__/connection-registry.external.spec.ts b/apps/api/src/connections/__tests__/connection-registry.external.spec.ts index 4860a4574..90c8b65f1 100644 --- a/apps/api/src/connections/__tests__/connection-registry.external.spec.ts +++ b/apps/api/src/connections/__tests__/connection-registry.external.spec.ts @@ -11,7 +11,6 @@ function build() { const storage = { saveConnection: jest.fn().mockResolvedValue(undefined), deleteConnection: jest.fn().mockResolvedValue(undefined), - deleteKvCacheConnectionData: jest.fn().mockResolvedValue(undefined), updateConnection: jest.fn().mockResolvedValue(undefined), getConnections: jest.fn().mockResolvedValue([]), }; diff --git a/apps/api/src/connections/__tests__/connection-registry.membership.spec.ts b/apps/api/src/connections/__tests__/connection-registry.membership.spec.ts index 45185522c..606e467ea 100644 --- a/apps/api/src/connections/__tests__/connection-registry.membership.spec.ts +++ b/apps/api/src/connections/__tests__/connection-registry.membership.spec.ts @@ -10,7 +10,6 @@ function build(env: Record = {}) { const storage = { saveConnection: jest.fn().mockResolvedValue(undefined), deleteConnection: jest.fn().mockResolvedValue(undefined), - deleteKvCacheConnectionData: jest.fn().mockResolvedValue(undefined), updateConnection: jest.fn().mockResolvedValue(undefined), getConnections: jest.fn().mockResolvedValue([]), }; diff --git a/apps/api/src/connections/__tests__/connection-registry.service.spec.ts b/apps/api/src/connections/__tests__/connection-registry.service.spec.ts index fe4a8a124..3368d1c3b 100644 --- a/apps/api/src/connections/__tests__/connection-registry.service.spec.ts +++ b/apps/api/src/connections/__tests__/connection-registry.service.spec.ts @@ -45,7 +45,6 @@ describe('ConnectionRegistry', () => { saveConnection: jest.fn().mockResolvedValue(undefined), getConnection: jest.fn().mockResolvedValue(null), deleteConnection: jest.fn().mockResolvedValue(undefined), - deleteKvCacheConnectionData: jest.fn().mockResolvedValue(undefined), updateConnection: jest.fn().mockResolvedValue(undefined), // Other methods not used in these tests saveAclEntries: jest.fn(), @@ -312,7 +311,6 @@ describe('ConnectionRegistry', () => { await registry.removeConnection(id); expect(mockStorage.deleteConnection).toHaveBeenCalledWith(id); - expect(mockStorage.deleteKvCacheConnectionData).toHaveBeenCalledWith(id); expect(() => registry.get(id)).toThrow(NotFoundException); }); @@ -519,7 +517,6 @@ describe('ConnectionRegistry with encryption', () => { saveConnection: jest.fn().mockResolvedValue(undefined), getConnection: jest.fn().mockResolvedValue(null), deleteConnection: jest.fn().mockResolvedValue(undefined), - deleteKvCacheConnectionData: jest.fn().mockResolvedValue(undefined), updateConnection: jest.fn().mockResolvedValue(undefined), saveAclEntries: jest.fn(), getAclEntries: jest.fn(), diff --git a/apps/api/src/connections/connection-registry.service.ts b/apps/api/src/connections/connection-registry.service.ts index 5c982a2ec..d54b60a9c 100644 --- a/apps/api/src/connections/connection-registry.service.ts +++ b/apps/api/src/connections/connection-registry.service.ts @@ -739,9 +739,6 @@ export class ConnectionRegistry implements OnModuleInit, OnModuleDestroy { this.runtimeCapabilityTracker.removeConnection(id); this.externalMetricsStore.clear(id); await this.storage.deleteConnection(id); - await this.storage.deleteKvCacheConnectionData(id).catch((err) => { - this.logger.warn(`Failed to delete KV cache data for ${id}: ${err instanceof Error ? err.message : err}`); - }); if (removedConfig) { this.usageTelemetry?.trackDbRemove({ diff --git a/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts b/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts index dcd54d45f..1d0339756 100644 --- a/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts +++ b/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts @@ -92,17 +92,38 @@ describe('KvCacheFootprintService', () => { expect(few.client.call).toHaveBeenCalledWith('CLIENT', ['LIST']); }); - it('probes briefly until a connection is detected and scans fully afterwards', async () => { + it('scans fully first and only probes a connection whose last snapshot was undetected', async () => { const { service, client } = setup(); client.call.mockImplementation(async (cmd: string) => (cmd === 'SCAN' ? ['7', []] : REPLIES[cmd])); client.getDbSize.mockResolvedValue(1_000_000); + const scans = () => client.call.mock.calls.filter(([cmd]: [string]) => cmd === 'SCAN').length; await service.triggerCollection('c1'); - expect(client.call.mock.calls.filter(([cmd]: [string]) => cmd === 'SCAN')).toHaveLength(2); + expect(scans()).toBe(200); - (service as any).detected.add('c1'); client.call.mockClear(); await service.triggerCollection('c1'); - expect(client.call.mock.calls.filter(([cmd]: [string]) => cmd === 'SCAN')).toHaveLength(200); + expect(scans()).toBe(2); + + (service as any).onConnectionRemoved('c1'); + client.call.mockClear(); + await service.triggerCollection('c1'); + expect(scans()).toBe(200); + }); + + it('deletes the data of a retired member once it is deleted from the registry', async () => { + const { service, storage, registry, license } = setup(); + license.hasFeature.mockReturnValue(false); + const member = { id: 'm1', name: 'member', isConnected: true, connectionType: 'direct' }; + const live = [{ id: 'c1', name: 'one', isConnected: true, connectionType: 'direct' }]; + registry.list.mockImplementation((options?: { includeRetired?: boolean }) => (options?.includeRetired ? [...live, member] : live)); + await (service as any).tick(); + expect(storage.deleteKvCacheConnectionData).not.toHaveBeenCalled(); + + registry.list.mockImplementation(() => live); + await (service as any).tick(); + await new Promise((resolve) => setImmediate(resolve)); + expect(storage.deleteKvCacheConnectionData).toHaveBeenCalledWith('m1'); + expect(storage.deleteKvCacheConnectionData).not.toHaveBeenCalledWith('c1'); }); it('keeps going when a listener throws', async () => { diff --git a/proprietary/kv-cache/kv-cache-footprint.service.ts b/proprietary/kv-cache/kv-cache-footprint.service.ts index c88edc1b6..cb39b07d5 100644 --- a/proprietary/kv-cache/kv-cache-footprint.service.ts +++ b/proprietary/kv-cache/kv-cache-footprint.service.ts @@ -69,7 +69,8 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On private readonly lastEvicted = new Map>(); private readonly sampleKeys = new Map(); private readonly undetectedUntil = new Map(); - private readonly detected = new Set(); + private readonly undetected = new Set(); + private retired = new Set(); private readonly memoryBlocked = new Set(); private readonly listeners: SnapshotListener[] = []; private readonly removalListeners: RemovalListener[] = []; @@ -92,6 +93,7 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On } protected async pollConnection(ctx: ConnectionContext): Promise { + this.sweepDeletedRetirees(); if (!this.license.hasFeature(Feature.KV_CACHE_MONITORING)) return; if (Date.now() < (this.undetectedUntil.get(ctx.connectionId) ?? 0)) return; await this.collect(ctx); @@ -105,7 +107,7 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On this.lastEvicted.delete(connectionId); this.sampleKeys.delete(connectionId); this.undetectedUntil.delete(connectionId); - this.detected.delete(connectionId); + this.undetected.delete(connectionId); const settled = active?.then( () => undefined, () => undefined, @@ -122,6 +124,15 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On .catch((error) => this.logger.warn(`Could not delete KV cache data for ${connectionId}: ${errorMessage(error)}`)); } + private sweepDeletedRetirees(): void { + const all = this.connectionRegistry.list({ includeRetired: true }).map((connection) => connection.id); + const live = new Set(this.connectionRegistry.list().map((connection) => connection.id)); + const known = new Set(all); + const deleted = [...this.retired].filter((id) => !known.has(id)); + this.retired = new Set(all.filter((id) => !live.has(id))); + for (const id of deleted) this.onConnectionRemoved(id); + } + private async deleteRemovedData(connectionId: string, generation: number): Promise { if (this.generationOf(connectionId) !== generation) return; if (this.isRegistered(connectionId)) return this.announceRestore(connectionId); @@ -212,7 +223,7 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On const current = () => this.generationOf(ctx.connectionId) === generation; const nodes = await this.nodesFor(ctx, plan); const ownDb = this.connectionRegistry.getConfig(ctx.connectionId)?.dbIndex ?? 0; - const probe = !this.detected.has(ctx.connectionId); + const probe = this.undetected.has(ctx.connectionId); const settled = await Promise.allSettled(nodes.map((node) => this.read(node, ownDb, probe))); const readings: NodeReading[] = []; const evicted = new Map(); @@ -244,10 +255,10 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On this.lastEvicted.set(ctx.connectionId, new Map([...(this.lastEvicted.get(ctx.connectionId) ?? []), ...evicted])); if (snapshot.detected) { this.undetectedUntil.delete(ctx.connectionId); - this.detected.add(ctx.connectionId); + this.undetected.delete(ctx.connectionId); } else { this.undetectedUntil.set(ctx.connectionId, snapshot.timestamp + this.undetectedRecheckMs); - this.detected.delete(ctx.connectionId); + this.undetected.add(ctx.connectionId); } this.rememberSampleKey(ctx.connectionId, readings); await this.announce(snapshot); From cb465e903ae95b0baf2db521041a7a90ad5f40ff Mon Sep 17 00:00:00 2001 From: jamby77 Date: Tue, 20 Oct 2026 11:38:38 +0300 Subject: [PATCH 14/31] Sweep deleted KV retirees on their own timer The sweep ran only while polling a connected connection, so a retired member deleted when nothing was connected kept its KV data. --- .../kv-cache-footprint.service.spec.ts | 32 +++++++++++-------- .../kv-cache/kv-cache-footprint.service.ts | 11 ++++++- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts b/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts index 1d0339756..7e3358ede 100644 --- a/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts +++ b/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts @@ -110,20 +110,24 @@ describe('KvCacheFootprintService', () => { expect(scans()).toBe(200); }); - it('deletes the data of a retired member once it is deleted from the registry', async () => { - const { service, storage, registry, license } = setup(); - license.hasFeature.mockReturnValue(false); - const member = { id: 'm1', name: 'member', isConnected: true, connectionType: 'direct' }; - const live = [{ id: 'c1', name: 'one', isConnected: true, connectionType: 'direct' }]; - registry.list.mockImplementation((options?: { includeRetired?: boolean }) => (options?.includeRetired ? [...live, member] : live)); - await (service as any).tick(); - expect(storage.deleteKvCacheConnectionData).not.toHaveBeenCalled(); - - registry.list.mockImplementation(() => live); - await (service as any).tick(); - await new Promise((resolve) => setImmediate(resolve)); - expect(storage.deleteKvCacheConnectionData).toHaveBeenCalledWith('m1'); - expect(storage.deleteKvCacheConnectionData).not.toHaveBeenCalledWith('c1'); + it('deletes the data of a retired member once it is deleted, even with no connected connections', async () => { + jest.useFakeTimers({ doNotFake: ['setImmediate', 'nextTick'] }); + const { service, storage, registry } = setup(); + const member = { id: 'm1', name: 'member', isConnected: false, connectionType: 'direct' }; + registry.list.mockImplementation((options?: { includeRetired?: boolean }) => (options?.includeRetired ? [member] : [])); + try { + await service.onModuleInit(); + await jest.advanceTimersByTimeAsync(300000); + expect(storage.deleteKvCacheConnectionData).not.toHaveBeenCalled(); + + registry.list.mockImplementation(() => []); + await jest.advanceTimersByTimeAsync(300000); + await new Promise((resolve) => setImmediate(resolve)); + expect(storage.deleteKvCacheConnectionData).toHaveBeenCalledWith('m1'); + } finally { + await service.onModuleDestroy(); + jest.useRealTimers(); + } }); it('keeps going when a listener throws', async () => { diff --git a/proprietary/kv-cache/kv-cache-footprint.service.ts b/proprietary/kv-cache/kv-cache-footprint.service.ts index cb39b07d5..4ec8968cc 100644 --- a/proprietary/kv-cache/kv-cache-footprint.service.ts +++ b/proprietary/kv-cache/kv-cache-footprint.service.ts @@ -71,6 +71,7 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On private readonly undetectedUntil = new Map(); private readonly undetected = new Set(); private retired = new Set(); + private sweepTimer: NodeJS.Timeout | null = null; private readonly memoryBlocked = new Set(); private readonly listeners: SnapshotListener[] = []; private readonly removalListeners: RemovalListener[] = []; @@ -86,6 +87,15 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On async onModuleInit(): Promise { this.start(); + this.sweepDeletedRetirees(); + this.sweepTimer = setInterval(() => this.sweepDeletedRetirees(), this.intervalMs); + this.sweepTimer.unref?.(); + } + + async onModuleDestroy(): Promise { + if (this.sweepTimer) clearInterval(this.sweepTimer); + this.sweepTimer = null; + await super.onModuleDestroy(); } protected getIntervalMs(): number { @@ -93,7 +103,6 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On } protected async pollConnection(ctx: ConnectionContext): Promise { - this.sweepDeletedRetirees(); if (!this.license.hasFeature(Feature.KV_CACHE_MONITORING)) return; if (Date.now() < (this.undetectedUntil.get(ctx.connectionId) ?? 0)) return; await this.collect(ctx); From c960e3fceef0751369b0c1eaad1d29a9817ab81c Mon Sep 17 00:00:00 2001 From: jamby77 Date: Tue, 20 Oct 2026 14:20:34 +0300 Subject: [PATCH 15/31] Keep the KV evicting state when a footprint read is partial - Tell snapshot listeners when some cluster nodes could not be read - Neither raise nor clear the evicting advisory from such a snapshot, since its memory ratio and eviction delta cover only part of the cluster --- .../__tests__/kv-cache-alerts.service.spec.ts | 20 +++++++++++++++++++ .../kv-cache-footprint.service.spec.ts | 19 +++++++++++++++++- .../kv-cache/kv-cache-alerts.service.ts | 10 ++++++---- .../kv-cache/kv-cache-footprint.service.ts | 12 +++++++---- 4 files changed, 52 insertions(+), 9 deletions(-) diff --git a/proprietary/kv-cache/__tests__/kv-cache-alerts.service.spec.ts b/proprietary/kv-cache/__tests__/kv-cache-alerts.service.spec.ts index 4ab92fc27..6c6cc547e 100644 --- a/proprietary/kv-cache/__tests__/kv-cache-alerts.service.spec.ts +++ b/proprietary/kv-cache/__tests__/kv-cache-alerts.service.spec.ts @@ -135,6 +135,26 @@ describe('KvCacheAlertsService eviction risk', () => { expect(evictingStates()).toEqual([true, true, true, true, true, false, false]); }); + it('keeps the evicting state when a snapshot only read some nodes', async () => { + const { service, pro } = setup(); + const evictingStates = () => + pro.dispatchKvCacheEvictionRisk.mock.calls.filter(([e]) => e.reason === 'evicting').map(([e]) => e.active); + await service.onSnapshot(snapshot({ evictedKeysDelta: 50 }), { partial: true }); + await service.onSnapshot(snapshot({ evictedKeysDelta: 50 })); + for (let i = 0; i < 3; i += 1) await service.onSnapshot(snapshot({ evictedKeysDelta: 0 }), { partial: true }); + await service.onSnapshot(snapshot({ evictedKeysDelta: 0 })); + expect(evictingStates()).toEqual([false, true, true, true, true, true]); + }); + + it('passes the partial flag from footprint snapshots through', async () => { + const { service, pro, footprint } = setup(); + service.onModuleInit(); + const listener = footprint.onSnapshot.mock.calls[0][0]; + await listener(snapshot({ evictedKeysDelta: 50 }), { partial: true }); + expect(pro.dispatchKvCacheEvictionRisk).toHaveBeenCalledWith(expect.objectContaining({ reason: 'evicting', active: false })); + service.onModuleDestroy(); + }); + it('forgets the evicting hold when the connection is removed', async () => { const { service, pro, remove } = setup(); service.onModuleInit(); diff --git a/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts b/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts index 622b5b913..478937a10 100644 --- a/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts +++ b/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts @@ -49,7 +49,7 @@ describe('KvCacheFootprintService', () => { const snapshot = await service.triggerCollection('c1'); expect(snapshot).toMatchObject({ connectionId: 'c1', detected: true, matchedKeys: 6, chunksEst: 6, evictedKeysDelta: null }); expect(storage.saveKvCacheFootprintSnapshot).toHaveBeenCalledWith(snapshot); - expect(listener).toHaveBeenCalledWith(snapshot); + expect(listener).toHaveBeenCalledWith(snapshot, { partial: false }); expect(service.getSampleKey('c1')).toBe('m/x@1@0@1…@bfloat16'); }); @@ -82,6 +82,23 @@ describe('KvCacheFootprintService', () => { expect(snapshot).toMatchObject({ matchedKeys: 6, scanComplete: false }); }); + it('tells snapshot listeners whether every node was read', async () => { + const { service, client } = setup(); + const listener = jest.fn(); + service.onSnapshot(listener); + const broken = fakeClient([]); + broken.getDbSize.mockRejectedValue(new Error('LOADING')); + const nodes = jest.spyOn(service as any, 'nodesFor').mockResolvedValue([ + { name: 'a', client }, + { name: 'b', client: broken }, + ]); + await service.triggerCollection('c1'); + expect(listener).toHaveBeenLastCalledWith(expect.objectContaining({ connectionId: 'c1' }), { partial: true }); + nodes.mockResolvedValue([{ name: 'a', client }]); + await service.triggerCollection('c1'); + expect(listener).toHaveBeenLastCalledWith(expect.objectContaining({ connectionId: 'c1' }), { partial: false }); + }); + it('fails the collection when every node fails', async () => { const { service, client } = setup(); client.getDbSize.mockRejectedValue(new Error('LOADING')); diff --git a/proprietary/kv-cache/kv-cache-alerts.service.ts b/proprietary/kv-cache/kv-cache-alerts.service.ts index ad45c3917..6e690c9c0 100644 --- a/proprietary/kv-cache/kv-cache-alerts.service.ts +++ b/proprietary/kv-cache/kv-cache-alerts.service.ts @@ -16,7 +16,7 @@ import { PrometheusService } from '@app/prometheus/prometheus.service'; import { LicenseService } from '@proprietary/licenses/license.service'; import { evictionConditions, shouldCheckHitRate, windowHitRates } from './kv-cache-alert-rules'; import { KvCacheEngineRegistry } from './kv-cache-engine-registry'; -import { KvCacheFootprintService } from './kv-cache-footprint.service'; +import { KvCacheFootprintService, type SnapshotContext } from './kv-cache-footprint.service'; const EVALUATE_INTERVAL_MS = 60_000; const EVICTION_REASONS: KvCacheEvictionReason[] = ['unevictable', 'evicting']; @@ -49,7 +49,7 @@ export class KvCacheAlertsService implements OnModuleInit, OnModuleDestroy { ) {} onModuleInit(): void { - this.footprint.onSnapshot((snapshot) => this.onSnapshot(snapshot)); + this.footprint.onSnapshot((snapshot, context) => this.onSnapshot(snapshot, context)); this.footprint.onConnectionRemoval((connectionId) => { this.exportedConnections.delete(connectionId); this.checkedWindows.delete(connectionId); @@ -84,7 +84,7 @@ export class KvCacheAlertsService implements OnModuleInit, OnModuleDestroy { return this.storage.saveKvCacheSettings({ ...current, ...definedUpdate, connectionId, updatedAt: Date.now() }); } - async onSnapshot(snapshot: KvCacheFootprintSnapshot): Promise { + async onSnapshot(snapshot: KvCacheFootprintSnapshot, context: SnapshotContext = { partial: false }): Promise { try { if (!this.license.hasFeature(Feature.KV_CACHE_MONITORING)) return; this.prometheus?.setKvCacheFootprint(snapshot.connectionId, snapshot.perModel); @@ -92,7 +92,9 @@ export class KvCacheAlertsService implements OnModuleInit, OnModuleDestroy { const settings = await this.getSettings(snapshot.connectionId); if (!settings.evictionAlertEnabled) return; const conditions = evictionConditions(snapshot); - conditions.evicting = this.holdEvicting(snapshot.connectionId, conditions.evicting); + conditions.evicting = context.partial + ? this.evictingCleanStreaks.has(snapshot.connectionId) + : this.holdEvicting(snapshot.connectionId, conditions.evicting); const instance = this.instanceFor(snapshot.connectionId); for (const reason of EVICTION_REASONS) { await this.webhookEventsPro.dispatchKvCacheEvictionRisk({ diff --git a/proprietary/kv-cache/kv-cache-footprint.service.ts b/proprietary/kv-cache/kv-cache-footprint.service.ts index 226b81539..a870a1952 100644 --- a/proprietary/kv-cache/kv-cache-footprint.service.ts +++ b/proprietary/kv-cache/kv-cache-footprint.service.ts @@ -26,7 +26,11 @@ import { import { hasLmcacheGlideClient, otherNonEmptyDbs, readMemoryStats, sampleLmcacheKeys, scanLmcacheKeys } from './footprint-scanner'; import { classifyKey, maskKey } from './key-classifier'; -export type SnapshotListener = (snapshot: KvCacheFootprintSnapshot) => void | Promise; +export interface SnapshotContext { + partial: boolean; +} + +export type SnapshotListener = (snapshot: KvCacheFootprintSnapshot, context: SnapshotContext) => void | Promise; export type RemovalListener = (connectionId: string) => void | Promise; interface NodeReading { @@ -278,7 +282,7 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On this.undetected.add(ctx.connectionId); } this.rememberSampleKey(ctx.connectionId, readings); - await this.announce(snapshot); + await this.announce(snapshot, { partial: failed !== undefined }); return snapshot; } @@ -340,10 +344,10 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On else this.sampleKeys.delete(connectionId); } - private async announce(snapshot: KvCacheFootprintSnapshot): Promise { + private async announce(snapshot: KvCacheFootprintSnapshot, context: SnapshotContext): Promise { for (const listener of this.listeners) { try { - await listener(snapshot); + await listener(snapshot, context); } catch (error) { this.logger.warn(`KV cache snapshot listener failed: ${errorMessage(error)}`); } From 9e0e91660e706b68c2d16366dc23e95814a861ec Mon Sep 17 00:00:00 2001 From: jamby77 Date: Tue, 20 Oct 2026 14:25:38 +0300 Subject: [PATCH 16/31] Mark KV footprint partial when cluster nodes are skipped - Count deregistered members and failed role reads as unread nodes - Set scanComplete false and partial true when any planned node is unread - Expose readPrimaryScanNodes; primaryScanNodes keeps its contract --- proprietary/key-analytics/scan-nodes.ts | 13 ++++- .../kv-cache-footprint.service.spec.ts | 55 ++++++++++++++++--- .../kv-cache/kv-cache-footprint.service.ts | 22 ++++---- 3 files changed, 68 insertions(+), 22 deletions(-) diff --git a/proprietary/key-analytics/scan-nodes.ts b/proprietary/key-analytics/scan-nodes.ts index 5f7303ad3..0267164c9 100644 --- a/proprietary/key-analytics/scan-nodes.ts +++ b/proprietary/key-analytics/scan-nodes.ts @@ -27,13 +27,24 @@ export function clusterScanNodes( } export async function primaryScanNodes(nodes: ScanNode[], logger: Logger, label: string): Promise { + return (await readPrimaryScanNodes(nodes, logger, label)).nodes; +} + +export async function readPrimaryScanNodes( + nodes: ScanNode[], + logger: Logger, + label: string, +): Promise<{ nodes: ScanNode[]; unreadable: number }> { const roles = await Promise.allSettled(nodes.map((node) => node.client.getRole())); - return nodes.filter((node, index) => { + let unreadable = 0; + const primaries = nodes.filter((node, index) => { const role = roles[index]; if (role.status === 'rejected') { + unreadable += 1; logger.warn(`${label} could not read the role of ${node.name}: ${role.reason instanceof Error ? role.reason.message : role.reason}`); return false; } return role.value.role === 'master'; }); + return { nodes: primaries, unreadable }; } diff --git a/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts b/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts index 478937a10..aceb45d32 100644 --- a/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts +++ b/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts @@ -74,10 +74,13 @@ describe('KvCacheFootprintService', () => { const { service, client } = setup(); const broken = fakeClient([]); broken.getDbSize.mockRejectedValue(new Error('LOADING')); - jest.spyOn(service as any, 'nodesFor').mockResolvedValue([ - { name: 'a', client }, - { name: 'b', client: broken }, - ]); + jest.spyOn(service as any, 'nodesFor').mockResolvedValue({ + nodes: [ + { name: 'a', client }, + { name: 'b', client: broken }, + ], + skipped: 0, + }); const snapshot = await service.triggerCollection('c1'); expect(snapshot).toMatchObject({ matchedKeys: 6, scanComplete: false }); }); @@ -88,17 +91,51 @@ describe('KvCacheFootprintService', () => { service.onSnapshot(listener); const broken = fakeClient([]); broken.getDbSize.mockRejectedValue(new Error('LOADING')); - const nodes = jest.spyOn(service as any, 'nodesFor').mockResolvedValue([ - { name: 'a', client }, - { name: 'b', client: broken }, - ]); + const nodes = jest.spyOn(service as any, 'nodesFor').mockResolvedValue({ + nodes: [ + { name: 'a', client }, + { name: 'b', client: broken }, + ], + skipped: 0, + }); await service.triggerCollection('c1'); expect(listener).toHaveBeenLastCalledWith(expect.objectContaining({ connectionId: 'c1' }), { partial: true }); - nodes.mockResolvedValue([{ name: 'a', client }]); + nodes.mockResolvedValue({ nodes: [{ name: 'a', client }], skipped: 0 }); await service.triggerCollection('c1'); expect(listener).toHaveBeenLastCalledWith(expect.objectContaining({ connectionId: 'c1' }), { partial: false }); }); + describe('cluster members that were never read', () => { + function clusterSetup(member: 'deregistered' | 'role-failed' | 'replica') { + const ctx = setup(); + const listener = jest.fn(); + ctx.service.onSnapshot(listener); + const other = fakeClient([]); + if (member === 'role-failed') other.getRole.mockRejectedValue(new Error('timeout')); + if (member === 'replica') other.getRole.mockResolvedValue({ role: 'slave' }); + ctx.registry.get.mockImplementation((id: string) => { + if (id === 'm1' && member === 'deregistered') throw new Error('not registered'); + return id === 'm1' ? other : ctx.client; + }); + jest.spyOn(ctx.service as any, 'planFor').mockReturnValue({ kind: 'cluster', members: [{ id: 'm1', name: 'm1' }] }); + return { ...ctx, listener }; + } + + it.each(['deregistered', 'role-failed'] as const)('marks the snapshot partial when a member is %s', async (member) => { + const { service, listener } = clusterSetup(member); + const snapshot = await service.triggerCollection('c1'); + expect(snapshot).toMatchObject({ scanComplete: false }); + expect(listener).toHaveBeenLastCalledWith(expect.objectContaining({ connectionId: 'c1' }), { partial: true }); + }); + + it('does not treat a replica as unread', async () => { + const { service, listener } = clusterSetup('replica'); + const snapshot = await service.triggerCollection('c1'); + expect(snapshot).toMatchObject({ scanComplete: true }); + expect(listener).toHaveBeenLastCalledWith(expect.objectContaining({ connectionId: 'c1' }), { partial: false }); + }); + }); + it('fails the collection when every node fails', async () => { const { service, client } = setup(); client.getDbSize.mockRejectedValue(new Error('LOADING')); diff --git a/proprietary/kv-cache/kv-cache-footprint.service.ts b/proprietary/kv-cache/kv-cache-footprint.service.ts index a870a1952..2f72b5b35 100644 --- a/proprietary/kv-cache/kv-cache-footprint.service.ts +++ b/proprietary/kv-cache/kv-cache-footprint.service.ts @@ -14,7 +14,7 @@ import type { StoragePort } from '@app/common/interfaces/storage-port.interface' import { ConnectionRegistry } from '@app/connections/connection-registry.service'; import { LicenseService } from '@proprietary/licenses/license.service'; import { collectionPlan, type CollectionPlan } from '../key-analytics/key-analytics-plan'; -import { clusterScanNodes, primaryScanNodes, type ScanNode } from '../key-analytics/scan-nodes'; +import { clusterScanNodes, readPrimaryScanNodes, type ScanNode } from '../key-analytics/scan-nodes'; import { buildSnapshot, DETECTION_MIN_MATCHES, @@ -242,7 +242,7 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On private async runCollection(ctx: ConnectionContext, plan: CollectionPlan): Promise { const generation = this.generationOf(ctx.connectionId); const current = () => this.generationOf(ctx.connectionId) === generation; - const nodes = await this.nodesFor(ctx, plan); + const { nodes, skipped } = await this.nodesFor(ctx, plan); const ownDb = this.connectionRegistry.getConfig(ctx.connectionId)?.dbIndex ?? 0; const probe = this.undetected.has(ctx.connectionId); const settled = await Promise.allSettled(nodes.map((node) => this.read(node, ownDb, probe))); @@ -268,7 +268,8 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On otherDbs: readings[0]?.otherDbs ?? [], clientDetected: readings.some((reading) => reading.glide), }); - const snapshot = failed ? { ...built, scanComplete: false } : built; + const partial = failed !== undefined || skipped > 0; + const snapshot = partial ? { ...built, scanComplete: false } : built; await this.deleting.get(ctx.connectionId); if (!current()) return null; await this.storage.saveKvCacheFootprintSnapshot(snapshot); @@ -282,7 +283,7 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On this.undetected.add(ctx.connectionId); } this.rememberSampleKey(ctx.connectionId, readings); - await this.announce(snapshot, { partial: failed !== undefined }); + await this.announce(snapshot, { partial }); return snapshot; } @@ -292,16 +293,13 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On return connection ? collectionPlan(connection, connections) : { kind: 'single' }; } - private async nodesFor(ctx: ConnectionContext, plan: CollectionPlan): Promise { + private async nodesFor(ctx: ConnectionContext, plan: CollectionPlan): Promise<{ nodes: ScanNode[]; skipped: number }> { const seed = { name: ctx.connectionName, client: ctx.client }; - if (plan.kind !== 'cluster') return [seed]; - const nodes = await primaryScanNodes( - clusterScanNodes(this.connectionRegistry, seed, plan.members, this.logger, 'KV cache'), - this.logger, - 'KV cache', - ); + if (plan.kind !== 'cluster') return { nodes: [seed], skipped: 0 }; + const registered = clusterScanNodes(this.connectionRegistry, seed, plan.members, this.logger, 'KV cache'); + const { nodes, unreadable } = await readPrimaryScanNodes(registered, this.logger, 'KV cache'); if (nodes.length === 0) throw new Error(`No reachable primary found for ${ctx.connectionName}`); - return nodes; + return { nodes, skipped: 1 + plan.members.length - registered.length + unreadable }; } private async read(node: ScanNode, ownDb: number, probe: boolean): Promise { From 6c85dba120887054abd74abe1f875d368dc92d91 Mon Sep 17 00:00:00 2001 From: jamby77 Date: Tue, 20 Oct 2026 15:05:27 +0300 Subject: [PATCH 17/31] Treat partial KV footprint reads as inconclusive for detection - A partial read with too few keys keeps the previous detection state and no longer starts the undetected back-off or probe scans - Count disconnected cluster members that still own slots as unread primaries, so the snapshot is partial with scanComplete false --- .../kv-cache-footprint.service.spec.ts | 85 ++++++++++++++++++- .../kv-cache/kv-cache-footprint.service.ts | 44 ++++++++-- 2 files changed, 123 insertions(+), 6 deletions(-) diff --git a/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts b/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts index aceb45d32..a933e2b97 100644 --- a/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts +++ b/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts @@ -35,7 +35,11 @@ function setup(keys = Array.from({ length: 6 }, (_, i) => KEY(i)), licensed = tr }), removed: (id: string) => removalListeners.forEach((listener) => listener(id)), } as any; - const storage = { saveKvCacheFootprintSnapshot: jest.fn(), deleteKvCacheConnectionData: jest.fn().mockResolvedValue(undefined) } as any; + const storage = { + saveKvCacheFootprintSnapshot: jest.fn(), + getKvCacheFootprintSnapshots: jest.fn().mockResolvedValue([]), + deleteKvCacheConnectionData: jest.fn().mockResolvedValue(undefined), + } as any; const license = { hasFeature: jest.fn((f: string) => licensed && f === Feature.KV_CACHE_MONITORING) } as any; const service = new KvCacheFootprintService(registry, storage, license); return { service, storage, client, registry, license }; @@ -136,6 +140,85 @@ describe('KvCacheFootprintService', () => { }); }); + describe('disconnected cluster members', () => { + function offlineSetup(clusterNodes: () => Promise) { + const ctx = setup(); + const listener = jest.fn(); + ctx.service.onSnapshot(listener); + ctx.registry.list.mockReturnValue([ + { id: 'c1', name: 'one', isConnected: true, connectionType: 'direct' }, + { + id: 'm1', + name: 'm1', + isConnected: false, + connectionType: 'direct', + membership: { seedId: 'c1', nodeId: 'n1', origin: 'auto', source: 'cluster' }, + }, + ]); + ctx.client.getClusterNodes = jest.fn(clusterNodes); + return { ...ctx, listener }; + } + + it('marks the snapshot partial when a disconnected member still owns slots as a primary', async () => { + const { service, listener } = offlineSetup(async () => [{ id: 'n1', flags: ['master', 'fail?'], slots: [[0, 5460]] }]); + const snapshot = await service.triggerCollection('c1'); + expect(snapshot).toMatchObject({ scanComplete: false }); + expect(listener).toHaveBeenLastCalledWith(expect.objectContaining({ connectionId: 'c1' }), { partial: true }); + }); + + it('marks the snapshot partial when the cluster view cannot be read', async () => { + const { service, listener } = offlineSetup(async () => { + throw new Error('timeout'); + }); + await service.triggerCollection('c1'); + expect(listener).toHaveBeenLastCalledWith(expect.objectContaining({ connectionId: 'c1' }), { partial: true }); + }); + + it.each([ + ['a replica', { id: 'n1', flags: ['slave'], slots: [] }], + ['a failed-over primary without slots', { id: 'n1', flags: ['master', 'fail'], slots: [] }], + ])('does not treat %s as unread', async (_label, node) => { + const { service, listener } = offlineSetup(async () => [node]); + const snapshot = await service.triggerCollection('c1'); + expect(snapshot).toMatchObject({ scanComplete: true }); + expect(listener).toHaveBeenLastCalledWith(expect.objectContaining({ connectionId: 'c1' }), { partial: false }); + }); + }); + + describe('partial reads with too few keys', () => { + function partialSetup() { + const ctx = setup([KEY(0)]); + const broken = fakeClient([]); + broken.getDbSize.mockRejectedValue(new Error('LOADING')); + jest.spyOn(ctx.service as any, 'nodesFor').mockResolvedValue({ + nodes: [ + { name: 'a', client: ctx.client }, + { name: 'b', client: broken }, + ], + skipped: 0, + }); + return ctx; + } + + it('keeps the previous detection instead of flipping to not detected', async () => { + const { service, storage } = partialSetup(); + storage.getKvCacheFootprintSnapshots.mockResolvedValue([{ connectionId: 'c1', detected: true }]); + const snapshot = await service.triggerCollection('c1'); + expect(snapshot).toMatchObject({ detected: true, scanComplete: false }); + expect(storage.getKvCacheFootprintSnapshots).toHaveBeenCalledWith({ connectionId: 'c1', limit: 1 }); + }); + + it('neither backs off polling nor switches the next scan to a probe', async () => { + const { service, storage, client } = partialSetup(); + await service.triggerCollection('c1'); + expect(storage.saveKvCacheFootprintSnapshot.mock.calls[0][0].detected).toBe(false); + expect((service as any).undetected.has('c1')).toBe(false); + client.getDbSize.mockClear(); + await (service as any).tick(); + expect(client.getDbSize).toHaveBeenCalledTimes(1); + }); + }); + it('fails the collection when every node fails', async () => { const { service, client } = setup(); client.getDbSize.mockRejectedValue(new Error('LOADING')); diff --git a/proprietary/kv-cache/kv-cache-footprint.service.ts b/proprietary/kv-cache/kv-cache-footprint.service.ts index 2f72b5b35..5491c8849 100644 --- a/proprietary/kv-cache/kv-cache-footprint.service.ts +++ b/proprietary/kv-cache/kv-cache-footprint.service.ts @@ -8,7 +8,7 @@ import { OnModuleInit, ServiceUnavailableException, } from '@nestjs/common'; -import { Feature, type KvCacheFootprintSnapshot } from '@betterdb/shared'; +import { Feature, type ConnectionStatus, type KvCacheFootprintSnapshot } from '@betterdb/shared'; import { MultiConnectionPoller, type ConnectionContext } from '@app/common/services/multi-connection-poller'; import type { StoragePort } from '@app/common/interfaces/storage-port.interface'; import { ConnectionRegistry } from '@app/connections/connection-registry.service'; @@ -269,16 +269,17 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On clientDetected: readings.some((reading) => reading.glide), }); const partial = failed !== undefined || skipped > 0; - const snapshot = partial ? { ...built, scanComplete: false } : built; + const detected = partial && !built.detected ? await this.previouslyDetected(ctx.connectionId) : built.detected; + const snapshot = partial ? { ...built, scanComplete: false, detected } : built; await this.deleting.get(ctx.connectionId); if (!current()) return null; await this.storage.saveKvCacheFootprintSnapshot(snapshot); if (!current()) return null; this.lastEvicted.set(ctx.connectionId, new Map([...(this.lastEvicted.get(ctx.connectionId) ?? []), ...evicted])); - if (snapshot.detected) { + if (built.detected) { this.undetectedUntil.delete(ctx.connectionId); this.undetected.delete(ctx.connectionId); - } else { + } else if (!partial) { this.undetectedUntil.set(ctx.connectionId, snapshot.timestamp + this.undetectedRecheckMs); this.undetected.add(ctx.connectionId); } @@ -299,7 +300,40 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On const registered = clusterScanNodes(this.connectionRegistry, seed, plan.members, this.logger, 'KV cache'); const { nodes, unreadable } = await readPrimaryScanNodes(registered, this.logger, 'KV cache'); if (nodes.length === 0) throw new Error(`No reachable primary found for ${ctx.connectionName}`); - return { nodes, skipped: 1 + plan.members.length - registered.length + unreadable }; + const offline = await this.offlinePrimaries(ctx.connectionId, nodes[0]); + return { nodes, skipped: 1 + plan.members.length - registered.length + unreadable + offline }; + } + + private async offlinePrimaries(connectionId: string, reachable: ScanNode): Promise { + const offline = this.connectionRegistry.list().filter((connection) => this.isOfflineMember(connection, connectionId)); + if (offline.length === 0) return 0; + try { + const owners = new Set( + (await reachable.client.getClusterNodes()) + .filter((node) => node.flags.includes('master') && node.slots.length > 0) + .map((node) => node.id), + ); + return offline.filter((connection) => owners.has(connection.membership?.nodeId ?? '')).length; + } catch (error) { + this.logger.warn(`KV cache could not read the cluster view from ${reachable.name}: ${errorMessage(error)}`); + return offline.length; + } + } + + private isOfflineMember(connection: ConnectionStatus, seedId: string): boolean { + const membership = connection.membership; + return ( + !connection.isConnected && + connection.connectionType !== 'external' && + membership?.seedId === seedId && + membership.source === 'cluster' && + membership.retiredAt === undefined + ); + } + + private async previouslyDetected(connectionId: string): Promise { + const [previous] = await this.storage.getKvCacheFootprintSnapshots({ connectionId, limit: 1 }); + return previous?.detected === true; } private async read(node: ScanNode, ownDb: number, probe: boolean): Promise { From b0facae99be92e3f4bfe5fabf85e3cbeae3b2ef1 Mon Sep 17 00:00:00 2001 From: jamby77 Date: Tue, 20 Oct 2026 15:20:01 +0300 Subject: [PATCH 18/31] Drop KV footprint runs for connections removed mid-collect --- .../kv-cache-footprint.service.spec.ts | 36 +++++++++++++++++++ .../kv-cache/kv-cache-footprint.service.ts | 8 +++-- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts b/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts index a933e2b97..b35a49508 100644 --- a/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts +++ b/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts @@ -497,6 +497,42 @@ describe('KvCacheFootprintService', () => { expect(order).toEqual(['delete', 'save']); }); + it('drops a poll that waited on the removal delete when the connection stays removed', async () => { + const { service, storage, client, registry } = setup(); + const listener = jest.fn(); + service.onSnapshot(listener); + let finishDelete: () => void = () => undefined; + storage.deleteKvCacheConnectionData.mockImplementationOnce( + () => new Promise((resolve) => (finishDelete = resolve)), + ); + registry.list.mockReturnValue([]); + registry.getConfig.mockReturnValue(null); + (service as any).onConnectionRemoved('c1'); + await new Promise((resolve) => setImmediate(resolve)); + expect(storage.deleteKvCacheConnectionData).toHaveBeenCalledWith('c1'); + const collecting = service.collect({ connectionId: 'c1', connectionName: 'one', client, host: 'h', port: 6379 }); + await new Promise((resolve) => setImmediate(resolve)); + finishDelete(); + await expect(collecting).resolves.toBeNull(); + expect(storage.saveKvCacheFootprintSnapshot).not.toHaveBeenCalled(); + expect(listener).not.toHaveBeenCalled(); + }); + + it('does not save or announce when the connection disappears mid-collect', async () => { + const { service, storage, client, registry } = setup(); + const listener = jest.fn(); + service.onSnapshot(listener); + let finishRead: (size: number) => void = () => undefined; + client.getDbSize.mockImplementationOnce(() => new Promise((resolve) => (finishRead = resolve))); + const collecting = service.collect({ connectionId: 'c1', connectionName: 'one', client, host: 'h', port: 6379 }); + await new Promise((resolve) => setImmediate(resolve)); + registry.getConfig.mockReturnValue(null); + finishRead(6); + await expect(collecting).resolves.toBeNull(); + expect(storage.saveKvCacheFootprintSnapshot).not.toHaveBeenCalled(); + expect(listener).not.toHaveBeenCalled(); + }); + it('deletes once for the latest removal when a connection is removed again before data is deleted', async () => { const { service, storage, registry } = setup(); registry.list.mockReturnValue([]); diff --git a/proprietary/kv-cache/kv-cache-footprint.service.ts b/proprietary/kv-cache/kv-cache-footprint.service.ts index 5491c8849..1f8b9aaac 100644 --- a/proprietary/kv-cache/kv-cache-footprint.service.ts +++ b/proprietary/kv-cache/kv-cache-footprint.service.ts @@ -223,7 +223,8 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On async collect(ctx: ConnectionContext): Promise { await this.deleting.get(ctx.connectionId); - if (this.connectionRegistry.getConfig(ctx.connectionId)?.host === 'agent') return null; + const config = this.connectionRegistry.getConfig(ctx.connectionId); + if (!config || config.host === 'agent') return null; const plan = this.planFor(ctx.connectionId); if (plan.kind === 'skip' || this.running.has(ctx.connectionId)) return null; const run = this.runCollection(ctx, plan); @@ -272,9 +273,10 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On const detected = partial && !built.detected ? await this.previouslyDetected(ctx.connectionId) : built.detected; const snapshot = partial ? { ...built, scanComplete: false, detected } : built; await this.deleting.get(ctx.connectionId); - if (!current()) return null; + const live = () => current() && Boolean(this.connectionRegistry.getConfig(ctx.connectionId)); + if (!live()) return null; await this.storage.saveKvCacheFootprintSnapshot(snapshot); - if (!current()) return null; + if (!live()) return null; this.lastEvicted.set(ctx.connectionId, new Map([...(this.lastEvicted.get(ctx.connectionId) ?? []), ...evicted])); if (built.detected) { this.undetectedUntil.delete(ctx.connectionId); From cbd30c296aa003f5f033e6e2a1b114652a389c15 Mon Sep 17 00:00:00 2001 From: jamby77 Date: Tue, 20 Oct 2026 20:15:02 +0300 Subject: [PATCH 19/31] Return the in-flight KV cache collection to concurrent callers --- .../kv-cache-footprint.service.spec.ts | 20 +++++++++++++++++++ .../kv-cache/kv-cache-footprint.service.ts | 4 +++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts b/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts index b42fb097a..8c0b1474b 100644 --- a/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts +++ b/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts @@ -78,6 +78,26 @@ describe('KvCacheFootprintService', () => { expect(storage.saveKvCacheFootprintSnapshot).toHaveBeenCalledTimes(1); }); + it('returns the in-flight collection to a trigger that starts in the same tick', async () => { + const { service, storage } = setup(); + const first = service.triggerCollection('c1'); + const second = service.triggerCollection('c1'); + const [a, b] = await Promise.all([first, second]); + expect(a).not.toBeNull(); + expect(b).toBe(a); + expect(storage.saveKvCacheFootprintSnapshot).toHaveBeenCalledTimes(1); + }); + + it('returns the in-flight collection to a poll that overlaps a manual trigger', async () => { + const { service, client } = setup(); + const ctx = { connectionId: 'c1', connectionName: 'one', client, host: 'h', port: 6379 }; + const manual = service.triggerCollection('c1'); + const polled = service.collect(ctx); + const [a, b] = await Promise.all([manual, polled]); + expect(a).not.toBeNull(); + expect(b).toBe(a); + }); + it('classifies each scanned key once per collection', async () => { const { service } = setup(); const classify = jest.spyOn(classifier, 'classifyKey'); diff --git a/proprietary/kv-cache/kv-cache-footprint.service.ts b/proprietary/kv-cache/kv-cache-footprint.service.ts index d1a17737b..26eb907fb 100644 --- a/proprietary/kv-cache/kv-cache-footprint.service.ts +++ b/proprietary/kv-cache/kv-cache-footprint.service.ts @@ -232,7 +232,9 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On const config = this.connectionRegistry.getConfig(ctx.connectionId); if (!config || config.host === 'agent') return null; const plan = planForConnection(this.connectionRegistry, ctx.connectionId); - if (plan.kind === 'skip' || this.running.has(ctx.connectionId)) return null; + if (plan.kind === 'skip') return null; + const running = this.running.get(ctx.connectionId); + if (running) return running; const run = this.runCollection(ctx, plan); this.running.set(ctx.connectionId, run); try { From f825c784c7604c2cb99f8c8ae623921aadadfa6b Mon Sep 17 00:00:00 2001 From: jamby77 Date: Tue, 20 Oct 2026 20:22:58 +0300 Subject: [PATCH 20/31] Keep the KV cache sample key on partial cluster reads --- .../kv-cache-footprint.service.spec.ts | 17 +++++++++++++++++ .../kv-cache/kv-cache-footprint.service.ts | 6 +++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts b/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts index 8c0b1474b..eaa27a32b 100644 --- a/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts +++ b/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts @@ -304,6 +304,23 @@ describe('KvCacheFootprintService', () => { expect(storage.getKvCacheFootprintSnapshots).toHaveBeenCalledWith({ connectionId: 'c1', limit: 1 }); }); + it('keeps the sample key when the readable nodes hold no LMCache keys', async () => { + const { service } = partialSetup(); + await service.triggerCollection('c1'); + expect(service.getSampleKey('c1')).toBe('m/x@1@0@1…@bfloat16'); + const broken = fakeClient([]); + broken.getDbSize.mockRejectedValue(new Error('LOADING')); + (service as any).nodesFor.mockResolvedValue({ + nodes: [ + { name: 'a', client: fakeClient([]) }, + { name: 'b', client: broken }, + ], + skipped: 0, + }); + await service.triggerCollection('c1'); + expect(service.getSampleKey('c1')).toBe('m/x@1@0@1…@bfloat16'); + }); + it('neither backs off polling nor switches the next scan to a probe', async () => { const { service, storage, client } = partialSetup(); await service.triggerCollection('c1'); diff --git a/proprietary/kv-cache/kv-cache-footprint.service.ts b/proprietary/kv-cache/kv-cache-footprint.service.ts index 26eb907fb..1908791d8 100644 --- a/proprietary/kv-cache/kv-cache-footprint.service.ts +++ b/proprietary/kv-cache/kv-cache-footprint.service.ts @@ -296,7 +296,7 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On this.undetectedUntil.set(ctx.connectionId, snapshot.timestamp + this.undetectedRecheckMs); this.undetected.add(ctx.connectionId); } - this.rememberSampleKey(ctx.connectionId, readings); + this.rememberSampleKey(ctx.connectionId, readings, partial); await this.announce(snapshot, { partial }); return snapshot; } @@ -384,10 +384,10 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On }; } - private rememberSampleKey(connectionId: string, readings: NodeReading[]): void { + private rememberSampleKey(connectionId: string, readings: NodeReading[], partial: boolean): void { const parsed = readings.flatMap((reading) => reading.observation.matchedKeys)[0]; if (parsed) this.sampleKeys.set(connectionId, maskKey(parsed)); - else this.sampleKeys.delete(connectionId); + else if (!partial) this.sampleKeys.delete(connectionId); } private async announce(snapshot: KvCacheFootprintSnapshot, context: SnapshotContext): Promise { From 8dc5a82c071112ddbaea33fd37e8503e07e59d2e Mon Sep 17 00:00:00 2001 From: jamby77 Date: Wed, 21 Oct 2026 01:52:58 +0300 Subject: [PATCH 21/31] Stub connection restore hook in engines service spec --- proprietary/kv-cache/__tests__/kv-cache-engines.service.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proprietary/kv-cache/__tests__/kv-cache-engines.service.spec.ts b/proprietary/kv-cache/__tests__/kv-cache-engines.service.spec.ts index 4b8c1e1bf..9fbd06516 100644 --- a/proprietary/kv-cache/__tests__/kv-cache-engines.service.spec.ts +++ b/proprietary/kv-cache/__tests__/kv-cache-engines.service.spec.ts @@ -262,7 +262,7 @@ describe('KvCacheEnginesService', () => { saveKvCacheEngine: jest.fn(async (e: any) => e), deleteKvCacheEngine: jest.fn(() => new Promise((resolve) => (finishDelete = () => resolve(true)))), } as any; - const footprint = { onConnectionRemoval: jest.fn() } as any; + const footprint = { onConnectionRemoval: jest.fn(), onConnectionRestore: jest.fn() } as any; const samples = { forgetEngine: jest.fn(), idle: jest.fn(), deleteEngine: jest.fn().mockResolvedValue(undefined), resetBaselines: jest.fn() } as any; const registry = new KvCacheEngineRegistry(storage, footprint, samples); await registry.onModuleInit(); From 704f62ee2630fcc9e0b665cdc06c0e118e77a632 Mon Sep 17 00:00:00 2001 From: jamby77 Date: Wed, 21 Oct 2026 05:52:37 +0300 Subject: [PATCH 22/31] Fail the node read when every sample command for a key is rejected --- .../__tests__/footprint-scanner.spec.ts | 22 +++++++++++++++++++ .../kv-cache-footprint.service.spec.ts | 21 ++++++++++++++++++ proprietary/kv-cache/footprint-scanner.ts | 5 +++-- 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/proprietary/kv-cache/__tests__/footprint-scanner.spec.ts b/proprietary/kv-cache/__tests__/footprint-scanner.spec.ts index 0ee31aca3..a97151946 100644 --- a/proprietary/kv-cache/__tests__/footprint-scanner.spec.ts +++ b/proprietary/kv-cache/__tests__/footprint-scanner.spec.ts @@ -114,6 +114,28 @@ describe('sampleLmcacheKeys', () => { expect(sample.memoryBytes).toBeNull(); }); + it('fails the read when every command for a key is rejected', async () => { + const closed = new Error('Connection is closed.'); + const client = sampleClient({ + [`MEMORY USAGE ${k(0)}`]: 120, + [`TTL ${k(0)}`]: 50, + [`MEMORY USAGE ${k(1, 'kv_bytes')}`]: closed, + [`TTL ${k(1, 'kv_bytes')}`]: closed, + [`EXISTS ${k(1, 'metadata')}`]: closed, + }); + await expect(sampleLmcacheKeys(client, p(k(0), k(1, 'kv_bytes')))).rejects.toThrow('Connection is closed.'); + }); + + it('keeps the ttl of a key whose MEMORY and sibling probes are rejected', async () => { + const client = sampleClient({ + [`MEMORY USAGE ${k(0, 'kv_bytes')}`]: new Error("ERR unknown command 'MEMORY'"), + [`TTL ${k(0, 'kv_bytes')}`]: 30, + [`EXISTS ${k(0, 'metadata')}`]: new Error('MOVED 1234 10.0.0.2:6379'), + }); + const [sample] = await sampleLmcacheKeys(client, p(k(0, 'kv_bytes'))); + expect(sample).toMatchObject({ memoryBytes: null, ttl: 30, siblingExists: null }); + }); + it('issues no commands for no keys', async () => { const client = sampleClient({}); expect(await sampleLmcacheKeys(client, [])).toEqual([]); diff --git a/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts b/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts index eaa27a32b..f73944526 100644 --- a/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts +++ b/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts @@ -196,6 +196,27 @@ describe('KvCacheFootprintService', () => { expect(listener).toHaveBeenLastCalledWith(expect.objectContaining({ connectionId: 'c1' }), { partial: false }); }); + it('marks the snapshot partial when a node drops its connection while sampling', async () => { + const { service, client } = setup(); + const listener = jest.fn(); + service.onSnapshot(listener); + const dropped = fakeClient(Array.from({ length: 6 }, (_, i) => KEY(i + 10))); + dropped.call.mockImplementation(async (cmd: string) => { + if (cmd === 'SCAN') return ['0', Array.from({ length: 6 }, (_, i) => KEY(i + 10))]; + throw new Error('Connection is closed.'); + }); + jest.spyOn(service as any, 'nodesFor').mockResolvedValue({ + nodes: [ + { name: 'a', client }, + { name: 'b', client: dropped }, + ], + skipped: 0, + }); + const snapshot = await service.triggerCollection('c1'); + expect(snapshot).toMatchObject({ matchedKeys: 6, scanComplete: false }); + expect(listener).toHaveBeenLastCalledWith(expect.objectContaining({ connectionId: 'c1' }), { partial: true }); + }); + describe('cluster members that were never read', () => { function clusterSetup(member: 'deregistered' | 'role-failed' | 'replica') { const ctx = setup(); diff --git a/proprietary/kv-cache/footprint-scanner.ts b/proprietary/kv-cache/footprint-scanner.ts index 22b38615a..134c96ea8 100644 --- a/proprietary/kv-cache/footprint-scanner.ts +++ b/proprietary/kv-cache/footprint-scanner.ts @@ -50,12 +50,12 @@ export async function scanLmcacheKeys( }; } -type Settled = { ok: true; value: unknown } | { ok: false }; +type Settled = { ok: true; value: unknown } | { ok: false; error: unknown }; const settle = (promise: Promise): Promise => promise.then( (value) => ({ ok: true as const, value }), - () => ({ ok: false as const }), + (error: unknown) => ({ ok: false as const, error }), ); async function sampleKey(client: DatabasePort, parsed: ParsedLmcacheKey): Promise { @@ -66,6 +66,7 @@ async function sampleKey(client: DatabasePort, parsed: ParsedLmcacheKey): Promis settle(client.call('TTL', [key])), sibling ? settle(client.call('EXISTS', [sibling])) : Promise.resolve(null), ]); + if (!memory.ok && !ttl.ok && exists?.ok !== true) throw ttl.error; return { key, parsed, From 8fa8378008a2ae9bfdade21e4a229e137cbb1d53 Mon Sep 17 00:00:00 2001 From: jamby77 Date: Wed, 21 Oct 2026 10:18:57 +0300 Subject: [PATCH 23/31] Mock SQL sample totals in truncated KV cache sample specs --- .../kv-cache/__tests__/kv-cache-samples.service.spec.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/proprietary/kv-cache/__tests__/kv-cache-samples.service.spec.ts b/proprietary/kv-cache/__tests__/kv-cache-samples.service.spec.ts index c3e51d3bd..1c9728861 100644 --- a/proprietary/kv-cache/__tests__/kv-cache-samples.service.spec.ts +++ b/proprietary/kv-cache/__tests__/kv-cache-samples.service.spec.ts @@ -195,22 +195,24 @@ describe('KvCacheSamplesService', () => { const boundary = [sample({ timestamp: 0, requestedTokens: 1, hitTokens: 1 }), sample({ timestamp: 0, engineId: 'e2', requestedTokens: 1 })]; const newer = Array.from({ length: SAMPLES_LIMIT - 1 }, (_, i) => sample({ timestamp: Math.floor(i / 2) + 1, engineId: i % 2 ? 'e2' : 'e1', requestedTokens: 2, hitTokens: 1 })); storage.getKvCacheEngineSamples.mockResolvedValue([...boundary, ...newer]); + storage.getKvCacheSampleTotals.mockResolvedValue({ requestedTokens: 4, hitTokens: 3 }); const result = await service.getSamples('c1', { from: 0, to: 1_000_000 }); expect(result.truncated).toBe(true); expect(result.buckets.some((bucket) => bucket.timestamp === 0)).toBe(false); expect(result.buckets[0]).toMatchObject({ timestamp: 1, requestedTokens: 2 }); - expect(result.rangeHitRate).toBe(0.5); + expect(result.rangeHitRate).toBe(0.75); }); it('keeps the newest rows when every fetched row shares the oldest timestamp', async () => { const { service, storage } = setup(); const rows = Array.from({ length: SAMPLES_LIMIT + 1 }, (_, i) => sample({ timestamp: 5, engineId: `e${i}`, requestedTokens: 2, hitTokens: 1 })); storage.getKvCacheEngineSamples.mockResolvedValue(rows); + storage.getKvCacheSampleTotals.mockResolvedValue({ requestedTokens: 4, hitTokens: 3 }); const result = await service.getSamples('c1', { from: 0, to: 1000 }); expect(result.truncated).toBe(true); expect(result.buckets).toHaveLength(SAMPLES_LIMIT); expect(result.buckets.some((bucket) => bucket.engineId === 'e0')).toBe(false); - expect(result.rangeHitRate).toBe(0.5); + expect(result.rangeHitRate).toBe(0.75); }); it('returns a null range hit rate when nothing was requested', async () => { From 46297e70989471d27af408ca1d880032e012937e Mon Sep 17 00:00:00 2001 From: jamby77 Date: Wed, 21 Oct 2026 10:19:00 +0300 Subject: [PATCH 24/31] Allow KV engine links once a removed connection registers again - Skip the removed-connection tombstone when the connection is registered - Drop a save only when a removal landed while it was in flight --- .../__tests__/kv-cache-engine-registry.spec.ts | 12 ++++++++---- proprietary/kv-cache/kv-cache-engine-registry.ts | 9 ++++++--- proprietary/kv-cache/kv-cache-footprint.service.ts | 4 ++++ 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/proprietary/kv-cache/__tests__/kv-cache-engine-registry.spec.ts b/proprietary/kv-cache/__tests__/kv-cache-engine-registry.spec.ts index 41f2f1c28..1b34dcce8 100644 --- a/proprietary/kv-cache/__tests__/kv-cache-engine-registry.spec.ts +++ b/proprietary/kv-cache/__tests__/kv-cache-engine-registry.spec.ts @@ -17,10 +17,11 @@ function setup(initial = [engine('a', 'c1', 'lmc-a'), engine('b', 'c2')]) { const footprint = { onConnectionRemoval: jest.fn((l) => (removal = l)), onConnectionRestore: jest.fn((l) => (restore = l)), + hasConnection: jest.fn().mockReturnValue(false), } as any; const samples = { forgetEngine: jest.fn(), settled: jest.fn().mockResolvedValue(undefined) } as any; const registry = new KvCacheEngineRegistry(storage, footprint, samples); - return { registry, storage, samples, removal: (id: string) => removal(id), restore: (id: string) => restore(id) }; + return { registry, storage, samples, footprint, removal: (id: string) => removal(id), restore: (id: string) => restore(id) }; } describe('KvCacheEngineRegistry', () => { @@ -283,9 +284,10 @@ describe('KvCacheEngineRegistry', () => { }); it('keeps a link created during cleanup and deletes the stored engine it supersedes', async () => { - const { registry, storage, removal, restore } = setup([engine('a', 'c1', 'lmc-a'), engine('old', 'c1')]); + const { registry, storage, footprint, removal, restore } = setup([engine('a', 'c1', 'lmc-a'), engine('old', 'c1')]); await registry.onModuleInit(); await removal('c1'); + footprint.hasConnection.mockReturnValue(true); await registry.save({ ...engine('new', 'c1', 'lmc-a'), name: 'fresh' }); await registry.save({ ...engine('dup', 'c1'), name: 'old' }); await restore('c1'); @@ -295,9 +297,10 @@ describe('KvCacheEngineRegistry', () => { }); it('does not reload engines after a removal that lands while a superseded engine is deleted', async () => { - const { registry, storage, removal, restore } = setup([engine('a', 'c1', 'lmc-a'), engine('b', 'c1')]); + const { registry, storage, footprint, removal, restore } = setup([engine('a', 'c1', 'lmc-a'), engine('b', 'c1')]); await registry.onModuleInit(); await removal('c1'); + footprint.hasConnection.mockReturnValue(true); await registry.save(engine('new', 'c1', 'lmc-a')); let finishDelete: () => void = () => undefined; storage.deleteKvCacheEngine.mockImplementationOnce(() => new Promise((resolve) => (finishDelete = () => resolve(true)))); @@ -310,9 +313,10 @@ describe('KvCacheEngineRegistry', () => { }); it('skips deleting a superseded engine once the connection is removed again', async () => { - const { registry, storage, removal, restore } = setup([engine('a', 'c1', 'lmc-a')]); + const { registry, storage, footprint, removal, restore } = setup([engine('a', 'c1', 'lmc-a')]); await registry.onModuleInit(); await removal('c1'); + footprint.hasConnection.mockReturnValue(true); await registry.save(engine('new', 'c1', 'lmc-a')); let finishPending: () => void = () => undefined; storage.deleteKvCacheEngine.mockImplementationOnce(() => new Promise((resolve) => (finishPending = () => resolve(true)))); diff --git a/proprietary/kv-cache/kv-cache-engine-registry.ts b/proprietary/kv-cache/kv-cache-engine-registry.ts index 7ff2d550d..f2a3aa8c7 100644 --- a/proprietary/kv-cache/kv-cache-engine-registry.ts +++ b/proprietary/kv-cache/kv-cache-engine-registry.ts @@ -102,11 +102,12 @@ export class KvCacheEngineRegistry implements OnModuleInit { return this.serialize(engine.id, async () => { if (this.removed.has(engine.id)) throw new NotFoundException(`KV cache engine ${engine.id} not found`); this.assertConnectionLive(engine.connectionId); + const removals = this.removals.get(engine.connectionId); const saved = await this.storage.saveKvCacheEngine(engine); if (this.removed.has(saved.id)) return saved; - if (this.removedConnections.has(saved.connectionId)) { + if (this.removals.get(saved.connectionId) !== removals) { await this.storage.deleteKvCacheEngine(saved.id); - this.assertConnectionLive(saved.connectionId); + throw new NotFoundException(`Connection '${saved.connectionId}' not found`); } const latest = this.engines.get(saved.id); const merged = @@ -170,7 +171,9 @@ export class KvCacheEngineRegistry implements OnModuleInit { } private assertConnectionLive(connectionId: string): void { - if (this.removedConnections.has(connectionId)) throw new NotFoundException(`Connection '${connectionId}' not found`); + if (this.removedConnections.has(connectionId) && !this.footprint.hasConnection(connectionId)) { + throw new NotFoundException(`Connection '${connectionId}' not found`); + } } private serialize(id: string, task: () => Promise): Promise { diff --git a/proprietary/kv-cache/kv-cache-footprint.service.ts b/proprietary/kv-cache/kv-cache-footprint.service.ts index 812c869c6..35fea8fd4 100644 --- a/proprietary/kv-cache/kv-cache-footprint.service.ts +++ b/proprietary/kv-cache/kv-cache-footprint.service.ts @@ -197,6 +197,10 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On return this.connectionRegistry.list({ includeRetired: true }).some((connection) => connection.id === connectionId); } + hasConnection(connectionId: string): boolean { + return Boolean(this.connectionRegistry.getConfig(connectionId)); + } + onConnectionRemoval(listener: RemovalListener): void { this.removalListeners.push(listener); } From bb74ba5df8ba6f436ae407996b13e192f0ee25ef Mon Sep 17 00:00:00 2001 From: jamby77 Date: Wed, 21 Oct 2026 10:19:08 +0300 Subject: [PATCH 25/31] Document the KV cache matched-key cap floor of 5 --- docs/kv-cache-monitoring.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/kv-cache-monitoring.md b/docs/kv-cache-monitoring.md index c972528f0..a36efe886 100644 --- a/docs/kv-cache-monitoring.md +++ b/docs/kv-cache-monitoring.md @@ -131,8 +131,8 @@ a bounded SCAN plus a sample, extrapolated to the whole keyspace. | `KV_CACHE_SAMPLE_KEYS` | `500` | Number of matched keys sampled with `MEMORY USAGE` and `TTL` | An unset, non-numeric or too-small value falls back to the default. The -interval must be at least `60000` (1 minute), and each key budget must be at -least `1`. +interval must be at least `60000` (1 minute), the matched-key cap at least +`5` (the detection threshold), and the other key budgets at least `1`. Per-model chunks and bytes come from the same sample, split in proportion to what the sample contains. The share of keys without a TTL and the LMCache @@ -337,7 +337,7 @@ label is the engine name, which must be unique per connection. | -------------------------------- | -------- | ---------------------------------------------------------------------------------------------- | | `KV_CACHE_FOOTPRINT_INTERVAL_MS` | `300000` | Footprint collection interval. An invalid value or one below `60000` falls back to the default | | `KV_CACHE_SCAN_MAX_KEYS` | `200000` | SCAN budget per collection. An invalid value or one below `1` falls back to the default | -| `KV_CACHE_MATCH_MAX_KEYS` | `2000` | Matched-key cap per collection. An invalid value or one below `1` falls back to the default | +| `KV_CACHE_MATCH_MAX_KEYS` | `2000` | Matched-key cap per collection. An invalid value or one below `5` falls back to the default | | `KV_CACHE_SAMPLE_KEYS` | `500` | Keys sampled for size and TTL. An invalid value or one below `1` falls back to the default | | `KV_CACHE_SCRAPE_INTERVAL_MS` | `30000` | Engine scrape interval. An invalid or sub-second value falls back to the default | | `KV_CACHE_SCRAPE_BLOCK_PRIVATE` | `false` | `true` also refuses private and loopback scrape targets | From d606e2a5da23133bf439857cc9ebbeb871bc8bc2 Mon Sep 17 00:00:00 2001 From: jamby77 Date: Wed, 21 Oct 2026 10:48:20 +0300 Subject: [PATCH 26/31] Run KV removal listeners after an in-flight snapshot announce - Track the announce of each collection per connection - Removal waits for that announce before notifying listeners, so alerts and Prometheus clear state after the last snapshot of the old generation instead of racing it --- .../kv-cache-footprint.service.spec.ts | 24 +++++++++++++ .../kv-cache/kv-cache-footprint.service.ts | 34 ++++++++++++++----- 2 files changed, 49 insertions(+), 9 deletions(-) diff --git a/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts b/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts index 41d959e34..7b46de7a5 100644 --- a/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts +++ b/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts @@ -741,6 +741,30 @@ describe('KvCacheFootprintService', () => { expect(listener).toHaveBeenCalledWith('c1'); }); + it('runs removal listeners only after an in-flight snapshot announce finishes', async () => { + const { service, registry } = setup(); + const order: string[] = []; + let release: () => void = () => undefined; + service.onSnapshot(async () => { + order.push('announce:start'); + await new Promise((resolve) => (release = resolve)); + order.push('announce:end'); + }); + service.onConnectionRemoval(() => { + order.push('removed'); + }); + const collecting = service.triggerCollection('c1'); + while (!order.includes('announce:start')) await new Promise((resolve) => setImmediate(resolve)); + registry.list.mockReturnValue([]); + (service as any).forget('c1'); + await new Promise((resolve) => setImmediate(resolve)); + expect(order).toEqual(['announce:start']); + release(); + await collecting; + await new Promise((resolve) => setImmediate(resolve)); + expect(order).toEqual(['announce:start', 'announce:end', 'removed']); + }); + it('deletes data for a connection removed from the registry before the poller saw it', async () => { const { service, storage, registry } = setup(); jest.spyOn(service as any, 'start').mockImplementation(() => undefined); diff --git a/proprietary/kv-cache/kv-cache-footprint.service.ts b/proprietary/kv-cache/kv-cache-footprint.service.ts index 35fea8fd4..888b15c3e 100644 --- a/proprietary/kv-cache/kv-cache-footprint.service.ts +++ b/proprietary/kv-cache/kv-cache-footprint.service.ts @@ -78,6 +78,7 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On private readonly running = new Map>(); private readonly generations = new Map(); private readonly deleting = new Map>(); + private readonly announcing = new Map>(); private readonly lastEvicted = new Map>(); private readonly sampleKeys = new Map(); private readonly undetectedUntil = new Map(); @@ -145,18 +146,27 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On () => undefined, () => undefined, ); - const pending = this.removalListeners.map(async (listener) => { - try { - await listener(connectionId); - } catch (error) { - this.logger.warn(`KV cache removal listener failed: ${errorMessage(error)}`); - } - }); - void Promise.all([...pending, settled]) + const announcing = this.announcing.get(connectionId); + const removed = announcing + ? announcing.then(() => this.announceRemoval(connectionId)) + : this.announceRemoval(connectionId); + void Promise.all([removed, settled]) .then(() => this.deleteRemovedData(connectionId, generation)) .catch((error) => this.logger.warn(`Could not delete KV cache data for ${connectionId}: ${errorMessage(error)}`)); } + private async announceRemoval(connectionId: string): Promise { + await Promise.all( + this.removalListeners.map(async (listener) => { + try { + await listener(connectionId); + } catch (error) { + this.logger.warn(`KV cache removal listener failed: ${errorMessage(error)}`); + } + }), + ); + } + private sweepDeletedRetirees(): void { const all = this.connectionRegistry.list({ includeRetired: true }).map((connection) => connection.id); const live = new Set(this.connectionRegistry.list().map((connection) => connection.id)); @@ -315,7 +325,13 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On this.undetected.add(ctx.connectionId); } this.rememberSampleKey(ctx.connectionId, readings, partial); - await this.announce(snapshot, { partial }); + const announced = this.announce(snapshot, { partial }); + this.announcing.set(ctx.connectionId, announced); + try { + await announced; + } finally { + if (this.announcing.get(ctx.connectionId) === announced) this.announcing.delete(ctx.connectionId); + } return snapshot; } From 9e8445416a5b911a2483f57659ed4d07a4873993 Mon Sep 17 00:00:00 2001 From: jamby77 Date: Wed, 21 Oct 2026 11:31:36 +0300 Subject: [PATCH 27/31] Say KV byte estimates read as unknown when MEMORY USAGE is blocked --- proprietary/kv-cache/kv-cache-footprint.service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proprietary/kv-cache/kv-cache-footprint.service.ts b/proprietary/kv-cache/kv-cache-footprint.service.ts index 67305bbc9..0cd15db10 100644 --- a/proprietary/kv-cache/kv-cache-footprint.service.ts +++ b/proprietary/kv-cache/kv-cache-footprint.service.ts @@ -455,7 +455,7 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On } if (this.memoryBlocked.has(nodeName)) return; this.memoryBlocked.add(nodeName); - this.logger.warn(`MEMORY USAGE returned nothing on ${nodeName}; KV cache byte estimates will read as zero`); + this.logger.warn(`MEMORY USAGE returned nothing on ${nodeName}; KV cache byte estimates will read as unknown`); } private sumMemory(stats: MemoryStats[]): MemoryStats { From e66a24efdc5d4b98c6b503762134183edeb2a9c9 Mon Sep 17 00:00:00 2001 From: jamby77 Date: Wed, 21 Oct 2026 12:44:13 +0300 Subject: [PATCH 28/31] Hold the unevictable alert on partial KV footprint snapshots - Partial snapshots reuse the last complete unevictable state, like evicting - Forget it on connection removal, disabled alerts or no LMCache detected --- .../__tests__/kv-cache-alerts.service.spec.ts | 37 +++++++++++++++++++ .../kv-cache/kv-cache-alerts.service.ts | 15 ++++++-- 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/proprietary/kv-cache/__tests__/kv-cache-alerts.service.spec.ts b/proprietary/kv-cache/__tests__/kv-cache-alerts.service.spec.ts index e57240cae..355b02bc9 100644 --- a/proprietary/kv-cache/__tests__/kv-cache-alerts.service.spec.ts +++ b/proprietary/kv-cache/__tests__/kv-cache-alerts.service.spec.ts @@ -153,6 +153,43 @@ describe('KvCacheAlertsService eviction risk', () => { expect(pro.dispatchKvCacheEvictionRisk).toHaveBeenLastCalledWith(expect.objectContaining({ reason: 'evicting', active: true })); }); + it('keeps the unevictable state when a snapshot only read some nodes', async () => { + const { service, pro } = setup(); + const unevictableStates = () => + pro.dispatchKvCacheEvictionRisk.mock.calls.filter(([e]) => e.reason === 'unevictable').map(([e]) => e.active); + await service.onSnapshot(snapshot({ noTtlRatio: 0.95 }), { partial: true }); + await service.onSnapshot(snapshot({ noTtlRatio: 0.95 })); + await service.onSnapshot(snapshot({ noTtlRatio: 0.1 }), { partial: true }); + await service.onSnapshot(snapshot({ noTtlRatio: null }), { partial: true }); + await service.onSnapshot(snapshot({ detected: false }), { partial: true }); + await service.onSnapshot(snapshot({ noTtlRatio: 0.1 })); + await service.onSnapshot(snapshot({ noTtlRatio: 0.95 }), { partial: true }); + expect(unevictableStates()).toEqual([false, true, true, true, true, false, false]); + }); + + it('forgets the unevictable state when the connection is removed or alerts are disabled', async () => { + const { service, pro, remove, storage } = setup(); + const lastUnevictable = () => + pro.dispatchKvCacheEvictionRisk.mock.calls.filter(([e]) => e.reason === 'unevictable').at(-1)?.[0].active; + service.onModuleInit(); + await service.onSnapshot(snapshot()); + remove('c1'); + await service.onSnapshot(snapshot(), { partial: true }); + expect(lastUnevictable()).toBe(false); + await service.onSnapshot(snapshot()); + storage.getKvCacheSettings.mockResolvedValueOnce({ + connectionId: 'c1', + hitRateAlertEnabled: true, + hitRateThreshold: 0.2, + evictionAlertEnabled: false, + updatedAt: 1, + }); + await service.onSnapshot(snapshot()); + await service.onSnapshot(snapshot(), { partial: true }); + expect(lastUnevictable()).toBe(false); + service.onModuleDestroy(); + }); + it('passes the partial flag from footprint snapshots through', async () => { const { service, pro, footprint } = setup(); service.onModuleInit(); diff --git a/proprietary/kv-cache/kv-cache-alerts.service.ts b/proprietary/kv-cache/kv-cache-alerts.service.ts index 3aa9666c0..7f80cd4bf 100644 --- a/proprietary/kv-cache/kv-cache-alerts.service.ts +++ b/proprietary/kv-cache/kv-cache-alerts.service.ts @@ -35,6 +35,7 @@ export class KvCacheAlertsService implements OnModuleInit, OnModuleDestroy { private readonly exportedConnections = new Set(); private readonly checkedWindows = new Map>(); private readonly evictingCleanStreaks = new Map(); + private readonly unevictableActive = new Set(); constructor( @Inject('STORAGE_CLIENT') private readonly storage: StoragePort, @@ -54,6 +55,7 @@ export class KvCacheAlertsService implements OnModuleInit, OnModuleDestroy { this.exportedConnections.delete(connectionId); this.checkedWindows.delete(connectionId); this.evictingCleanStreaks.delete(connectionId); + this.unevictableActive.delete(connectionId); this.prometheus?.clearKvCache(connectionId); }); this.timer = setInterval(() => void this.evaluateHitRates(), EVALUATE_INTERVAL_MS); @@ -119,12 +121,19 @@ export class KvCacheAlertsService implements OnModuleInit, OnModuleDestroy { ): Record { if (!enabled || (!partial && !snapshot.detected)) { this.evictingCleanStreaks.delete(snapshot.connectionId); + this.unevictableActive.delete(snapshot.connectionId); return { unevictable: false, evicting: false }; } + if (partial) { + return { + unevictable: this.unevictableActive.has(snapshot.connectionId), + evicting: this.evictingCleanStreaks.has(snapshot.connectionId), + }; + } const conditions = evictionConditions(snapshot); - conditions.evicting = partial - ? this.evictingCleanStreaks.has(snapshot.connectionId) - : this.holdEvicting(snapshot.connectionId, conditions.evicting); + if (conditions.unevictable) this.unevictableActive.add(snapshot.connectionId); + else this.unevictableActive.delete(snapshot.connectionId); + conditions.evicting = this.holdEvicting(snapshot.connectionId, conditions.evicting); return conditions; } From d00ca4aa10706ba3b3ca3be49cd422ffa1e26392 Mon Sep 17 00:00:00 2001 From: jamby77 Date: Wed, 21 Oct 2026 15:19:45 +0300 Subject: [PATCH 29/31] Hold new KV cache saves until pending removal listeners finish --- .../kv-cache-footprint.service.spec.ts | 30 +++++++++++++++++++ .../kv-cache/kv-cache-footprint.service.ts | 10 +++++++ 2 files changed, 40 insertions(+) diff --git a/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts b/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts index 6956fe526..9c1398ebe 100644 --- a/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts +++ b/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts @@ -1000,6 +1000,36 @@ describe('KvCacheFootprintService', () => { expect(order).toEqual(['announce:start', 'announce:end', 'removed']); }); + it('keeps the snapshot of a re-registered connection when the older announce finishes after it', async () => { + const { service } = setup(); + const order: string[] = []; + let release: () => void = () => undefined; + let calls = 0; + service.onSnapshot(async () => { + calls += 1; + if (calls > 1) { + order.push('snapshot:new'); + return; + } + order.push('announce:start'); + await new Promise((resolve) => (release = resolve)); + order.push('announce:end'); + }); + service.onConnectionRemoval(() => { + order.push('removed'); + }); + const stale = service.triggerCollection('c1'); + while (!order.includes('announce:start')) await new Promise((resolve) => setImmediate(resolve)); + (service as any).forget('c1'); + const fresh = service.triggerCollection('c1'); + await new Promise((resolve) => setImmediate(resolve)); + release(); + await stale; + await expect(fresh).resolves.not.toBeNull(); + await new Promise((resolve) => setImmediate(resolve)); + expect(order).toEqual(['announce:start', 'announce:end', 'removed', 'snapshot:new']); + }); + it('deletes data for a connection removed from the registry before the poller saw it', async () => { const { service, storage, registry } = setup(); jest.spyOn(service as any, 'start').mockImplementation(() => undefined); diff --git a/proprietary/kv-cache/kv-cache-footprint.service.ts b/proprietary/kv-cache/kv-cache-footprint.service.ts index 261660ed1..92f9e37db 100644 --- a/proprietary/kv-cache/kv-cache-footprint.service.ts +++ b/proprietary/kv-cache/kv-cache-footprint.service.ts @@ -102,6 +102,7 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On private readonly running = new Map>(); private readonly generations = new Map(); private readonly deleting = new Map>(); + private readonly removing = new Map>(); private readonly announcing = new Map>(); private readonly lastEvicted = new Map>(); private readonly sampleKeys = new Map(); @@ -179,6 +180,14 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On const removed = announcing ? announcing.then(() => this.announceRemoval(connectionId)) : this.announceRemoval(connectionId); + const removal = removed.then( + () => undefined, + () => undefined, + ); + this.removing.set(connectionId, removal); + void removal.then(() => { + if (this.removing.get(connectionId) === removal) this.removing.delete(connectionId); + }); void Promise.all([removed, settled]) .then(() => this.deleteRemovedData(connectionId, generation)) .catch((error) => this.logger.warn(`Could not delete KV cache data for ${connectionId}: ${errorMessage(error)}`)); @@ -352,6 +361,7 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On const detected = partial && !built.detected ? await this.previouslyDetected(ctx.connectionId) : built.detected; const snapshot = partial ? { ...built, scanComplete: false, detected } : built; await this.deleting.get(ctx.connectionId); + if (current()) await this.removing.get(ctx.connectionId); const live = () => current() && Boolean(this.connectionRegistry.getConfig(ctx.connectionId)); if (!live()) return null; await this.storage.saveKvCacheFootprintSnapshot(snapshot); From dc3d5520b9a18d17d0abfa5628540dbe49caae16 Mon Sep 17 00:00:00 2001 From: jamby77 Date: Wed, 21 Oct 2026 17:12:13 +0300 Subject: [PATCH 30/31] Update KV partial miss counts only for live collections --- .../kv-cache-footprint.service.spec.ts | 19 +++++++++++++++++++ .../kv-cache/kv-cache-footprint.service.ts | 17 +++++++---------- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts b/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts index 9c44ddc72..600d480e9 100644 --- a/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts +++ b/proprietary/kv-cache/__tests__/kv-cache-footprint.service.spec.ts @@ -828,6 +828,25 @@ describe('KvCacheFootprintService', () => { return ctx; } + it('does not let a discarded run touch the miss count of a re-registered connection', async () => { + const { service, storage, client, registry } = partialSetup(); + storage.getKvCacheFootprintSnapshots.mockResolvedValue([{ connectionId: 'c1', detected: true }]); + const listed = registry.list(); + let finishRead: (size: number) => void = () => undefined; + client.getDbSize.mockImplementationOnce(() => new Promise((resolve) => (finishRead = resolve))); + const stale = service.triggerCollection('c1'); + await new Promise((resolve) => setImmediate(resolve)); + registry.list.mockReturnValue([]); + (service as any).forget('c1'); + registry.list.mockReturnValue(listed); + await new Promise((resolve) => setImmediate(resolve)); + finishRead(0); + await expect(stale).resolves.toBeNull(); + for (let i = 0; i < PARTIAL_MISSES_BEFORE_CLEAR; i += 1) await service.triggerCollection('c1'); + const saved = storage.saveKvCacheFootprintSnapshot.mock.calls.map(([snapshot]: [any]) => snapshot.detected); + expect(saved).toEqual([true, true, false]); + }); + it('keeps polling when slot owners were left unscanned and nothing was detected', async () => { const { service, storage } = partialSetup(); await (service as any).tick(); diff --git a/proprietary/kv-cache/kv-cache-footprint.service.ts b/proprietary/kv-cache/kv-cache-footprint.service.ts index ec5e556c3..b34dff921 100644 --- a/proprietary/kv-cache/kv-cache-footprint.service.ts +++ b/proprietary/kv-cache/kv-cache-footprint.service.ts @@ -405,8 +405,9 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On coverage: this.coverage(targets, readIndexes), }); const partial = failed !== undefined || skipped > 0; - const detected = partial && !built.detected ? await this.carriedDetection(ctx.connectionId) : built.detected; - if (!partial || built.detected) this.partialMisses.delete(ctx.connectionId); + const missed = partial && !built.detected; + const misses = missed ? (this.partialMisses.get(ctx.connectionId) ?? 0) + 1 : 0; + const detected = missed ? await this.carriedDetection(ctx.connectionId, misses) : built.detected; const keepsEarlier = partial && detected; const snapshot = partial ? { ...built, scanComplete: false, detected } : built; await this.deleting.get(ctx.connectionId); @@ -416,10 +417,12 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On await this.storage.saveKvCacheFootprintSnapshot(snapshot); if (!live()) return null; this.lastEvicted.set(ctx.connectionId, evicted); + if (missed) this.partialMisses.set(ctx.connectionId, misses); + else this.partialMisses.delete(ctx.connectionId); if (built.detected) { this.undetectedUntil.delete(ctx.connectionId); this.undetected.delete(ctx.connectionId); - } else if (!snapshot.detected && (!partial || this.partialMissesExhausted(ctx.connectionId))) { + } else if (!snapshot.detected && (!partial || misses >= PARTIAL_MISSES_BEFORE_CLEAR)) { this.undetectedUntil.set(ctx.connectionId, snapshot.timestamp + this.undetectedRecheckMs); this.undetected.add(ctx.connectionId); } @@ -527,9 +530,7 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On ); } - private async carriedDetection(connectionId: string): Promise { - const misses = (this.partialMisses.get(connectionId) ?? 0) + 1; - this.partialMisses.set(connectionId, misses); + private async carriedDetection(connectionId: string, misses: number): Promise { if (misses >= PARTIAL_MISSES_BEFORE_CLEAR) return false; const [previous] = await this.storage.getKvCacheFootprintSnapshots({ connectionId, limit: 1 }); return previous?.detected === true; @@ -592,10 +593,6 @@ export class KvCacheFootprintService extends MultiConnectionPoller implements On }; } - private partialMissesExhausted(connectionId: string): boolean { - return (this.partialMisses.get(connectionId) ?? 0) >= PARTIAL_MISSES_BEFORE_CLEAR; - } - private rememberSampleKey(connectionId: string, readings: NodeReading[], keepsEarlier: boolean): void { const parsed = readings.flatMap((reading) => reading.observation.matchedKeys)[0]; if (parsed) this.sampleKeys.set(connectionId, maskKey(parsed)); From b13507ea68b8398dd57a4147ca952301a01f12d5 Mon Sep 17 00:00:00 2001 From: jamby77 Date: Wed, 21 Oct 2026 17:32:24 +0300 Subject: [PATCH 31/31] Roll back a KV engine save only while its connection stays removed --- .../kv-cache-engine-registry.spec.ts | 50 +++++++++++++++++++ .../kv-cache/kv-cache-engine-registry.ts | 5 +- 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/proprietary/kv-cache/__tests__/kv-cache-engine-registry.spec.ts b/proprietary/kv-cache/__tests__/kv-cache-engine-registry.spec.ts index 0dcee5de5..db236a27c 100644 --- a/proprietary/kv-cache/__tests__/kv-cache-engine-registry.spec.ts +++ b/proprietary/kv-cache/__tests__/kv-cache-engine-registry.spec.ts @@ -290,6 +290,56 @@ describe('KvCacheEngineRegistry', () => { expect(samples.settled).toHaveBeenCalled(); }); + it('keeps a create whose save overlapped a remove and re-register of its connection', async () => { + const { registry, storage, footprint, removal, restore } = setup(); + await registry.onModuleInit(); + let finishSave: () => void = () => undefined; + storage.saveKvCacheEngine.mockImplementationOnce((e: any) => new Promise((resolve) => (finishSave = () => resolve(e)))); + const pending = registry.save(engine('racing', 'c1')); + await new Promise((resolve) => setImmediate(resolve)); + await removal('c1'); + footprint.hasConnection.mockReturnValue(true); + await restore('c1'); + finishSave(); + await expect(pending).resolves.toMatchObject({ id: 'racing' }); + expect(storage.deleteKvCacheEngine).not.toHaveBeenCalledWith('racing'); + expect(registry.get('racing')).not.toBeNull(); + }); + + it('keeps an update whose save overlapped a remove and re-register and reloads it on restore', async () => { + const { registry, storage, footprint, removal, restore } = setup(); + await registry.onModuleInit(); + let finishSave: () => void = () => undefined; + storage.saveKvCacheEngine.mockImplementationOnce((e: any) => new Promise((resolve) => (finishSave = () => resolve(e)))); + const renamed = { ...engine('a', 'c1', 'lmc-a'), name: 'renamed' }; + const pending = registry.save(renamed).catch((error) => error); + await new Promise((resolve) => setImmediate(resolve)); + const removing = removal('c1'); + footprint.hasConnection.mockReturnValue(true); + finishSave(); + await removing; + await pending; + expect(storage.deleteKvCacheEngine).not.toHaveBeenCalledWith('a'); + storage.getKvCacheEngines.mockResolvedValue([renamed, engine('b', 'c2')]); + await restore('c1'); + expect(registry.get('a')?.name).toBe('renamed'); + }); + + it('drops an update whose save overlapped a removal that was never undone', async () => { + const { registry, storage, removal } = setup(); + await registry.onModuleInit(); + let finishSave: () => void = () => undefined; + storage.saveKvCacheEngine.mockImplementationOnce((e: any) => new Promise((resolve) => (finishSave = () => resolve(e)))); + const pending = registry.save({ ...engine('a', 'c1', 'lmc-a'), name: 'renamed' }); + await new Promise((resolve) => setImmediate(resolve)); + const removing = removal('c1'); + finishSave(); + await removing; + await expect(pending).rejects.toBeInstanceOf(NotFoundException); + expect(storage.deleteKvCacheEngine).toHaveBeenCalledWith('a'); + expect(registry.get('a')).toBeNull(); + }); + it('reloads the engines of a connection that was registered again before cleanup', async () => { const { registry, storage, removal, restore } = setup(); await registry.onModuleInit(); diff --git a/proprietary/kv-cache/kv-cache-engine-registry.ts b/proprietary/kv-cache/kv-cache-engine-registry.ts index 8c96a8d90..a3bd6281f 100644 --- a/proprietary/kv-cache/kv-cache-engine-registry.ts +++ b/proprietary/kv-cache/kv-cache-engine-registry.ts @@ -104,7 +104,10 @@ export class KvCacheEngineRegistry implements OnModuleInit { this.assertConnectionLive(engine.connectionId); const removals = this.removals.get(engine.connectionId); const saved = await this.storage.saveKvCacheEngine(engine); - if (this.removals.get(saved.connectionId) !== removals) { + if (this.removals.get(saved.connectionId) !== removals && !this.footprint.hasConnection(saved.connectionId)) { + this.engines.delete(saved.id); + this.statusVersions.delete(saved.id); + this.persistedSeenAt.delete(saved.id); await this.storage.deleteKvCacheEngine(saved.id); throw new NotFoundException(`Connection '${saved.connectionId}' not found`); }