Add benchmark scenarios and dashboard

README.md

@@ -30,6 +30,12 @@ Run a quick local benchmark:
 python bench/run_bench.py --requests 100 --concurrency 10
 ```
 
+Open the local dashboard:
+
+```text
+http://localhost:8000/dashboard
+```
+
 ## Docker
 
 ```bash
@@ -38,6 +44,26 @@ docker compose up --build
 
 That starts Redis, one gateway, and two workers.
 
+## Benchmarks
+
+The scenario runner starts local gateway and worker processes, sends load, then writes JSON and markdown results.
+
+```bash
+python bench/run_scenarios.py
+```
+
+Latest local run:
+
+| Scenario | RPS | p50 ms | p95 ms | Cache hits | Failures |
+| --- | ---: | ---: | ---: | ---: | ---: |
+| cache enabled | 211.37 | 5.54 | 312.52 | 71 | 0 |
+| cache disabled | 24.92 | 318.85 | 326.38 | 0 | 0 |
+| slow worker, round-robin | 25.68 | 67.3 | 1033.24 | 0 | 0 |
+| slow worker, load-aware | 128.51 | 4.94 | 425.03 | 0 | 0 |
+| dead worker recovery | 698.97 | 11.15 | 28.1 | 0 | 0 |
+
+Full output is in `bench/results/latest.md` and `bench/results/latest.json`.
+
 ## Training
 
 The worker can load a small trained text model artifact from `worker/app/artifacts/tiny_text_model.json`.
@@ -57,7 +83,8 @@ If it is using the Mac GPU, the output includes `'device': 'mps'`.
 - Worker service with a tiny text classifier and optional trained artifact
 - Redis-backed cache when Redis is available, with an in-memory fallback
 - Retry on failed worker calls and basic load-aware routing
-- Simple benchmark script
+- Scenario benchmark runner with saved results
+- Local dashboard for gateway and worker state
 - Unit tests for routing, caching, metrics, and the model
 
 This is still intentionally small. The useful part is the system behavior around the model, not the model itself.

bench/results/latest.json

@@ -0,0 +1,212 @@
+{
+  "generated_by": "python bench/run_scenarios.py",
+  "results": [
+    {
+      "name": "cache_enabled",
+      "description": "One delayed worker, four repeated inputs, cache enabled.",
+      "router_strategy": "load_aware",
+      "cache_enabled": true,
+      "requests": 80,
+      "concurrency": 8,
+      "successes": 80,
+      "failures": 0,
+      "cache_hits": 71,
+      "cache_hit_rate": 0.8875,
+      "total_ms": 378.48,
+      "requests_per_second": 211.37,
+      "p50_latency_ms": 5.54,
+      "p95_latency_ms": 312.52,
+      "gateway_metrics": {
+        "requests": 80,
+        "cache_hits": 71,
+        "worker_errors": 0,
+        "cache_hit_rate": 0.8875,
+        "requests_per_second": 159.35,
+        "p50_latency_ms": 0.01,
+        "p95_latency_ms": 294.25
+      },
+      "workers": [
+        {
+          "url": "http://localhost:8311",
+          "in_flight": 0,
+          "requests": 9,
+          "failures": 0,
+          "latency_ema_ms": 268.22,
+          "last_error": null,
+          "strategy": "load_aware"
+        }
+      ]
+    },
+    {
+      "name": "cache_disabled",
+      "description": "Same traffic as cache_enabled, but cache disabled.",
+      "router_strategy": "load_aware",
+      "cache_enabled": false,
+      "requests": 80,
+      "concurrency": 8,
+      "successes": 80,
+      "failures": 0,
+      "cache_hits": 0,
+      "cache_hit_rate": 0.0,
+      "total_ms": 3210.55,
+      "requests_per_second": 24.92,
+      "p50_latency_ms": 318.85,
+      "p95_latency_ms": 326.38,
+      "gateway_metrics": {
+        "requests": 80,
+        "cache_hits": 0,
+        "worker_errors": 0,
+        "cache_hit_rate": 0.0,
+        "requests_per_second": 23.93,
+        "p50_latency_ms": 309.16,
+        "p95_latency_ms": 316.54
+      },
+      "workers": [
+        {
+          "url": "http://localhost:8311",
+          "in_flight": 0,
+          "requests": 80,
+          "failures": 0,
+          "latency_ema_ms": 294.34,
+          "last_error": null,
+          "strategy": "load_aware"
+        }
+      ]
+    },
+    {
+      "name": "slow_worker_round_robin",
+      "description": "One fast worker and one slow worker with round-robin routing.",
+      "router_strategy": "round_robin",
+      "cache_enabled": false,
+      "requests": 80,
+      "concurrency": 12,
+      "successes": 80,
+      "failures": 0,
+      "cache_hits": 0,
+      "cache_hit_rate": 0.0,
+      "total_ms": 3115.31,
+      "requests_per_second": 25.68,
+      "p50_latency_ms": 67.3,
+      "p95_latency_ms": 1033.24,
+      "gateway_metrics": {
+        "requests": 80,
+        "cache_hits": 0,
+        "worker_errors": 0,
+        "cache_hit_rate": 0.0,
+        "requests_per_second": 24.67,
+        "p50_latency_ms": 108.0,
+        "p95_latency_ms": 1029.4
+      },
+      "workers": [
+        {
+          "url": "http://localhost:8311",
+          "in_flight": 0,
+          "requests": 57,
+          "failures": 0,
+          "latency_ema_ms": 11.48,
+          "last_error": null,
+          "strategy": "round_robin"
+        },
+        {
+          "url": "http://localhost:8312",
+          "in_flight": 0,
+          "requests": 40,
+          "failures": 17,
+          "latency_ema_ms": 777.5,
+          "last_error": "ReadTimeout",
+          "strategy": "round_robin"
+        }
+      ]
+    },
+    {
+      "name": "slow_worker_load_aware",
+      "description": "Same workers as slow_worker_round_robin, using load-aware routing.",
+      "router_strategy": "load_aware",
+      "cache_enabled": false,
+      "requests": 80,
+      "concurrency": 12,
+      "successes": 80,
+      "failures": 0,
+      "cache_hits": 0,
+      "cache_hit_rate": 0.0,
+      "total_ms": 622.53,
+      "requests_per_second": 128.51,
+      "p50_latency_ms": 4.94,
+      "p95_latency_ms": 425.03,
+      "gateway_metrics": {
+        "requests": 80,
+        "cache_hits": 0,
+        "worker_errors": 0,
+        "cache_hit_rate": 0.0,
+        "requests_per_second": 109.34,
+        "p50_latency_ms": 3.61,
+        "p95_latency_ms": 408.65
+      },
+      "workers": [
+        {
+          "url": "http://localhost:8311",
+          "in_flight": 0,
+          "requests": 74,
+          "failures": 0,
+          "latency_ema_ms": 3.82,
+          "last_error": null,
+          "strategy": "load_aware"
+        },
+        {
+          "url": "http://localhost:8312",
+          "in_flight": 0,
+          "requests": 6,
+          "failures": 0,
+          "latency_ema_ms": 365.08,
+          "last_error": null,
+          "strategy": "load_aware"
+        }
+      ]
+    },
+    {
+      "name": "dead_worker_recovery",
+      "description": "One worker always fails, one worker stays healthy.",
+      "router_strategy": "load_aware",
+      "cache_enabled": false,
+      "requests": 60,
+      "concurrency": 10,
+      "successes": 60,
+      "failures": 0,
+      "cache_hits": 0,
+      "cache_hit_rate": 0.0,
+      "total_ms": 85.84,
+      "requests_per_second": 698.97,
+      "p50_latency_ms": 11.15,
+      "p95_latency_ms": 28.1,
+      "gateway_metrics": {
+        "requests": 60,
+        "cache_hits": 0,
+        "worker_errors": 0,
+        "cache_hit_rate": 0.0,
+        "requests_per_second": 275.53,
+        "p50_latency_ms": 8.53,
+        "p95_latency_ms": 19.94
+      },
+      "workers": [
+        {
+          "url": "http://localhost:8311",
+          "in_flight": 0,
+          "requests": 5,
+          "failures": 5,
+          "latency_ema_ms": 0.0,
+          "last_error": "HTTPStatusError",
+          "strategy": "load_aware"
+        },
+        {
+          "url": "http://localhost:8312",
+          "in_flight": 0,
+          "requests": 60,
+          "failures": 0,
+          "latency_ema_ms": 8.83,
+          "last_error": null,
+          "strategy": "load_aware"
+        }
+      ]
+    }
+  ]
+}

bench/results/latest.md

@@ -0,0 +1,17 @@
+# Benchmark results
+
+Generated with `python bench/run_scenarios.py` on the local machine.
+
+| Scenario | Router | Cache | Requests | RPS | p50 ms | p95 ms | Cache hits | Failures |
+| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: |
+| cache_enabled | load_aware | True | 80 | 211.37 | 5.54 | 312.52 | 71 | 0 |
+| cache_disabled | load_aware | False | 80 | 24.92 | 318.85 | 326.38 | 0 | 0 |
+| slow_worker_round_robin | round_robin | False | 80 | 25.68 | 67.3 | 1033.24 | 0 | 0 |
+| slow_worker_load_aware | load_aware | False | 80 | 128.51 | 4.94 | 425.03 | 0 | 0 |
+| dead_worker_recovery | load_aware | False | 60 | 698.97 | 11.15 | 28.1 | 0 | 0 |
+
+Notes:
+
+- Cache scenarios repeat four text inputs to show cache behavior.
+- Slow-worker scenarios use one normal worker and one worker with 95 ms artificial delay.
+- Dead-worker recovery uses one worker with a 100 percent simulated failure rate.

bench/run_bench.py

@@ -2,6 +2,7 @@
 
 import argparse
 import asyncio
+import json
 import statistics
 import time
 
@@ -21,42 +22,52 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument("--url", default="http://localhost:8000/predict")
     parser.add_argument("--requests", type=int, default=100)
     parser.add_argument("--concurrency", type=int, default=10)
+    parser.add_argument("--unique-requests", action="store_true")
     return parser.parse_args()
 
 
-async def call(client: httpx.AsyncClient, url: str, index: int) -> tuple[float, bool]:
+async def call(client: httpx.AsyncClient, url: str, index: int, *, unique_requests: bool) -> tuple[float, bool, bool]:
     started = time.perf_counter()
-    response = await client.post(url, json={"text": SAMPLES[index % len(SAMPLES)], "request_id": str(index)})
+    text = SAMPLES[index % len(SAMPLES)]
+    if unique_requests:
+        text = f"{text} {index}"
+    response = await client.post(url, json={"text": text, "request_id": str(index)})
     latency = (time.perf_counter() - started) * 1000
-    return latency, response.status_code == 200
+    cached = False
+    if response.status_code == 200:
+        cached = bool(response.json().get("cached"))
+    return latency, response.status_code == 200, cached
 
 
 async def main() -> None:
     args = parse_args()
     semaphore = asyncio.Semaphore(args.concurrency)
 
     async with httpx.AsyncClient(timeout=5) as client:
-        async def limited(index: int) -> tuple[float, bool]:
+        async def limited(index: int) -> tuple[float, bool, bool]:
             async with semaphore:
-                return await call(client, args.url, index)
+                return await call(client, args.url, index, unique_requests=args.unique_requests)
 
         started = time.perf_counter()
         results = await asyncio.gather(*(limited(index) for index in range(args.requests)))
         total_ms = (time.perf_counter() - started) * 1000
 
-    latencies = [latency for latency, ok in results if ok]
-    failures = sum(1 for _, ok in results if not ok)
+    latencies = [latency for latency, ok, _ in results if ok]
+    failures = sum(1 for _, ok, _ in results if not ok)
+    cache_hits = sum(1 for _, ok, cached in results if ok and cached)
     p95 = statistics.quantiles(latencies, n=20)[18] if len(latencies) >= 20 else max(latencies, default=0)
 
-    print({
+    print(json.dumps({
         "requests": args.requests,
         "successes": len(latencies),
         "failures": failures,
+        "cache_hits": cache_hits,
+        "cache_hit_rate": round(cache_hits / len(latencies), 4) if latencies else 0,
         "total_ms": round(total_ms, 2),
         "requests_per_second": round(args.requests / (total_ms / 1000), 2),
         "p50_latency_ms": round(statistics.median(latencies), 2) if latencies else 0,
         "p95_latency_ms": round(p95, 2),
-    })
+    }, indent=2))
 
 
 if __name__ == "__main__":