diff --git a/README.md b/README.md index 041cda7..a6e6556 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,12 @@ Open the local dashboard: http://localhost:8000/dashboard ``` +Run live traffic against the dashboard: + +```bash +python bench/replay_traffic.py --seconds-per-stage 8 --rps 18 +``` + ## Docker ```bash @@ -56,17 +62,17 @@ Latest local run: | Scenario | RPS | p50 ms | p95 ms | Cache hits | Failures | | --- | ---: | ---: | ---: | ---: | ---: | -| cache enabled | 211.37 | 5.54 | 312.52 | 71 | 0 | -| cache disabled | 24.92 | 318.85 | 326.38 | 0 | 0 | -| slow worker, round-robin | 25.68 | 67.3 | 1033.24 | 0 | 0 | -| slow worker, load-aware | 128.51 | 4.94 | 425.03 | 0 | 0 | -| dead worker recovery | 698.97 | 11.15 | 28.1 | 0 | 0 | +| cache enabled | 728.95 | 5.01 | 52.04 | 72 | 0 | +| cache disabled | 185.7 | 41.1 | 53.09 | 0 | 0 | +| slow worker, round-robin | 187.36 | 57.79 | 111.91 | 0 | 0 | +| slow worker, load-aware | 688.48 | 5.21 | 111.9 | 0 | 0 | +| dead worker recovery | 707.98 | 11.37 | 28.93 | 0 | 0 | Full output is in `bench/results/latest.md` and `bench/results/latest.json`. ## Training -The worker can load a small trained text model artifact from `worker/app/artifacts/tiny_text_model.json`. +The worker can load a small trained text model artifact from `worker/app/artifacts/tiny_text_model.json`. The current model trains on generated service-status examples, so the point is the local training and serving path, not benchmark-grade NLP accuracy. On Apple Silicon, the training script uses PyTorch MPS. It refuses to run on CPU by default so a slow accidental CPU run fails fast. @@ -77,6 +83,21 @@ python training/train_text_model.py If it is using the Mac GPU, the output includes `'device': 'mps'`. +Latest training run on an M3 Pro: + +| Metric | Value | +| --- | ---: | +| Device | mps | +| Epochs | 320 | +| Train examples | 3,276 | +| Test examples | 820 | +| Train time | 0.4351 s | +| Examples/sec | 2,409,512.37 | +| Train accuracy | 1.0 | +| Test accuracy | 1.0 | + +The full run output is in `training/runs/latest.md` and `training/runs/latest.json`. + ## What is in here now - FastAPI gateway with `/predict`, `/health`, `/workers`, and `/metrics` @@ -85,6 +106,7 @@ If it is using the Mac GPU, the output includes `'device': 'mps'`. - Retry on failed worker calls and basic load-aware routing - Scenario benchmark runner with saved results - Local dashboard for gateway and worker state +- Worker delay/failure controls for live demos - Unit tests for routing, caching, metrics, and the model This is still intentionally small. The useful part is the system behavior around the model, not the model itself. diff --git a/bench/replay_traffic.py b/bench/replay_traffic.py new file mode 100644 index 0000000..e51f3b3 --- /dev/null +++ b/bench/replay_traffic.py @@ -0,0 +1,133 @@ +from __future__ import annotations + +import argparse +import asyncio +import json +import time +from pathlib import Path +from typing import Any + +import httpx + + +ROOT = Path(__file__).resolve().parents[1] +RESULT_PATH = ROOT / "bench" / "results" / "replay-latest.json" +TEXTS = [ + "fast stable reliable service", + "cache hit keeps service fast", + "great latency stable route", + "slow broken worker timeout", + "worker crash caused retry", + "request completed healthy worker", +] + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--base-url", default="http://localhost:8000") + parser.add_argument("--seconds-per-stage", type=float, default=8.0) + parser.add_argument("--rps", type=float, default=18.0) + parser.add_argument("--output", type=Path, default=RESULT_PATH) + return parser.parse_args() + + +async def control_worker(client: httpx.AsyncClient, base_url: str, index: int, *, delay_ms: int, fail_rate: float) -> bool: + try: + response = await client.post( + f"{base_url}/workers/{index}/control", + json={"delay_ms": delay_ms, "fail_rate": fail_rate}, + timeout=1.5, + ) + return response.status_code == 200 + except httpx.HTTPError: + return False + + +async def reset_workers(client: httpx.AsyncClient, base_url: str) -> None: + try: + workers = (await client.get(f"{base_url}/workers", timeout=1.5)).json() + except httpx.HTTPError: + return + + for index, _ in enumerate(workers): + await control_worker(client, base_url, index, delay_ms=0, fail_rate=0) + + +async def snapshot(client: httpx.AsyncClient, base_url: str, stage: str) -> dict[str, Any]: + metrics = (await client.get(f"{base_url}/metrics")).json() + workers = (await client.get(f"{base_url}/workers")).json() + return { + "stage": stage, + "metrics": metrics, + "workers": workers, + } + + +async def send_stage(client: httpx.AsyncClient, base_url: str, stage: str, seconds: float, rps: float) -> dict[str, Any]: + deadline = time.perf_counter() + seconds + interval = 1 / rps + sent = 0 + ok = 0 + started = time.perf_counter() + + while time.perf_counter() < deadline: + text = TEXTS[sent % len(TEXTS)] + if stage in {"slow-worker", "flaky-worker"}: + text = f"{text} {stage} live {sent}" + + try: + response = await client.post( + f"{base_url}/predict", + json={"text": text, "request_id": f"{stage}-{sent}"}, + timeout=3.0, + ) + ok += int(response.status_code == 200) + except httpx.HTTPError: + pass + + sent += 1 + elapsed = time.perf_counter() - started + sleep_for = max(0.0, sent * interval - elapsed) + await asyncio.sleep(sleep_for) + + snap = await snapshot(client, base_url, stage) + return { + "stage": stage, + "sent": sent, + "ok": ok, + **snap, + } + + +async def main() -> None: + args = parse_args() + base_url = args.base_url.rstrip("/") + stages: list[dict[str, Any]] = [] + + async with httpx.AsyncClient(timeout=5.0) as client: + await reset_workers(client, base_url) + stages.append(await send_stage(client, base_url, "baseline", args.seconds_per_stage, args.rps)) + + await control_worker(client, base_url, 1, delay_ms=350, fail_rate=0) + stages.append(await send_stage(client, base_url, "slow-worker", args.seconds_per_stage, args.rps)) + + await reset_workers(client, base_url) + await control_worker(client, base_url, 0, delay_ms=0, fail_rate=0.45) + stages.append(await send_stage(client, base_url, "flaky-worker", args.seconds_per_stage, args.rps)) + + await reset_workers(client, base_url) + stages.append(await send_stage(client, base_url, "recovery", args.seconds_per_stage, args.rps)) + + payload = { + "base_url": base_url, + "seconds_per_stage": args.seconds_per_stage, + "target_rps": args.rps, + "stages": stages, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(payload, indent=2) + "\n") + print(json.dumps(payload, indent=2)) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/bench/results/latest.json b/bench/results/latest.json index 900d514..b709c26 100644 --- a/bench/results/latest.json +++ b/bench/results/latest.json @@ -10,30 +10,36 @@ "concurrency": 8, "successes": 80, "failures": 0, - "cache_hits": 71, - "cache_hit_rate": 0.8875, - "total_ms": 378.48, - "requests_per_second": 211.37, - "p50_latency_ms": 5.54, - "p95_latency_ms": 312.52, + "cache_hits": 72, + "cache_hit_rate": 0.9, + "total_ms": 109.75, + "requests_per_second": 728.95, + "p50_latency_ms": 5.01, + "p95_latency_ms": 52.04, "gateway_metrics": { "requests": 80, - "cache_hits": 71, + "cache_hits": 72, "worker_errors": 0, - "cache_hit_rate": 0.8875, - "requests_per_second": 159.35, - "p50_latency_ms": 0.01, - "p95_latency_ms": 294.25 + "cache_hit_rate": 0.9, + "requests_per_second": 356.94, + "p50_latency_ms": 0.0, + "p95_latency_ms": 41.44 }, "workers": [ { "url": "http://localhost:8311", "in_flight": 0, - "requests": 9, + "requests": 8, "failures": 0, - "latency_ema_ms": 268.22, + "latency_ema_ms": 41.54, "last_error": null, - "strategy": "load_aware" + "strategy": "load_aware", + "index": 0, + "worker_id": "worker-a", + "model_version": "tiny-text-mps-v1", + "delay_ms": 35, + "fail_rate": 0.0, + "control_status": "ok" } ] }, @@ -48,18 +54,18 @@ "failures": 0, "cache_hits": 0, "cache_hit_rate": 0.0, - "total_ms": 3210.55, - "requests_per_second": 24.92, - "p50_latency_ms": 318.85, - "p95_latency_ms": 326.38, + "total_ms": 430.79, + "requests_per_second": 185.7, + "p50_latency_ms": 41.1, + "p95_latency_ms": 53.09, "gateway_metrics": { "requests": 80, "cache_hits": 0, "worker_errors": 0, "cache_hit_rate": 0.0, - "requests_per_second": 23.93, - "p50_latency_ms": 309.16, - "p95_latency_ms": 316.54 + "requests_per_second": 143.14, + "p50_latency_ms": 38.43, + "p95_latency_ms": 42.38 }, "workers": [ { @@ -67,9 +73,15 @@ "in_flight": 0, "requests": 80, "failures": 0, - "latency_ema_ms": 294.34, + "latency_ema_ms": 38.69, "last_error": null, - "strategy": "load_aware" + "strategy": "load_aware", + "index": 0, + "worker_id": "worker-a", + "model_version": "tiny-text-mps-v1", + "delay_ms": 35, + "fail_rate": 0.0, + "control_status": "ok" } ] }, @@ -84,37 +96,49 @@ "failures": 0, "cache_hits": 0, "cache_hit_rate": 0.0, - "total_ms": 3115.31, - "requests_per_second": 25.68, - "p50_latency_ms": 67.3, - "p95_latency_ms": 1033.24, + "total_ms": 426.99, + "requests_per_second": 187.36, + "p50_latency_ms": 57.79, + "p95_latency_ms": 111.91, "gateway_metrics": { "requests": 80, "cache_hits": 0, "worker_errors": 0, "cache_hit_rate": 0.0, - "requests_per_second": 24.67, - "p50_latency_ms": 108.0, - "p95_latency_ms": 1029.4 + "requests_per_second": 143.63, + "p50_latency_ms": 95.87, + "p95_latency_ms": 103.0 }, "workers": [ { "url": "http://localhost:8311", "in_flight": 0, - "requests": 57, + "requests": 40, "failures": 0, - "latency_ema_ms": 11.48, + "latency_ema_ms": 1.7, "last_error": null, - "strategy": "round_robin" + "strategy": "round_robin", + "index": 0, + "worker_id": "worker-fast", + "model_version": "tiny-text-mps-v1", + "delay_ms": 0, + "fail_rate": 0.0, + "control_status": "ok" }, { "url": "http://localhost:8312", "in_flight": 0, "requests": 40, - "failures": 17, - "latency_ema_ms": 777.5, - "last_error": "ReadTimeout", - "strategy": "round_robin" + "failures": 0, + "latency_ema_ms": 97.45, + "last_error": null, + "strategy": "round_robin", + "index": 1, + "worker_id": "worker-slow", + "model_version": "tiny-text-mps-v1", + "delay_ms": 95, + "fail_rate": 0.0, + "control_status": "ok" } ] }, @@ -129,18 +153,18 @@ "failures": 0, "cache_hits": 0, "cache_hit_rate": 0.0, - "total_ms": 622.53, - "requests_per_second": 128.51, - "p50_latency_ms": 4.94, - "p95_latency_ms": 425.03, + "total_ms": 116.2, + "requests_per_second": 688.48, + "p50_latency_ms": 5.21, + "p95_latency_ms": 111.9, "gateway_metrics": { "requests": 80, "cache_hits": 0, "worker_errors": 0, "cache_hit_rate": 0.0, - "requests_per_second": 109.34, - "p50_latency_ms": 3.61, - "p95_latency_ms": 408.65 + "requests_per_second": 333.04, + "p50_latency_ms": 3.75, + "p95_latency_ms": 101.86 }, "workers": [ { @@ -148,18 +172,30 @@ "in_flight": 0, "requests": 74, "failures": 0, - "latency_ema_ms": 3.82, + "latency_ema_ms": 3.51, "last_error": null, - "strategy": "load_aware" + "strategy": "load_aware", + "index": 0, + "worker_id": "worker-fast", + "model_version": "tiny-text-mps-v1", + "delay_ms": 0, + "fail_rate": 0.0, + "control_status": "ok" }, { "url": "http://localhost:8312", "in_flight": 0, "requests": 6, "failures": 0, - "latency_ema_ms": 365.08, + "latency_ema_ms": 102.25, "last_error": null, - "strategy": "load_aware" + "strategy": "load_aware", + "index": 1, + "worker_id": "worker-slow", + "model_version": "tiny-text-mps-v1", + "delay_ms": 95, + "fail_rate": 0.0, + "control_status": "ok" } ] }, @@ -174,18 +210,18 @@ "failures": 0, "cache_hits": 0, "cache_hit_rate": 0.0, - "total_ms": 85.84, - "requests_per_second": 698.97, - "p50_latency_ms": 11.15, - "p95_latency_ms": 28.1, + "total_ms": 84.75, + "requests_per_second": 707.98, + "p50_latency_ms": 11.37, + "p95_latency_ms": 28.93, "gateway_metrics": { "requests": 60, "cache_hits": 0, "worker_errors": 0, "cache_hit_rate": 0.0, - "requests_per_second": 275.53, - "p50_latency_ms": 8.53, - "p95_latency_ms": 19.94 + "requests_per_second": 281.19, + "p50_latency_ms": 8.46, + "p95_latency_ms": 22.61 }, "workers": [ { @@ -195,16 +231,28 @@ "failures": 5, "latency_ema_ms": 0.0, "last_error": "HTTPStatusError", - "strategy": "load_aware" + "strategy": "load_aware", + "index": 0, + "worker_id": "worker-dead", + "model_version": "tiny-text-mps-v1", + "delay_ms": 0, + "fail_rate": 1.0, + "control_status": "ok" }, { "url": "http://localhost:8312", "in_flight": 0, "requests": 60, "failures": 0, - "latency_ema_ms": 8.83, + "latency_ema_ms": 10.88, "last_error": null, - "strategy": "load_aware" + "strategy": "load_aware", + "index": 1, + "worker_id": "worker-healthy", + "model_version": "tiny-text-mps-v1", + "delay_ms": 0, + "fail_rate": 0.0, + "control_status": "ok" } ] } diff --git a/bench/results/latest.md b/bench/results/latest.md index bf46122..32f641a 100644 --- a/bench/results/latest.md +++ b/bench/results/latest.md @@ -4,11 +4,11 @@ Generated with `python bench/run_scenarios.py` on the local machine. | Scenario | Router | Cache | Requests | RPS | p50 ms | p95 ms | Cache hits | Failures | | --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | -| cache_enabled | load_aware | True | 80 | 211.37 | 5.54 | 312.52 | 71 | 0 | -| cache_disabled | load_aware | False | 80 | 24.92 | 318.85 | 326.38 | 0 | 0 | -| slow_worker_round_robin | round_robin | False | 80 | 25.68 | 67.3 | 1033.24 | 0 | 0 | -| slow_worker_load_aware | load_aware | False | 80 | 128.51 | 4.94 | 425.03 | 0 | 0 | -| dead_worker_recovery | load_aware | False | 60 | 698.97 | 11.15 | 28.1 | 0 | 0 | +| cache_enabled | load_aware | True | 80 | 728.95 | 5.01 | 52.04 | 72 | 0 | +| cache_disabled | load_aware | False | 80 | 185.7 | 41.1 | 53.09 | 0 | 0 | +| slow_worker_round_robin | round_robin | False | 80 | 187.36 | 57.79 | 111.91 | 0 | 0 | +| slow_worker_load_aware | load_aware | False | 80 | 688.48 | 5.21 | 111.9 | 0 | 0 | +| dead_worker_recovery | load_aware | False | 60 | 707.98 | 11.37 | 28.93 | 0 | 0 | Notes: diff --git a/bench/results/replay-latest.json b/bench/results/replay-latest.json new file mode 100644 index 0000000..01c5064 --- /dev/null +++ b/bench/results/replay-latest.json @@ -0,0 +1,191 @@ +{ + "base_url": "http://localhost:8500", + "seconds_per_stage": 8.0, + "target_rps": 18.0, + "stages": [ + { + "stage": "baseline", + "sent": 144, + "ok": 144, + "metrics": { + "requests": 144, + "cache_hits": 138, + "worker_errors": 0, + "cache_hit_rate": 0.9583, + "requests_per_second": 11.92, + "p50_latency_ms": 0.05, + "p95_latency_ms": 0.09 + }, + "workers": [ + { + "url": "http://localhost:8511", + "in_flight": 0, + "requests": 4, + "failures": 0, + "latency_ema_ms": 1.71, + "last_error": null, + "strategy": "load_aware", + "index": 0, + "worker_id": "worker-fast", + "model_version": "tiny-text-mps-v1", + "delay_ms": 0, + "fail_rate": 0.0, + "control_status": "ok" + }, + { + "url": "http://localhost:8512", + "in_flight": 0, + "requests": 2, + "failures": 0, + "latency_ema_ms": 1.31, + "last_error": null, + "strategy": "load_aware", + "index": 1, + "worker_id": "worker-control", + "model_version": "tiny-text-mps-v1", + "delay_ms": 0, + "fail_rate": 0.0, + "control_status": "ok" + } + ] + }, + { + "stage": "slow-worker", + "sent": 144, + "ok": 144, + "metrics": { + "requests": 288, + "cache_hits": 138, + "worker_errors": 0, + "cache_hit_rate": 0.4792, + "requests_per_second": 14.32, + "p50_latency_ms": 1.32, + "p95_latency_ms": 5.11 + }, + "workers": [ + { + "url": "http://localhost:8511", + "in_flight": 0, + "requests": 147, + "failures": 0, + "latency_ema_ms": 4.59, + "last_error": null, + "strategy": "load_aware", + "index": 0, + "worker_id": "worker-fast", + "model_version": "tiny-text-mps-v1", + "delay_ms": 0, + "fail_rate": 0.0, + "control_status": "ok" + }, + { + "url": "http://localhost:8512", + "in_flight": 0, + "requests": 3, + "failures": 0, + "latency_ema_ms": 72.27, + "last_error": null, + "strategy": "load_aware", + "index": 1, + "worker_id": "worker-control", + "model_version": "tiny-text-mps-v1", + "delay_ms": 350, + "fail_rate": 0.0, + "control_status": "ok" + } + ] + }, + { + "stage": "flaky-worker", + "sent": 144, + "ok": 144, + "metrics": { + "requests": 432, + "cache_hits": 138, + "worker_errors": 0, + "cache_hit_rate": 0.3194, + "requests_per_second": 15.35, + "p50_latency_ms": 3.96, + "p95_latency_ms": 5.34 + }, + "workers": [ + { + "url": "http://localhost:8511", + "in_flight": 0, + "requests": 151, + "failures": 1, + "latency_ema_ms": 3.33, + "last_error": "HTTPStatusError", + "strategy": "load_aware", + "index": 0, + "worker_id": "worker-fast", + "model_version": "tiny-text-mps-v1", + "delay_ms": 0, + "fail_rate": 0.45, + "control_status": "ok" + }, + { + "url": "http://localhost:8512", + "in_flight": 0, + "requests": 144, + "failures": 0, + "latency_ema_ms": 4.55, + "last_error": null, + "strategy": "load_aware", + "index": 1, + "worker_id": "worker-control", + "model_version": "tiny-text-mps-v1", + "delay_ms": 0, + "fail_rate": 0.0, + "control_status": "ok" + } + ] + }, + { + "stage": "recovery", + "sent": 144, + "ok": 144, + "metrics": { + "requests": 576, + "cache_hits": 282, + "worker_errors": 0, + "cache_hit_rate": 0.4896, + "requests_per_second": 15.92, + "p50_latency_ms": 3.45, + "p95_latency_ms": 5.25 + }, + "workers": [ + { + "url": "http://localhost:8511", + "in_flight": 0, + "requests": 151, + "failures": 1, + "latency_ema_ms": 3.33, + "last_error": "HTTPStatusError", + "strategy": "load_aware", + "index": 0, + "worker_id": "worker-fast", + "model_version": "tiny-text-mps-v1", + "delay_ms": 0, + "fail_rate": 0.0, + "control_status": "ok" + }, + { + "url": "http://localhost:8512", + "in_flight": 0, + "requests": 144, + "failures": 0, + "latency_ema_ms": 4.55, + "last_error": null, + "strategy": "load_aware", + "index": 1, + "worker_id": "worker-control", + "model_version": "tiny-text-mps-v1", + "delay_ms": 0, + "fail_rate": 0.0, + "control_status": "ok" + } + ] + } + ] +} diff --git a/docs/architecture.md b/docs/architecture.md index 2552661..e2b1ea2 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -35,3 +35,14 @@ The benchmark runner uses both modes against the same slow-worker scenario so th `training/train_text_model.py` trains the tiny text model with PyTorch. On Apple Silicon it uses `mps`, and it exits instead of silently falling back to CPU unless `--allow-cpu` is passed. The trained artifact is plain JSON so the worker can load it without depending on PyTorch at inference time. + +The latest checked-in run trained on `mps` in under a second. The run file records device, epochs, examples/sec, train accuracy, and test accuracy so the performance claim is reproducible. + +## Live failure demo + +Workers expose a small local control endpoint for demo runs: + +- `GET /config` returns delay and failure settings. +- `POST /control` updates artificial delay and failure rate. + +The gateway wraps those controls at `POST /workers/{index}/control`, and the dashboard exposes inputs for delay and fail rate. `bench/replay_traffic.py` uses the same endpoint to move the system through baseline, slow-worker, flaky-worker, and recovery stages. diff --git a/docs/demo.md b/docs/demo.md new file mode 100644 index 0000000..59747e7 --- /dev/null +++ b/docs/demo.md @@ -0,0 +1,32 @@ +# Demo script + +This is the local demo flow I use to show the system moving. + +Terminal 1: + +```bash +docker compose up --build +``` + +Open: + +```text +http://localhost:8000/dashboard +``` + +Terminal 2: + +```bash +python bench/replay_traffic.py --seconds-per-stage 8 --rps 18 +``` + +What to watch: + +- Baseline traffic warms up the gateway. +- The slow-worker stage adds delay to worker 1 through the gateway control endpoint. +- The flaky-worker stage sets worker 0 to a 45 percent failure rate. +- Recovery resets both workers. + +The dashboard should show request count, p95 latency, worker failure counts, delay, and fail rate moving as the replay runs. + +The replay writes `bench/results/replay-latest.json` for later inspection. diff --git a/gateway/app/dashboard.py b/gateway/app/dashboard.py index 47db5d1..aa3c2fa 100644 --- a/gateway/app/dashboard.py +++ b/gateway/app/dashboard.py @@ -114,6 +114,29 @@ .bad { color: #fca5a5; } + input { + width: 76px; + border: 1px solid #334155; + border-radius: 6px; + background: #020617; + color: #e2e8f0; + padding: 7px 8px; + font: 12px ui-monospace, SFMono-Regular, Menlo, monospace; + } + button { + border: 1px solid #38bdf8; + border-radius: 6px; + background: #082f49; + color: #bae6fd; + padding: 8px 10px; + font: 12px ui-monospace, SFMono-Regular, Menlo, monospace; + cursor: pointer; + } + button.secondary { + border-color: #475569; + background: #111827; + color: #cbd5e1; + } @media (max-width: 760px) { header { display: block; @@ -174,6 +197,9 @@