Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions pytest.ini
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
[pytest]
pythonpath = .
172 changes: 172 additions & 0 deletions scripts/traffic_telemetry.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,172 @@
"""
Ingress Gateway Traffic Telemetry & Upstream Latency Anomaly Diagnostic for RouteKernel.
Collects latency percentiles, tracks status code distributions, computes EWMA z-score anomaly
spikes, and exports standard Prometheus exposition format.
"""

from __future__ import annotations

from collections import deque
from dataclasses import dataclass
import math
import time
from typing import Dict, List, Optional, Tuple


@dataclass
class AnomalyEvent:
timestamp: float
upstream_target: str
observed_latency_ms: float
baseline_mean_ms: float
z_score: float
severity: str
message: str


class TrafficTelemetry:
"""
Real-time routing traffic telemetry rotary with statistical anomaly detection.
"""

def __init__(
self,
window_size: int = 1000,
ewma_alpha: float = 0.1,
z_score_threshold: float = 3.0,
min_samples_for_anomaly: int = 20,
):
self.window_size = max(10, window_size)
self.ewma_alpha = max(0.01, min(0.99, ewma_alpha))
self.z_score_threshold = z_score_threshold
self.min_samples = min_samples_for_anomaly

self.total_requests = 0
self.status_codes: Dict[int, int] = {}
self.upstream_requests: Dict[str, int] = {}
self.upstream_errors: Dict[str, int] = {}

# Upstream target -> sliding window of latency_ms
self.latency_windows: Dict[str, deque[float]] = {}
# Upstream target -> EWMA state
self.ewma_latencies: Dict[str, float] = {}

self.anomalies: List[AnomalyEvent] = []

def record_request(
self,
upstream_target: str,
status_code: int,
latency_ms: float,
timestamp: Optional[float] = None,
) -> Optional[AnomalyEvent]:
"""
Records a routed request outcome and analyzes for statistical latency anomalies.
"""
ts = timestamp if timestamp is not None else time.time()
lat = max(0.0, float(latency_ms))

self.total_requests += 1
self.status_codes[status_code] = self.status_codes.get(status_code, 0) + 1
self.upstream_requests[upstream_target] = self.upstream_requests.get(upstream_target, 0) + 1

if status_code >= 500 or status_code == 0:
self.upstream_errors[upstream_target] = self.upstream_errors.get(upstream_target, 0) + 1

if upstream_target not in self.latency_windows:
self.latency_windows[upstream_target] = deque(maxlen=self.window_size)
self.ewma_latencies[upstream_target] = lat

window = self.latency_windows[upstream_target]

anomaly: Optional[AnomalyEvent] = None

# Anomaly Detection before updating window
if len(window) >= self.min_samples:
mean = sum(window) / len(window)
variance = sum((x - mean) ** 2 for x in window) / (len(window) - 1)
raw_std_dev = math.sqrt(variance)
# Guard against zero variance dividing by zero: use floor of 5% mean or 0.1ms
std_dev = max(raw_std_dev, 0.05 * mean, 0.1)

z = (lat - mean) / std_dev
if z >= self.z_score_threshold:
sev = "CRITICAL" if z >= 4.5 else "WARNING"
anomaly = AnomalyEvent(
timestamp=ts,
upstream_target=upstream_target,
observed_latency_ms=round(lat, 2),
baseline_mean_ms=round(mean, 2),
z_score=round(z, 2),
severity=sev,
message=f"Latency spike detected on {upstream_target}: {lat:.1f}ms vs baseline {mean:.1f}ms (z={z:.2f})",
)
self.anomalies.append(anomaly)

# Update statistics
window.append(lat)
prev_ewma = self.ewma_latencies[upstream_target]
self.ewma_latencies[upstream_target] = (self.ewma_alpha * lat) + ((1.0 - self.ewma_alpha) * prev_ewma)

return anomaly

def get_percentiles(self, upstream_target: Optional[str] = None) -> Dict[str, float]:
"""
Computes p50, p90, p99 latencies for a target or across all recorded latencies.
"""
samples: List[float] = []
if upstream_target:
if upstream_target in self.latency_windows:
samples = list(self.latency_windows[upstream_target])
else:
for w in self.latency_windows.values():
samples.extend(w)

if not samples:
return {"p50": 0.0, "p90": 0.0, "p99": 0.0, "max": 0.0}

sorted_samples = sorted(samples)
n = len(sorted_samples)

def pct(p: float) -> float:
idx = min(n - 1, max(0, int(math.ceil((p / 100.0) * n)) - 1))
return round(sorted_samples[idx], 2)

return {
"p50": pct(50),
"p90": pct(90),
"p99": pct(99),
"max": round(sorted_samples[-1], 2),
}

def export_prometheus(self, service: str = "routekernel") -> str:
"""
Generates standard Prometheus exposition text format.
"""
lines = [
f"# HELP routekernel_requests_total Total HTTP requests routed through kernel",
f"# TYPE routekernel_requests_total counter",
f'routekernel_requests_total{{service="{service}"}} {self.total_requests}',
f"# HELP routekernel_upstream_errors_total Total 5xx or connection failures per upstream",
f"# TYPE routekernel_upstream_errors_total counter",
]

for target, errors in self.upstream_errors.items():
lines.append(f'routekernel_upstream_errors_total{{service="{service}",upstream="{target}"}} {errors}')

pcts = self.get_percentiles()
lines.extend([
f"# HELP routekernel_latency_ms Latency percentiles in milliseconds",
f"# TYPE routekernel_latency_ms gauge",
f'routekernel_latency_ms{{service="{service}",quantile="0.50"}} {pcts["p50"]}',
f'routekernel_latency_ms{{service="{service}",quantile="0.90"}} {pcts["p90"]}',
f'routekernel_latency_ms{{service="{service}",quantile="0.99"}} {pcts["p99"]}',
f"# HELP routekernel_anomalies_total Total statistical latency anomalies triggered",
f"# TYPE routekernel_anomalies_total counter",
f'routekernel_anomalies_total{{service="{service}"}} {len(self.anomalies)}',
])

for code, count in self.status_codes.items():
lines.append(f'routekernel_http_status_total{{service="{service}",status="{code}"}} {count}')

return "\n".join(lines) + "\n"
59 changes: 59 additions & 0 deletions tests/test_traffic_telemetry.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
import pytest
from scripts.traffic_telemetry import TrafficTelemetry

def test_traffic_telemetry_recording_and_percentiles():
telemetry = TrafficTelemetry(window_size=100)

for i in range(1, 101):
telemetry.record_request("upstream-auth", 200, float(i))

assert telemetry.total_requests == 100
assert telemetry.status_codes[200] == 100
assert telemetry.upstream_requests["upstream-auth"] == 100

pcts = telemetry.get_percentiles("upstream-auth")
assert pcts["p50"] == 50.0
assert pcts["p90"] == 90.0
assert pcts["p99"] == 99.0
assert pcts["max"] == 100.0


def test_traffic_telemetry_anomaly_detection_spike():
telemetry = TrafficTelemetry(window_size=100, min_samples_for_anomaly=25, z_score_threshold=3.0)

# Establish baseline around 20ms with low variance
for _ in range(30):
telemetry.record_request("upstream-payment", 200, 20.0)

# Sudden high latency spike
anomaly = telemetry.record_request("upstream-payment", 200, 250.0)
assert anomaly is not None
assert anomaly.severity == "CRITICAL"
assert anomaly.z_score > 3.0
assert len(telemetry.anomalies) == 1
assert "Latency spike detected" in anomaly.message


def test_traffic_telemetry_error_tracking():
telemetry = TrafficTelemetry()

telemetry.record_request("upstream-billing", 200, 15.0)
telemetry.record_request("upstream-billing", 502, 35.0)
telemetry.record_request("upstream-billing", 504, 5000.0)

assert telemetry.upstream_errors["upstream-billing"] == 2
assert telemetry.status_codes[502] == 1
assert telemetry.status_codes[504] == 1


def test_traffic_telemetry_prometheus_export():
telemetry = TrafficTelemetry()
telemetry.record_request("upstream-orders", 200, 25.0)
telemetry.record_request("upstream-orders", 500, 150.0)

text = telemetry.export_prometheus("routekernel-prod")
assert 'routekernel_requests_total{service="routekernel-prod"} 2' in text
assert 'routekernel_upstream_errors_total{service="routekernel-prod",upstream="upstream-orders"} 1' in text
assert 'routekernel_http_status_total{service="routekernel-prod",status="200"} 1' in text
assert 'routekernel_http_status_total{service="routekernel-prod",status="500"} 1' in text
assert 'routekernel_latency_ms{service="routekernel-prod",quantile="0.50"}' in text
Loading