From 04403736ad39722071c2cd13cd08b2812d0517a6 Mon Sep 17 00:00:00 2001 From: RAM PRASAD ATLURI <116977270+ATLURI0001@users.noreply.github.com> Date: Sat, 21 Nov 2026 21:00:22 +0530 Subject: [PATCH] feat(upstream): implement RFC 2439 route flap damping and oscillation suppressor --- scripts/flap_damper.py | 162 ++++++++++++++++++++++++++++++++++++++ tests/test_flap_damper.py | 124 +++++++++++++++++++++++++++++ 2 files changed, 286 insertions(+) create mode 100644 scripts/flap_damper.py create mode 100644 tests/test_flap_damper.py diff --git a/scripts/flap_damper.py b/scripts/flap_damper.py new file mode 100644 index 0000000..1910960 --- /dev/null +++ b/scripts/flap_damper.py @@ -0,0 +1,162 @@ +"""RFC 2439-inspired Upstream Route Flap Damping and Oscillation Damper for RouteKernel.""" + +from __future__ import annotations + +import math +import time +from dataclasses import dataclass +from enum import Enum +from typing import Dict, Optional + + +class UpstreamState(str, Enum): + HEALTHY = "HEALTHY" + FLAPPING = "FLAPPING" + SUPPRESSED = "SUPPRESSED" + + +@dataclass(frozen=True) +class FlapDampingConfig: + """Configuration governing penalty accumulation, decay half-life, and suppression thresholds.""" + + penalty_per_flap: float = 1000.0 + half_life_seconds: float = 15.0 + suppress_threshold: float = 2000.0 + reuse_threshold: float = 750.0 + max_suppress_seconds: float = 60.0 + + def __post_init__(self) -> None: + if self.penalty_per_flap <= 0: + raise ValueError("penalty_per_flap must be > 0") + if self.half_life_seconds <= 0: + raise ValueError("half_life_seconds must be > 0") + if self.suppress_threshold <= self.reuse_threshold: + raise ValueError("suppress_threshold must be strictly greater than reuse_threshold") + if self.reuse_threshold < 0: + raise ValueError("reuse_threshold must be >= 0") + if self.max_suppress_seconds <= 0: + raise ValueError("max_suppress_seconds must be > 0") + + +class UpstreamTrackRecord: + def __init__(self, upstream_id: str, initial_timestamp: float) -> None: + self.upstream_id = upstream_id + self.penalty = 0.0 + self.last_update = initial_timestamp + self.last_health = True + self.state = UpstreamState.HEALTHY + self.suppressed_at: Optional[float] = None + self.total_flaps = 0 + + def decay_to(self, current_time: float, half_life: float) -> None: + delta = max(0.0, current_time - self.last_update) + if delta > 0 and self.penalty > 0: + decay_factor = math.pow(2.0, -delta / half_life) + self.penalty *= decay_factor + if self.penalty < 1e-4: + self.penalty = 0.0 + self.last_update = current_time + + +class RouteFlapDamper: + """Manages routing penalties for flapping upstreams to guard load balancer topology.""" + + def __init__(self, config: Optional[FlapDampingConfig] = None) -> None: + self.config = config or FlapDampingConfig() + self._records: Dict[str, UpstreamTrackRecord] = {} + + def _get_or_create(self, upstream_id: str, now: float) -> UpstreamTrackRecord: + if upstream_id not in self._records: + self._records[upstream_id] = UpstreamTrackRecord(upstream_id, now) + return self._records[upstream_id] + + def record_state_change( + self, upstream_id: str, is_healthy: bool, timestamp: Optional[float] = None + ) -> UpstreamState: + """Register a health status transition for an upstream host. Flaps increment penalty.""" + now = time.time() if timestamp is None else timestamp + rec = self._get_or_create(upstream_id, now) + + # Decay existing penalty up to this point + rec.decay_to(now, self.config.half_life_seconds) + + # Detect oscillation (health transition from True -> False or False -> True) + if rec.last_health != is_healthy: + rec.penalty += self.config.penalty_per_flap + rec.total_flaps += 1 + rec.last_health = is_healthy + + # Evaluate suppression criteria + if rec.state == UpstreamState.SUPPRESSED: + # Check maximum suppression timeout + if rec.suppressed_at is not None and (now - rec.suppressed_at) >= self.config.max_suppress_seconds: + rec.state = UpstreamState.FLAPPING if rec.penalty > self.config.reuse_threshold else UpstreamState.HEALTHY + rec.suppressed_at = None + elif rec.penalty <= self.config.reuse_threshold: + rec.state = UpstreamState.HEALTHY if is_healthy else UpstreamState.FLAPPING + rec.suppressed_at = None + else: + if rec.penalty >= self.config.suppress_threshold: + rec.state = UpstreamState.SUPPRESSED + rec.suppressed_at = now + elif rec.penalty > self.config.reuse_threshold: + rec.state = UpstreamState.FLAPPING + else: + rec.state = UpstreamState.HEALTHY + + return rec.state + + def is_routable(self, upstream_id: str, timestamp: Optional[float] = None) -> bool: + """Check if an upstream target is permitted to receive ingress traffic.""" + if upstream_id not in self._records: + return True + + now = time.time() if timestamp is None else timestamp + rec = self._records[upstream_id] + rec.decay_to(now, self.config.half_life_seconds) + + if rec.state == UpstreamState.SUPPRESSED: + # Check if auto-reuse criteria or max duration has expired + if rec.suppressed_at is not None and (now - rec.suppressed_at) >= self.config.max_suppress_seconds: + rec.state = UpstreamState.FLAPPING if rec.penalty > self.config.reuse_threshold else UpstreamState.HEALTHY + rec.suppressed_at = None + return rec.last_health + if rec.penalty <= self.config.reuse_threshold: + rec.state = UpstreamState.HEALTHY if rec.last_health else UpstreamState.FLAPPING + rec.suppressed_at = None + return rec.last_health + return False + + return rec.last_health + + def get_upstream_status(self, upstream_id: str, timestamp: Optional[float] = None) -> dict: + """Retrieve diagnostic snapshot for a specific upstream.""" + now = time.time() if timestamp is None else timestamp + if upstream_id not in self._records: + return { + "upstream_id": upstream_id, + "state": UpstreamState.HEALTHY.value, + "penalty": 0.0, + "total_flaps": 0, + "routable": True, + } + + rec = self._records[upstream_id] + rec.decay_to(now, self.config.half_life_seconds) + routable = self.is_routable(upstream_id, now) + + return { + "upstream_id": upstream_id, + "state": rec.state.value, + "penalty": round(rec.penalty, 2), + "total_flaps": rec.total_flaps, + "routable": routable, + "suppressed": rec.state == UpstreamState.SUPPRESSED, + } + + def reset(self, upstream_id: Optional[str] = None) -> None: + """Reset penalties and suppressions for one or all upstreams.""" + if upstream_id is not None: + self._records.pop(upstream_id, None) + else: + self._records.clear() diff --git a/tests/test_flap_damper.py b/tests/test_flap_damper.py new file mode 100644 index 0000000..dd73a39 --- /dev/null +++ b/tests/test_flap_damper.py @@ -0,0 +1,124 @@ +"""Unit tests for RouteFlapDamper and RFC 2439 oscillation suppression.""" + +from __future__ import annotations + +import pytest +from scripts.flap_damper import FlapDampingConfig, RouteFlapDamper, UpstreamState + + +def test_flap_damping_config_validation() -> None: + # Valid config + cfg = FlapDampingConfig(penalty_per_flap=500.0, half_life_seconds=10.0, suppress_threshold=1500.0, reuse_threshold=500.0) + assert cfg.penalty_per_flap == 500.0 + + # Invalid thresholds (suppress <= reuse) + with pytest.raises(ValueError, match="strictly greater"): + FlapDampingConfig(suppress_threshold=500.0, reuse_threshold=500.0) + + # Invalid penalty + with pytest.raises(ValueError, match="penalty_per_flap"): + FlapDampingConfig(penalty_per_flap=-10.0) + + +def test_flap_damping_stable_upstream() -> None: + damper = RouteFlapDamper() + assert damper.is_routable("api-1") is True + + # Reporting same healthy state causes no flap penalty + state = damper.record_state_change("api-1", True, timestamp=1000.0) + assert state == UpstreamState.HEALTHY + + status = damper.get_upstream_status("api-1", timestamp=1000.0) + assert status["penalty"] == 0.0 + assert status["total_flaps"] == 0 + assert status["routable"] is True + + +def test_flap_damping_escalation_to_suppressed() -> None: + cfg = FlapDampingConfig( + penalty_per_flap=1100.0, + half_life_seconds=20.0, + suppress_threshold=2000.0, + reuse_threshold=500.0, + ) + damper = RouteFlapDamper(cfg) + t = 1000.0 + + # First flap: True -> False (+1100 penalty) + state = damper.record_state_change("db-primary", False, timestamp=t) + assert state == UpstreamState.FLAPPING + assert damper.is_routable("db-primary", timestamp=t) is False # unhealthy + + # Second flap: False -> True (+1100 penalty, total > 2000 >= suppress_threshold) + t += 1.0 + state = damper.record_state_change("db-primary", True, timestamp=t) + assert state == UpstreamState.SUPPRESSED + + # Even though health is True, suppression blocks routing! + assert damper.is_routable("db-primary", timestamp=t) is False + status = damper.get_upstream_status("db-primary", timestamp=t) + assert status["suppressed"] is True + assert status["total_flaps"] == 2 + + +def test_flap_damping_exponential_decay_and_reuse() -> None: + cfg = FlapDampingConfig( + penalty_per_flap=1200.0, + half_life_seconds=10.0, + suppress_threshold=2000.0, + reuse_threshold=600.0, + max_suppress_seconds=100.0, + ) + damper = RouteFlapDamper(cfg) + t = 1000.0 + + # Suppress upstream with 2 rapid flaps (>2000 penalty) + damper.record_state_change("srv-1", False, timestamp=t) + t += 1.0 + damper.record_state_change("srv-1", True, timestamp=t) + assert damper.get_upstream_status("srv-1", timestamp=t)["suppressed"] is True + + # Advance 10 seconds (1 half-life): penalty decays from ~2319 to ~1160 + t += 10.0 + status = damper.get_upstream_status("srv-1", timestamp=t) + assert 1050.0 < status["penalty"] < 1250.0 + assert status["suppressed"] is True # still > reuse_threshold (600) + + # Advance another 10 seconds (2 half-lives total): penalty decays from ~1000 to ~500 (< 600 reuse_threshold) + t += 10.0 + status = damper.get_upstream_status("srv-1", timestamp=t) + assert status["penalty"] < 600.0 + assert status["suppressed"] is False + assert status["routable"] is True + + +def test_flap_damping_max_suppress_timeout() -> None: + cfg = FlapDampingConfig( + penalty_per_flap=2000.0, + half_life_seconds=100.0, + suppress_threshold=2000.0, + reuse_threshold=500.0, + max_suppress_seconds=30.0, + ) + damper = RouteFlapDamper(cfg) + t = 1000.0 + + damper.record_state_change("payment-gw", False, timestamp=t) + t += 1.0 + damper.record_state_change("payment-gw", True, timestamp=t) + assert damper.get_upstream_status("payment-gw", timestamp=t)["suppressed"] is True + + # Advance 35 seconds (exceeding max_suppress_seconds=30) + t += 35.0 + status = damper.get_upstream_status("payment-gw", timestamp=t) + assert status["suppressed"] is False + assert status["routable"] is True + + +def test_flap_damping_reset() -> None: + damper = RouteFlapDamper() + damper.record_state_change("redis-1", False, timestamp=1000.0) + assert damper.get_upstream_status("redis-1")["total_flaps"] == 1 + + damper.reset("redis-1") + assert damper.get_upstream_status("redis-1")["total_flaps"] == 0