Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
162 changes: 162 additions & 0 deletions scripts/flap_damper.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,162 @@
"""RFC 2439-inspired Upstream Route Flap Damping and Oscillation Damper for RouteKernel."""

from __future__ import annotations

import math
import time
from dataclasses import dataclass
from enum import Enum
from typing import Dict, Optional


class UpstreamState(str, Enum):
HEALTHY = "HEALTHY"
FLAPPING = "FLAPPING"
SUPPRESSED = "SUPPRESSED"


@dataclass(frozen=True)
class FlapDampingConfig:
"""Configuration governing penalty accumulation, decay half-life, and suppression thresholds."""

penalty_per_flap: float = 1000.0
half_life_seconds: float = 15.0
suppress_threshold: float = 2000.0
reuse_threshold: float = 750.0
max_suppress_seconds: float = 60.0

def __post_init__(self) -> None:
if self.penalty_per_flap <= 0:
raise ValueError("penalty_per_flap must be > 0")
if self.half_life_seconds <= 0:
raise ValueError("half_life_seconds must be > 0")
if self.suppress_threshold <= self.reuse_threshold:
raise ValueError("suppress_threshold must be strictly greater than reuse_threshold")
if self.reuse_threshold < 0:
raise ValueError("reuse_threshold must be >= 0")
if self.max_suppress_seconds <= 0:
raise ValueError("max_suppress_seconds must be > 0")


class UpstreamTrackRecord:
def __init__(self, upstream_id: str, initial_timestamp: float) -> None:
self.upstream_id = upstream_id
self.penalty = 0.0
self.last_update = initial_timestamp
self.last_health = True
self.state = UpstreamState.HEALTHY
self.suppressed_at: Optional[float] = None
self.total_flaps = 0

def decay_to(self, current_time: float, half_life: float) -> None:
delta = max(0.0, current_time - self.last_update)
if delta > 0 and self.penalty > 0:
decay_factor = math.pow(2.0, -delta / half_life)
self.penalty *= decay_factor
if self.penalty < 1e-4:
self.penalty = 0.0
self.last_update = current_time


class RouteFlapDamper:
"""Manages routing penalties for flapping upstreams to guard load balancer topology."""

def __init__(self, config: Optional[FlapDampingConfig] = None) -> None:
self.config = config or FlapDampingConfig()
self._records: Dict[str, UpstreamTrackRecord] = {}

def _get_or_create(self, upstream_id: str, now: float) -> UpstreamTrackRecord:
if upstream_id not in self._records:
self._records[upstream_id] = UpstreamTrackRecord(upstream_id, now)
return self._records[upstream_id]

def record_state_change(
self, upstream_id: str, is_healthy: bool, timestamp: Optional[float] = None
) -> UpstreamState:
"""Register a health status transition for an upstream host. Flaps increment penalty."""
now = time.time() if timestamp is None else timestamp
rec = self._get_or_create(upstream_id, now)

# Decay existing penalty up to this point
rec.decay_to(now, self.config.half_life_seconds)

# Detect oscillation (health transition from True -> False or False -> True)
if rec.last_health != is_healthy:
rec.penalty += self.config.penalty_per_flap
rec.total_flaps += 1
rec.last_health = is_healthy

# Evaluate suppression criteria
if rec.state == UpstreamState.SUPPRESSED:
# Check maximum suppression timeout
if rec.suppressed_at is not None and (now - rec.suppressed_at) >= self.config.max_suppress_seconds:
rec.state = UpstreamState.FLAPPING if rec.penalty > self.config.reuse_threshold else UpstreamState.HEALTHY
rec.suppressed_at = None
elif rec.penalty <= self.config.reuse_threshold:
rec.state = UpstreamState.HEALTHY if is_healthy else UpstreamState.FLAPPING
rec.suppressed_at = None
else:
if rec.penalty >= self.config.suppress_threshold:
rec.state = UpstreamState.SUPPRESSED
rec.suppressed_at = now
elif rec.penalty > self.config.reuse_threshold:
rec.state = UpstreamState.FLAPPING
else:
rec.state = UpstreamState.HEALTHY

return rec.state

def is_routable(self, upstream_id: str, timestamp: Optional[float] = None) -> bool:
"""Check if an upstream target is permitted to receive ingress traffic."""
if upstream_id not in self._records:
return True

now = time.time() if timestamp is None else timestamp
rec = self._records[upstream_id]
rec.decay_to(now, self.config.half_life_seconds)

if rec.state == UpstreamState.SUPPRESSED:
# Check if auto-reuse criteria or max duration has expired
if rec.suppressed_at is not None and (now - rec.suppressed_at) >= self.config.max_suppress_seconds:
rec.state = UpstreamState.FLAPPING if rec.penalty > self.config.reuse_threshold else UpstreamState.HEALTHY
rec.suppressed_at = None
return rec.last_health
if rec.penalty <= self.config.reuse_threshold:
rec.state = UpstreamState.HEALTHY if rec.last_health else UpstreamState.FLAPPING
rec.suppressed_at = None
return rec.last_health
return False

return rec.last_health

def get_upstream_status(self, upstream_id: str, timestamp: Optional[float] = None) -> dict:
"""Retrieve diagnostic snapshot for a specific upstream."""
now = time.time() if timestamp is None else timestamp
if upstream_id not in self._records:
return {
"upstream_id": upstream_id,
"state": UpstreamState.HEALTHY.value,
"penalty": 0.0,
"total_flaps": 0,
"routable": True,
}

rec = self._records[upstream_id]
rec.decay_to(now, self.config.half_life_seconds)
routable = self.is_routable(upstream_id, now)

return {
"upstream_id": upstream_id,
"state": rec.state.value,
"penalty": round(rec.penalty, 2),
"total_flaps": rec.total_flaps,
"routable": routable,
"suppressed": rec.state == UpstreamState.SUPPRESSED,
}

def reset(self, upstream_id: Optional[str] = None) -> None:
"""Reset penalties and suppressions for one or all upstreams."""
if upstream_id is not None:
self._records.pop(upstream_id, None)
else:
self._records.clear()
124 changes: 124 additions & 0 deletions tests/test_flap_damper.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
"""Unit tests for RouteFlapDamper and RFC 2439 oscillation suppression."""

from __future__ import annotations

import pytest
from scripts.flap_damper import FlapDampingConfig, RouteFlapDamper, UpstreamState


def test_flap_damping_config_validation() -> None:
# Valid config
cfg = FlapDampingConfig(penalty_per_flap=500.0, half_life_seconds=10.0, suppress_threshold=1500.0, reuse_threshold=500.0)
assert cfg.penalty_per_flap == 500.0

# Invalid thresholds (suppress <= reuse)
with pytest.raises(ValueError, match="strictly greater"):
FlapDampingConfig(suppress_threshold=500.0, reuse_threshold=500.0)

# Invalid penalty
with pytest.raises(ValueError, match="penalty_per_flap"):
FlapDampingConfig(penalty_per_flap=-10.0)


def test_flap_damping_stable_upstream() -> None:
damper = RouteFlapDamper()
assert damper.is_routable("api-1") is True

# Reporting same healthy state causes no flap penalty
state = damper.record_state_change("api-1", True, timestamp=1000.0)
assert state == UpstreamState.HEALTHY

status = damper.get_upstream_status("api-1", timestamp=1000.0)
assert status["penalty"] == 0.0
assert status["total_flaps"] == 0
assert status["routable"] is True


def test_flap_damping_escalation_to_suppressed() -> None:
cfg = FlapDampingConfig(
penalty_per_flap=1100.0,
half_life_seconds=20.0,
suppress_threshold=2000.0,
reuse_threshold=500.0,
)
damper = RouteFlapDamper(cfg)
t = 1000.0

# First flap: True -> False (+1100 penalty)
state = damper.record_state_change("db-primary", False, timestamp=t)
assert state == UpstreamState.FLAPPING
assert damper.is_routable("db-primary", timestamp=t) is False # unhealthy

# Second flap: False -> True (+1100 penalty, total > 2000 >= suppress_threshold)
t += 1.0
state = damper.record_state_change("db-primary", True, timestamp=t)
assert state == UpstreamState.SUPPRESSED

# Even though health is True, suppression blocks routing!
assert damper.is_routable("db-primary", timestamp=t) is False
status = damper.get_upstream_status("db-primary", timestamp=t)
assert status["suppressed"] is True
assert status["total_flaps"] == 2


def test_flap_damping_exponential_decay_and_reuse() -> None:
cfg = FlapDampingConfig(
penalty_per_flap=1200.0,
half_life_seconds=10.0,
suppress_threshold=2000.0,
reuse_threshold=600.0,
max_suppress_seconds=100.0,
)
damper = RouteFlapDamper(cfg)
t = 1000.0

# Suppress upstream with 2 rapid flaps (>2000 penalty)
damper.record_state_change("srv-1", False, timestamp=t)
t += 1.0
damper.record_state_change("srv-1", True, timestamp=t)
assert damper.get_upstream_status("srv-1", timestamp=t)["suppressed"] is True

# Advance 10 seconds (1 half-life): penalty decays from ~2319 to ~1160
t += 10.0
status = damper.get_upstream_status("srv-1", timestamp=t)
assert 1050.0 < status["penalty"] < 1250.0
assert status["suppressed"] is True # still > reuse_threshold (600)

# Advance another 10 seconds (2 half-lives total): penalty decays from ~1000 to ~500 (< 600 reuse_threshold)
t += 10.0
status = damper.get_upstream_status("srv-1", timestamp=t)
assert status["penalty"] < 600.0
assert status["suppressed"] is False
assert status["routable"] is True


def test_flap_damping_max_suppress_timeout() -> None:
cfg = FlapDampingConfig(
penalty_per_flap=2000.0,
half_life_seconds=100.0,
suppress_threshold=2000.0,
reuse_threshold=500.0,
max_suppress_seconds=30.0,
)
damper = RouteFlapDamper(cfg)
t = 1000.0

damper.record_state_change("payment-gw", False, timestamp=t)
t += 1.0
damper.record_state_change("payment-gw", True, timestamp=t)
assert damper.get_upstream_status("payment-gw", timestamp=t)["suppressed"] is True

# Advance 35 seconds (exceeding max_suppress_seconds=30)
t += 35.0
status = damper.get_upstream_status("payment-gw", timestamp=t)
assert status["suppressed"] is False
assert status["routable"] is True


def test_flap_damping_reset() -> None:
damper = RouteFlapDamper()
damper.record_state_change("redis-1", False, timestamp=1000.0)
assert damper.get_upstream_status("redis-1")["total_flaps"] == 1

damper.reset("redis-1")
assert damper.get_upstream_status("redis-1")["total_flaps"] == 0
Loading