diff --git a/scripts/circuit_breaker.py b/scripts/circuit_breaker.py new file mode 100644 index 0000000..fa24ed7 --- /dev/null +++ b/scripts/circuit_breaker.py @@ -0,0 +1,183 @@ +""" +RouteKernel: Upstream Circuit Breaker & Adaptive Health Probe Rotary. +Implements a 3-state resilience circuit breaker with consecutive failure tripping, +exponential cooldown backoff, and canary probe verification. +""" + +from __future__ import annotations + +import time +from enum import Enum +from typing import Callable, Optional, TypeVar, Any, Dict + +T = TypeVar("T") + + +class CircuitState(str, Enum): + CLOSED = "closed" + OPEN = "open" + HALF_OPEN = "half_open" + + +class CircuitOpenError(Exception): + """Raised when an execution call is rejected because the circuit is open.""" + + def __init__(self, message: str, cooldown_remaining_sec: float): + super().__init__(message) + self.cooldown_remaining_sec = max(0.0, cooldown_remaining_sec) + + +class UpstreamCircuitBreaker: + """ + Guards upstream services against cascading overload by tripping on repeated failures + and validating recovery via canary probes. + """ + + def __init__( + self, + failure_threshold: int = 5, + recovery_timeout_sec: float = 10.0, + canary_success_threshold: int = 2, + max_recovery_timeout_sec: float = 120.0, + backoff_multiplier: float = 2.0, + ): + if failure_threshold <= 0: + raise ValueError("failure_threshold must be greater than zero") + if recovery_timeout_sec <= 0: + raise ValueError("recovery_timeout_sec must be positive") + if canary_success_threshold <= 0: + raise ValueError("canary_success_threshold must be greater than zero") + + self.failure_threshold = failure_threshold + self.base_recovery_timeout_sec = recovery_timeout_sec + self.current_recovery_timeout_sec = recovery_timeout_sec + self.canary_success_threshold = canary_success_threshold + self.max_recovery_timeout_sec = max_recovery_timeout_sec + self.backoff_multiplier = backoff_multiplier + + self._state: CircuitState = CircuitState.CLOSED + self._consecutive_failures: int = 0 + self._consecutive_canary_successes: int = 0 + self._last_state_change: float = time.time() + self._last_failure_time: Optional[float] = None + self._trip_count: int = 0 + self._total_calls: int = 0 + + @property + def state(self) -> CircuitState: + self._evaluate_state_transition() + return self._state + + def _evaluate_state_transition(self) -> None: + """Transitions OPEN to HALF_OPEN when the recovery cooldown expires.""" + if self._state == CircuitState.OPEN: + elapsed = time.time() - self._last_state_change + if elapsed >= self.current_recovery_timeout_sec: + self._state = CircuitState.HALF_OPEN + self._consecutive_canary_successes = 0 + self._last_state_change = time.time() + + def allow_request(self) -> bool: + """Checks if a request is permitted under the current circuit state.""" + self._evaluate_state_transition() + + if self._state == CircuitState.CLOSED: + return True + + if self._state == CircuitState.HALF_OPEN: + # In half-open, permit canary probes up to threshold + return True + + # OPEN state: reject + return False + + def record_success(self) -> None: + """Records a successful execution.""" + self._total_calls += 1 + self._evaluate_state_transition() + + if self._state == CircuitState.HALF_OPEN: + self._consecutive_canary_successes += 1 + if self._consecutive_canary_successes >= self.canary_success_threshold: + # Fully recovered! + self._state = CircuitState.CLOSED + self._consecutive_failures = 0 + self._consecutive_canary_successes = 0 + self.current_recovery_timeout_sec = self.base_recovery_timeout_sec + self._last_state_change = time.time() + elif self._state == CircuitState.CLOSED: + self._consecutive_failures = 0 + + def record_failure(self) -> None: + """Records a failed execution.""" + self._total_calls += 1 + now = time.time() + self._last_failure_time = now + + if self._state == CircuitState.CLOSED: + self._consecutive_failures += 1 + if self._consecutive_failures >= self.failure_threshold: + # Trip to OPEN + self._trip_circuit(now) + elif self._state == CircuitState.HALF_OPEN: + # Canary probe failed: immediately trip back to OPEN with backoff + self.current_recovery_timeout_sec = min( + self.max_recovery_timeout_sec, + self.current_recovery_timeout_sec * self.backoff_multiplier, + ) + self._trip_circuit(now) + + def _trip_circuit(self, now: float) -> None: + self._state = CircuitState.OPEN + self._last_state_change = now + self._consecutive_canary_successes = 0 + self._trip_count += 1 + + def execute(self, func: Callable[..., T], *args: Any, **kwargs: Any) -> T: + """Executes func protected by the circuit breaker.""" + if not self.allow_request(): + remaining = max( + 0.0, + self.current_recovery_timeout_sec - (time.time() - self._last_state_change), + ) + raise CircuitOpenError( + f"Circuit is OPEN. Cooldown remaining: {remaining:.2f}s", + cooldown_remaining_sec=remaining, + ) + + try: + result = func(*args, **kwargs) + self.record_success() + return result + except Exception: + self.record_failure() + raise + + def get_telemetry(self) -> Dict[str, Any]: + """Returns current operational telemetry snapshot.""" + self._evaluate_state_transition() + remaining = 0.0 + if self._state == CircuitState.OPEN: + remaining = max( + 0.0, + self.current_recovery_timeout_sec - (time.time() - self._last_state_change), + ) + + return { + "state": self._state.value, + "consecutive_failures": self._consecutive_failures, + "consecutive_canary_successes": self._consecutive_canary_successes, + "trip_count": self._trip_count, + "total_calls": self._total_calls, + "cooldown_remaining_sec": round(remaining, 3), + "current_recovery_timeout_sec": self.current_recovery_timeout_sec, + } + + def reset(self) -> None: + """Resets the circuit breaker to clean closed state.""" + self._state = CircuitState.CLOSED + self._consecutive_failures = 0 + self._consecutive_canary_successes = 0 + self.current_recovery_timeout_sec = self.base_recovery_timeout_sec + self._last_state_change = time.time() + self._last_failure_time = None diff --git a/tests/test_circuit_breaker.py b/tests/test_circuit_breaker.py new file mode 100644 index 0000000..2326655 --- /dev/null +++ b/tests/test_circuit_breaker.py @@ -0,0 +1,114 @@ +""" +Unit tests for RouteKernel Upstream Circuit Breaker & Adaptive Health Probe Rotary. +""" + +import sys +from pathlib import Path +import time +import pytest + +# Ensure scripts dir is on sys.path +SCRIPTS_DIR = Path(__file__).resolve().parent.parent / "scripts" +if str(SCRIPTS_DIR) not in sys.path: + sys.path.insert(0, str(SCRIPTS_DIR)) + +from circuit_breaker import UpstreamCircuitBreaker, CircuitState, CircuitOpenError + + +def test_circuit_breaker_normal_closed_execution(): + cb = UpstreamCircuitBreaker(failure_threshold=3, recovery_timeout_sec=0.1) + assert cb.state == CircuitState.CLOSED + + # Successful call + result = cb.execute(lambda x, y: x + y, 10, 20) + assert result == 30 + assert cb.state == CircuitState.CLOSED + assert cb.get_telemetry()["consecutive_failures"] == 0 + + +def test_circuit_breaker_trips_to_open_on_failures(): + cb = UpstreamCircuitBreaker(failure_threshold=3, recovery_timeout_sec=0.2) + + def failing_call(): + raise RuntimeError("upstream timeout") + + # 1st failure + with pytest.raises(RuntimeError): + cb.execute(failing_call) + assert cb.state == CircuitState.CLOSED + + # 2nd failure + with pytest.raises(RuntimeError): + cb.execute(failing_call) + assert cb.state == CircuitState.CLOSED + + # 3rd failure: trips to OPEN! + with pytest.raises(RuntimeError): + cb.execute(failing_call) + assert cb.state == CircuitState.OPEN + assert cb.get_telemetry()["trip_count"] == 1 + + # 4th call is rejected immediately with CircuitOpenError without invoking target + with pytest.raises(CircuitOpenError) as exc_info: + cb.execute(lambda: "should not run") + assert exc_info.value.cooldown_remaining_sec > 0 + + +def test_circuit_breaker_half_open_recovery(): + # Short cooldown for test speed + cb = UpstreamCircuitBreaker( + failure_threshold=2, + recovery_timeout_sec=0.05, + canary_success_threshold=2, + ) + + def failing_call(): + raise ValueError("error") + + # Trip to OPEN + for _ in range(2): + with pytest.raises(ValueError): + cb.execute(failing_call) + assert cb.state == CircuitState.OPEN + + # Sleep past recovery cooldown + time.sleep(0.06) + + # State transitions to HALF_OPEN + assert cb.state == CircuitState.HALF_OPEN + + # 1st canary probe succeeds + cb.execute(lambda: "ok1") + assert cb.state == CircuitState.HALF_OPEN + + # 2nd canary probe succeeds: resets to CLOSED + cb.execute(lambda: "ok2") + assert cb.state == CircuitState.CLOSED + assert cb.get_telemetry()["consecutive_failures"] == 0 + + +def test_circuit_breaker_half_open_failure_backoff(): + cb = UpstreamCircuitBreaker( + failure_threshold=2, + recovery_timeout_sec=0.05, + canary_success_threshold=2, + backoff_multiplier=2.0, + ) + + # Trip to OPEN + for _ in range(2): + with pytest.raises(ZeroDivisionError): + cb.execute(lambda: 1 / 0) + assert cb.state == CircuitState.OPEN + + # Wait for cooldown to expire + time.sleep(0.06) + assert cb.state == CircuitState.HALF_OPEN + + # Canary probe fails: trips back to OPEN with doubled cooldown (0.10s) + with pytest.raises(ZeroDivisionError): + cb.execute(lambda: 1 / 0) + + assert cb.state == CircuitState.OPEN + assert cb.get_telemetry()["current_recovery_timeout_sec"] == 0.10 + assert cb.get_telemetry()["trip_count"] == 2