129 lines
4.1 KiB
Python
129 lines
4.1 KiB
Python
|
|
"""通用熔断器:保护下游资源免受连续失败冲击。
|
|||
|
|
|
|||
|
|
状态机:
|
|||
|
|
CLOSED → 连续失败达 failure_threshold → OPEN
|
|||
|
|
OPEN → 经过 recovery_timeout → HALF_OPEN
|
|||
|
|
HALF_OPEN → 一次成功 → CLOSED / 一次失败 → OPEN
|
|||
|
|
|
|||
|
|
供 FlowOrchestrator 对 DB 校验失败、WeChat 启动失败等做熔断保护。
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
from __future__ import annotations
|
|||
|
|
|
|||
|
|
import enum
|
|||
|
|
import logging
|
|||
|
|
import time
|
|||
|
|
import threading
|
|||
|
|
|
|||
|
|
logger = logging.getLogger("woc-bridge")
|
|||
|
|
|
|||
|
|
|
|||
|
|
class CircuitState(enum.Enum):
|
|||
|
|
"""熔断器状态。"""
|
|||
|
|
CLOSED = "closed" # 正常工作,请求通过
|
|||
|
|
OPEN = "open" # 熔断中,请求拒绝
|
|||
|
|
HALF_OPEN = "half_open" # 半开,允许一次试探请求
|
|||
|
|
|
|||
|
|
|
|||
|
|
class CircuitBreaker:
|
|||
|
|
"""熔断器。
|
|||
|
|
|
|||
|
|
Attributes:
|
|||
|
|
name: 熔断器名称(如 "db_verify")
|
|||
|
|
failure_threshold: 连续失败次数阈值,默认 5
|
|||
|
|
recovery_timeout: 熔断恢复秒数,默认 30
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
def __init__(
|
|||
|
|
self,
|
|||
|
|
name: str,
|
|||
|
|
failure_threshold: int = 5,
|
|||
|
|
recovery_timeout: float = 30.0,
|
|||
|
|
) -> None:
|
|||
|
|
"""初始化熔断器。
|
|||
|
|
|
|||
|
|
Args:
|
|||
|
|
name: 熔断器名称
|
|||
|
|
failure_threshold: 连续失败次数阈值
|
|||
|
|
recovery_timeout: 熔断恢复秒数
|
|||
|
|
"""
|
|||
|
|
self.name = name
|
|||
|
|
self.failure_threshold = failure_threshold
|
|||
|
|
self.recovery_timeout = recovery_timeout
|
|||
|
|
self._state = CircuitState.CLOSED
|
|||
|
|
self._fail_count = 0
|
|||
|
|
self._last_failure_time: float = 0.0
|
|||
|
|
self._lock = threading.Lock()
|
|||
|
|
|
|||
|
|
@property
|
|||
|
|
def state(self) -> CircuitState:
|
|||
|
|
"""当前状态。
|
|||
|
|
|
|||
|
|
OPEN 状态超过 recovery_timeout 自动转 HALF_OPEN。
|
|||
|
|
"""
|
|||
|
|
with self._lock:
|
|||
|
|
if (
|
|||
|
|
self._state == CircuitState.OPEN
|
|||
|
|
and time.monotonic() - self._last_failure_time >= self.recovery_timeout
|
|||
|
|
):
|
|||
|
|
self._state = CircuitState.HALF_OPEN
|
|||
|
|
logger.info(
|
|||
|
|
"[breaker:%s] OPEN -> HALF_OPEN (after %.0fs)",
|
|||
|
|
self.name, self.recovery_timeout,
|
|||
|
|
)
|
|||
|
|
return self._state
|
|||
|
|
|
|||
|
|
def allow(self) -> bool:
|
|||
|
|
"""是否允许请求通过。
|
|||
|
|
|
|||
|
|
CLOSED / HALF_OPEN 允许;OPEN 拒绝。
|
|||
|
|
HALF_OPEN 时只允许一次试探(调用方需配合 record_success/failure)。
|
|||
|
|
"""
|
|||
|
|
return self.state in (CircuitState.CLOSED, CircuitState.HALF_OPEN)
|
|||
|
|
|
|||
|
|
def record_success(self) -> None:
|
|||
|
|
"""记录一次成功。
|
|||
|
|
|
|||
|
|
CLOSED: 重置失败计数。
|
|||
|
|
HALF_OPEN: 转 CLOSED,恢复服务。
|
|||
|
|
"""
|
|||
|
|
with self._lock:
|
|||
|
|
if self._state == CircuitState.HALF_OPEN:
|
|||
|
|
self._state = CircuitState.CLOSED
|
|||
|
|
logger.info("[breaker:%s] HALF_OPEN -> CLOSED", self.name)
|
|||
|
|
self._fail_count = 0
|
|||
|
|
|
|||
|
|
def record_failure(self) -> None:
|
|||
|
|
"""记录一次失败。
|
|||
|
|
|
|||
|
|
CLOSED: 失败计数 +1,达阈值转 OPEN。
|
|||
|
|
HALF_OPEN: 立即转 OPEN。
|
|||
|
|
OPEN: 更新 last_failure_time(延长熔断)。
|
|||
|
|
"""
|
|||
|
|
with self._lock:
|
|||
|
|
self._last_failure_time = time.monotonic()
|
|||
|
|
if self._state == CircuitState.HALF_OPEN:
|
|||
|
|
self._state = CircuitState.OPEN
|
|||
|
|
logger.warning(
|
|||
|
|
"[breaker:%s] HALF_OPEN -> OPEN (probe failed)", self.name
|
|||
|
|
)
|
|||
|
|
return
|
|||
|
|
self._fail_count += 1
|
|||
|
|
if (
|
|||
|
|
self._state == CircuitState.CLOSED
|
|||
|
|
and self._fail_count >= self.failure_threshold
|
|||
|
|
):
|
|||
|
|
self._state = CircuitState.OPEN
|
|||
|
|
logger.warning(
|
|||
|
|
"[breaker:%s] CLOSED -> OPEN (fail_count=%d)",
|
|||
|
|
self.name, self._fail_count,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def reset(self) -> None:
|
|||
|
|
"""手动重置熔断器到 CLOSED 状态。"""
|
|||
|
|
with self._lock:
|
|||
|
|
self._state = CircuitState.CLOSED
|
|||
|
|
self._fail_count = 0
|
|||
|
|
self._last_failure_time = 0.0
|
|||
|
|
logger.info("[breaker:%s] reset to CLOSED", self.name)
|