WechatOnCloud/bridge/woc_bridge/ui/circuit_breaker.py

129 lines
4.1 KiB
Python
Raw Normal View History

"""通用熔断器:保护下游资源免受连续失败冲击。
状态机
CLOSED 连续失败达 failure_threshold OPEN
OPEN 经过 recovery_timeout HALF_OPEN
HALF_OPEN 一次成功 CLOSED / 一次失败 OPEN
FlowOrchestrator DB 校验失败WeChat 启动失败等做熔断保护
"""
from __future__ import annotations
import enum
import logging
import time
import threading
logger = logging.getLogger("woc-bridge")
class CircuitState(enum.Enum):
"""熔断器状态。"""
CLOSED = "closed" # 正常工作,请求通过
OPEN = "open" # 熔断中,请求拒绝
HALF_OPEN = "half_open" # 半开,允许一次试探请求
class CircuitBreaker:
"""熔断器。
Attributes:
name: 熔断器名称 "db_verify"
failure_threshold: 连续失败次数阈值默认 5
recovery_timeout: 熔断恢复秒数默认 30
"""
def __init__(
self,
name: str,
failure_threshold: int = 5,
recovery_timeout: float = 30.0,
) -> None:
"""初始化熔断器。
Args:
name: 熔断器名称
failure_threshold: 连续失败次数阈值
recovery_timeout: 熔断恢复秒数
"""
self.name = name
self.failure_threshold = failure_threshold
self.recovery_timeout = recovery_timeout
self._state = CircuitState.CLOSED
self._fail_count = 0
self._last_failure_time: float = 0.0
self._lock = threading.Lock()
@property
def state(self) -> CircuitState:
"""当前状态。
OPEN 状态超过 recovery_timeout 自动转 HALF_OPEN
"""
with self._lock:
if (
self._state == CircuitState.OPEN
and time.monotonic() - self._last_failure_time >= self.recovery_timeout
):
self._state = CircuitState.HALF_OPEN
logger.info(
"[breaker:%s] OPEN -> HALF_OPEN (after %.0fs)",
self.name, self.recovery_timeout,
)
return self._state
def allow(self) -> bool:
"""是否允许请求通过。
CLOSED / HALF_OPEN 允许OPEN 拒绝
HALF_OPEN 时只允许一次试探调用方需配合 record_success/failure
"""
return self.state in (CircuitState.CLOSED, CircuitState.HALF_OPEN)
def record_success(self) -> None:
"""记录一次成功。
CLOSED: 重置失败计数
HALF_OPEN: CLOSED恢复服务
"""
with self._lock:
if self._state == CircuitState.HALF_OPEN:
self._state = CircuitState.CLOSED
logger.info("[breaker:%s] HALF_OPEN -> CLOSED", self.name)
self._fail_count = 0
def record_failure(self) -> None:
"""记录一次失败。
CLOSED: 失败计数 +1达阈值转 OPEN
HALF_OPEN: 立即转 OPEN
OPEN: 更新 last_failure_time延长熔断
"""
with self._lock:
self._last_failure_time = time.monotonic()
if self._state == CircuitState.HALF_OPEN:
self._state = CircuitState.OPEN
logger.warning(
"[breaker:%s] HALF_OPEN -> OPEN (probe failed)", self.name
)
return
self._fail_count += 1
if (
self._state == CircuitState.CLOSED
and self._fail_count >= self.failure_threshold
):
self._state = CircuitState.OPEN
logger.warning(
"[breaker:%s] CLOSED -> OPEN (fail_count=%d)",
self.name, self._fail_count,
)
def reset(self) -> None:
"""手动重置熔断器到 CLOSED 状态。"""
with self._lock:
self._state = CircuitState.CLOSED
self._fail_count = 0
self._last_failure_time = 0.0
logger.info("[breaker:%s] reset to CLOSED", self.name)