Kris
d52e9b518d
新增trace_ctx实现链路ID上下文管理与日志注入，添加boot.py启动管理器、startup.py启动任务执行器、backoff.py退避策略工具以及trace.py启动链路追踪模块，完善channel运行时基础能力
124 lines
4.1 KiB
Python
124 lines
4.1 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
import random
|
|
import time
|
|
from dataclasses import dataclass
|
|
from typing import Any
|
|
from collections.abc import Awaitable, Callable
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
BackoffFn = Callable[[int], float]
|
|
|
|
|
|
@dataclass
|
|
class BackoffConfig:
|
|
base_delay: float = 5.0
|
|
max_delay: float = 300.0
|
|
exponent: float = 2.0
|
|
jitter: bool = True
|
|
jitter_factor: float = 0.1
|
|
max_retries: int = 10
|
|
|
|
def compute_delay(self, attempt: int) -> float:
|
|
delay = self.base_delay * (self.exponent**attempt)
|
|
delay = min(delay, self.max_delay)
|
|
if self.jitter:
|
|
jitter_amount = delay * self.jitter_factor
|
|
delay += random.uniform(-jitter_amount, jitter_amount)
|
|
delay = max(delay, 0.01)
|
|
return delay
|
|
|
|
|
|
class ErrorBackoff:
|
|
"""错误退避控制器 — 管理重试逻辑与指数退避。
|
|
|
|
使用按 key 分锁策略，不同 key 之间的退避操作互不阻塞，
|
|
避免粗粒度单锁成为高并发场景下的全局瓶颈。
|
|
"""
|
|
|
|
def __init__(self, config: BackoffConfig | None = None):
|
|
self.config = config or BackoffConfig()
|
|
self._attempts: dict[str, int] = {}
|
|
self._last_error: dict[str, float] = {}
|
|
self._locks: dict[str, asyncio.Lock] = {}
|
|
self._global_lock = asyncio.Lock()
|
|
|
|
def _get_lock(self, key: str) -> asyncio.Lock:
|
|
"""获取指定 key 的独立锁，必要时惰性创建。"""
|
|
lock = self._locks.get(key)
|
|
if lock is None:
|
|
lock = asyncio.Lock()
|
|
self._locks[key] = lock
|
|
return lock
|
|
|
|
async def execute(
|
|
self,
|
|
key: str,
|
|
operation: Callable[[], Awaitable[Any]],
|
|
*,
|
|
on_retry: Callable[[int, Exception], Awaitable[None]] | None = None,
|
|
) -> Any:
|
|
from yuxi.channel.errors import _infer_severity
|
|
from yuxi.channel.protocols import ErrorSeverity
|
|
|
|
lock = self._get_lock(key)
|
|
async with lock:
|
|
attempt = self._attempts.get(key, 0)
|
|
|
|
for i in range(self.config.max_retries + 1):
|
|
try:
|
|
result = await operation()
|
|
self._attempts[key] = 0
|
|
self._last_error.pop(key, None)
|
|
return result
|
|
except Exception as e:
|
|
self._attempts[key] = attempt + i + 1
|
|
self._last_error[key] = time.time()
|
|
|
|
if _infer_severity(e) == ErrorSeverity.FATAL:
|
|
logger.error("Operation '%s' failed with fatal error: %s", key, e)
|
|
raise
|
|
|
|
if i >= self.config.max_retries:
|
|
logger.error("Operation '%s' failed after %d retries: %s", key, self.config.max_retries, e)
|
|
raise
|
|
|
|
delay = self.config.compute_delay(attempt + i)
|
|
logger.warning(
|
|
"Operation '%s' failed (attempt %d/%d), retrying in %.2fs: %s",
|
|
key,
|
|
i + 1,
|
|
self.config.max_retries + 1,
|
|
delay,
|
|
e,
|
|
)
|
|
|
|
if on_retry:
|
|
await on_retry(i, e)
|
|
|
|
await asyncio.sleep(delay)
|
|
|
|
def get_attempts(self, key: str) -> int:
|
|
return self._attempts.get(key, 0)
|
|
|
|
def reset(self, key: str) -> None:
|
|
self._attempts.pop(key, None)
|
|
self._last_error.pop(key, None)
|
|
self._locks.pop(key, None)
|
|
|
|
def is_backing_off(self, key: str, cooldown_seconds: float = 300.0) -> bool:
|
|
last_error = self._last_error.get(key)
|
|
if last_error is None:
|
|
return False
|
|
return time.time() - last_error < cooldown_seconds
|
|
|
|
def get_backoff_remaining(self, key: str, cooldown_seconds: float = 300.0) -> float:
|
|
last_error = self._last_error.get(key)
|
|
if last_error is None:
|
|
return 0.0
|
|
remaining = cooldown_seconds - (time.time() - last_error)
|
|
return max(remaining, 0.0)
|