205 lines
7.6 KiB
Python
205 lines
7.6 KiB
Python
import logging
|
|
import time
|
|
from collections.abc import Callable
|
|
from dataclasses import dataclass, field
|
|
|
|
from yuxi.channel.monitoring.event_loop_monitor import EventLoopHealth
|
|
from yuxi.channel.monitoring.health_monitor import (
|
|
DEFAULT_CONNECTION_GRACE,
|
|
DEFAULT_STALE_EVENT_THRESHOLD,
|
|
evaluate_channel_health,
|
|
)
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
DEFAULT_READINESS_CACHE_MS = 1000.0
|
|
|
|
|
|
@dataclass
|
|
class ChannelReadiness:
|
|
key: str
|
|
healthy: bool
|
|
reason: str
|
|
detail: str | None = None
|
|
|
|
|
|
@dataclass
|
|
class ReadinessResult:
|
|
ready: bool
|
|
channels: list[ChannelReadiness]
|
|
failing: list[str] = field(default_factory=list)
|
|
uptime_ms: float = 0.0
|
|
event_loop_degraded: bool = False
|
|
event_loop_reasons: list[str] = field(default_factory=list)
|
|
event_loop: EventLoopHealth | None = None
|
|
checked_at: float = 0.0
|
|
cache_ttl_ms: float = 1000.0
|
|
|
|
|
|
def _should_ignore_readiness_failure(
|
|
evaluation_reason: str,
|
|
snap_data: dict,
|
|
) -> bool:
|
|
if evaluation_reason == "unmanaged":
|
|
return True
|
|
if evaluation_reason == "stale-socket":
|
|
return True
|
|
if evaluation_reason == "not-running" and snap_data.get("restart_pending") is True:
|
|
return True
|
|
return False
|
|
|
|
|
|
class ReadinessChecker:
|
|
def __init__(
|
|
self,
|
|
stale_event_threshold: float = DEFAULT_STALE_EVENT_THRESHOLD,
|
|
connect_grace: float = DEFAULT_CONNECTION_GRACE,
|
|
cache_ttl_ms: float = DEFAULT_READINESS_CACHE_MS,
|
|
started_at: float | None = None,
|
|
get_startup_pending: Callable[[], bool] | None = None,
|
|
get_startup_pending_reason: Callable[[], str] | None = None,
|
|
get_event_loop_health: Callable[[], EventLoopHealth | None] | None = None,
|
|
should_skip_channel_readiness: Callable[[], bool] | None = None,
|
|
):
|
|
self._stale_event_threshold = stale_event_threshold
|
|
self._connect_grace = connect_grace
|
|
self._cache_ttl_ms = max(0.0, cache_ttl_ms)
|
|
self._started_at = started_at if started_at is not None else time.monotonic()
|
|
self._get_startup_pending = get_startup_pending
|
|
self._get_startup_pending_reason = get_startup_pending_reason
|
|
self._get_event_loop_health = get_event_loop_health
|
|
self._should_skip_channel_readiness = should_skip_channel_readiness
|
|
self._last_result: ReadinessResult | None = None
|
|
self._last_check_at: float = 0.0
|
|
|
|
async def check(self, *, force: bool = False) -> ReadinessResult:
|
|
now = time.monotonic()
|
|
uptime_ms = (now - self._started_at) * 1000
|
|
|
|
if self._get_startup_pending and self._get_startup_pending():
|
|
reason = self._get_startup_pending_reason() if self._get_startup_pending_reason else "startup-sidecars"
|
|
result = ReadinessResult(
|
|
ready=False,
|
|
channels=[],
|
|
failing=[reason],
|
|
uptime_ms=uptime_ms,
|
|
checked_at=now,
|
|
cache_ttl_ms=self._cache_ttl_ms,
|
|
)
|
|
result = self._attach_event_loop(result)
|
|
return result
|
|
|
|
if self._should_skip_channel_readiness and self._should_skip_channel_readiness():
|
|
result = ReadinessResult(
|
|
ready=True,
|
|
channels=[],
|
|
failing=[],
|
|
uptime_ms=uptime_ms,
|
|
checked_at=now,
|
|
cache_ttl_ms=self._cache_ttl_ms,
|
|
)
|
|
result = self._attach_event_loop(result)
|
|
return result
|
|
|
|
if not force and self._last_result is not None:
|
|
elapsed = (now - self._last_check_at) * 1000
|
|
if elapsed < self._cache_ttl_ms:
|
|
cached = ReadinessResult(
|
|
ready=self._last_result.ready,
|
|
channels=list(self._last_result.channels),
|
|
failing=list(self._last_result.failing),
|
|
uptime_ms=uptime_ms,
|
|
event_loop_degraded=self._last_result.event_loop_degraded,
|
|
event_loop_reasons=list(self._last_result.event_loop_reasons),
|
|
event_loop=self._last_result.event_loop,
|
|
checked_at=self._last_result.checked_at,
|
|
cache_ttl_ms=self._last_result.cache_ttl_ms,
|
|
)
|
|
cached = self._attach_event_loop(cached)
|
|
return cached
|
|
|
|
channel_evaluations: list[ChannelReadiness] = []
|
|
failing: list[str] = []
|
|
|
|
try:
|
|
from yuxi.channel.runtime.manager import gateway
|
|
|
|
report = gateway.get_health()
|
|
for key, snap_data in report.channels.items():
|
|
evaluation = evaluate_channel_health(
|
|
enabled=snap_data.get("enabled"),
|
|
configured=snap_data.get("configured"),
|
|
running=snap_data.get("running", False),
|
|
connected=snap_data.get("connected", False),
|
|
busy=snap_data.get("busy", False),
|
|
active_runs=snap_data.get("active_runs", 0),
|
|
last_run_activity_at=snap_data.get("last_run_activity_at"),
|
|
last_transport_activity_at=snap_data.get("last_transport_activity_at"),
|
|
last_start_at=snap_data.get("last_start_at"),
|
|
reconnect_attempts=snap_data.get("reconnect_attempts", 0),
|
|
now=now,
|
|
stale_event_threshold=self._stale_event_threshold,
|
|
connect_grace=self._connect_grace,
|
|
)
|
|
if evaluation.reason == "unmanaged":
|
|
continue
|
|
|
|
effective_healthy = (
|
|
True
|
|
if not evaluation.healthy and _should_ignore_readiness_failure(evaluation.reason, snap_data)
|
|
else evaluation.healthy
|
|
)
|
|
|
|
channel_evaluations.append(
|
|
ChannelReadiness(
|
|
key=key,
|
|
healthy=effective_healthy,
|
|
reason=evaluation.reason,
|
|
detail=evaluation.detail,
|
|
)
|
|
)
|
|
|
|
if not effective_healthy:
|
|
failing.append(key)
|
|
except Exception as e:
|
|
logger.warning("ReadinessChecker: failed to evaluate channels: %s", e)
|
|
failing.append("evaluation-error")
|
|
|
|
all_ready = len(failing) == 0
|
|
|
|
result = ReadinessResult(
|
|
ready=all_ready,
|
|
channels=channel_evaluations,
|
|
failing=failing,
|
|
uptime_ms=uptime_ms,
|
|
checked_at=now,
|
|
cache_ttl_ms=self._cache_ttl_ms,
|
|
)
|
|
result = self._attach_event_loop(result)
|
|
if result.event_loop_degraded:
|
|
all_ready = False
|
|
result.ready = all_ready
|
|
|
|
self._last_result = result
|
|
self._last_check_at = now
|
|
return result
|
|
|
|
def _attach_event_loop(self, result: ReadinessResult) -> ReadinessResult:
|
|
try:
|
|
if self._get_event_loop_health:
|
|
event_loop = self._get_event_loop_health()
|
|
else:
|
|
from yuxi.channel.monitoring.event_loop_monitor import event_loop_monitor
|
|
|
|
event_loop = event_loop_monitor.health()
|
|
if event_loop is not None:
|
|
result.event_loop = event_loop
|
|
result.event_loop_degraded = event_loop.degraded
|
|
result.event_loop_reasons = list(event_loop.reasons)
|
|
except Exception as e:
|
|
logger.debug("ReadinessChecker: failed to get event loop health: %s", e)
|
|
return result
|
|
|
|
|
|
readiness_checker = ReadinessChecker()
|