import logging import time from collections.abc import Callable from dataclasses import dataclass, field from yuxi.channel.monitoring.event_loop_monitor import EventLoopHealth from yuxi.channel.monitoring.health_monitor import ( DEFAULT_CONNECTION_GRACE, DEFAULT_STALE_EVENT_THRESHOLD, evaluate_channel_health, ) logger = logging.getLogger(__name__) DEFAULT_READINESS_CACHE_MS = 1000.0 @dataclass class ChannelReadiness: key: str healthy: bool reason: str detail: str | None = None @dataclass class ReadinessResult: ready: bool channels: list[ChannelReadiness] failing: list[str] = field(default_factory=list) uptime_ms: float = 0.0 event_loop_degraded: bool = False event_loop_reasons: list[str] = field(default_factory=list) event_loop: EventLoopHealth | None = None checked_at: float = 0.0 cache_ttl_ms: float = 1000.0 def _should_ignore_readiness_failure( evaluation_reason: str, snap_data: dict, ) -> bool: if evaluation_reason == "unmanaged": return True if evaluation_reason == "stale-socket": return True if evaluation_reason == "not-running" and snap_data.get("restart_pending") is True: return True return False class ReadinessChecker: def __init__( self, stale_event_threshold: float = DEFAULT_STALE_EVENT_THRESHOLD, connect_grace: float = DEFAULT_CONNECTION_GRACE, cache_ttl_ms: float = DEFAULT_READINESS_CACHE_MS, started_at: float | None = None, get_startup_pending: Callable[[], bool] | None = None, get_startup_pending_reason: Callable[[], str] | None = None, get_event_loop_health: Callable[[], EventLoopHealth | None] | None = None, should_skip_channel_readiness: Callable[[], bool] | None = None, ): self._stale_event_threshold = stale_event_threshold self._connect_grace = connect_grace self._cache_ttl_ms = max(0.0, cache_ttl_ms) self._started_at = started_at if started_at is not None else time.monotonic() self._get_startup_pending = get_startup_pending self._get_startup_pending_reason = get_startup_pending_reason self._get_event_loop_health = get_event_loop_health self._should_skip_channel_readiness = should_skip_channel_readiness self._last_result: ReadinessResult | None = None self._last_check_at: float = 0.0 async def check(self, *, force: bool = False) -> ReadinessResult: now = time.monotonic() uptime_ms = (now - self._started_at) * 1000 if self._get_startup_pending and self._get_startup_pending(): reason = self._get_startup_pending_reason() if self._get_startup_pending_reason else "startup-sidecars" result = ReadinessResult( ready=False, channels=[], failing=[reason], uptime_ms=uptime_ms, checked_at=now, cache_ttl_ms=self._cache_ttl_ms, ) result = self._attach_event_loop(result) return result if self._should_skip_channel_readiness and self._should_skip_channel_readiness(): result = ReadinessResult( ready=True, channels=[], failing=[], uptime_ms=uptime_ms, checked_at=now, cache_ttl_ms=self._cache_ttl_ms, ) result = self._attach_event_loop(result) return result if not force and self._last_result is not None: elapsed = (now - self._last_check_at) * 1000 if elapsed < self._cache_ttl_ms: cached = ReadinessResult( ready=self._last_result.ready, channels=list(self._last_result.channels), failing=list(self._last_result.failing), uptime_ms=uptime_ms, event_loop_degraded=self._last_result.event_loop_degraded, event_loop_reasons=list(self._last_result.event_loop_reasons), event_loop=self._last_result.event_loop, checked_at=self._last_result.checked_at, cache_ttl_ms=self._last_result.cache_ttl_ms, ) cached = self._attach_event_loop(cached) return cached channel_evaluations: list[ChannelReadiness] = [] failing: list[str] = [] try: from yuxi.channel.runtime.manager import gateway report = gateway.get_health() for key, snap_data in report.channels.items(): evaluation = evaluate_channel_health( enabled=snap_data.get("enabled"), configured=snap_data.get("configured"), running=snap_data.get("running", False), connected=snap_data.get("connected", False), busy=snap_data.get("busy", False), active_runs=snap_data.get("active_runs", 0), last_run_activity_at=snap_data.get("last_run_activity_at"), last_transport_activity_at=snap_data.get("last_transport_activity_at"), last_start_at=snap_data.get("last_start_at"), reconnect_attempts=snap_data.get("reconnect_attempts", 0), now=now, stale_event_threshold=self._stale_event_threshold, connect_grace=self._connect_grace, ) if evaluation.reason == "unmanaged": continue effective_healthy = ( True if not evaluation.healthy and _should_ignore_readiness_failure(evaluation.reason, snap_data) else evaluation.healthy ) channel_evaluations.append( ChannelReadiness( key=key, healthy=effective_healthy, reason=evaluation.reason, detail=evaluation.detail, ) ) if not effective_healthy: failing.append(key) except Exception as e: logger.warning("ReadinessChecker: failed to evaluate channels: %s", e) failing.append("evaluation-error") all_ready = len(failing) == 0 result = ReadinessResult( ready=all_ready, channels=channel_evaluations, failing=failing, uptime_ms=uptime_ms, checked_at=now, cache_ttl_ms=self._cache_ttl_ms, ) result = self._attach_event_loop(result) if result.event_loop_degraded: all_ready = False result.ready = all_ready self._last_result = result self._last_check_at = now return result def _attach_event_loop(self, result: ReadinessResult) -> ReadinessResult: try: if self._get_event_loop_health: event_loop = self._get_event_loop_health() else: from yuxi.channel.monitoring.event_loop_monitor import event_loop_monitor event_loop = event_loop_monitor.health() if event_loop is not None: result.event_loop = event_loop result.event_loop_degraded = event_loop.degraded result.event_loop_reasons = list(event_loop.reasons) except Exception as e: logger.debug("ReadinessChecker: failed to get event loop health: %s", e) return result readiness_checker = ReadinessChecker()