import asyncio import logging import time from yuxi.channel.cron.types import CronFailureAlert, CronJob, DeliveryHandler logger = logging.getLogger(__name__) ALERT_COOLDOWN_MS = 600_000 RECOVERY_COOLDOWN_MS = 300_000 _GLOBAL_ALERT_MAX_PER_MINUTE = 10 _global_alert_timestamps: list[int] = [] def _global_rate_check(now_ms: int) -> bool: cutoff = now_ms - 60_000 global _global_alert_timestamps _global_alert_timestamps = [t for t in _global_alert_timestamps if t > cutoff] if len(_global_alert_timestamps) >= _GLOBAL_ALERT_MAX_PER_MINUTE: return False _global_alert_timestamps.append(now_ms) return True def _format_cooldown_ms(ms: int) -> str: if ms < 60_000: return f"{ms // 1000}s" if ms < 3_600_000: return f"{ms // 60_000}min" return f"{ms // 3_600_000}h" def format_alert_message(job: CronJob, consecutive_errors: int, last_error: str | None) -> str: job_label = job.name or job.id return ( f"🚨 Cron 任务连续失败告警\n\n" f"任务:{job_label}\n" f"连续失败次数:{consecutive_errors}\n" f"告警触发阈值:{job.failure_alert.threshold}\n" f"最近错误:{last_error or '未知'}\n\n" f"任务已自动标记为 FAILED,冷却期 {_format_cooldown_ms(job.failure_alert.cooldown_ms)} 内不再告警" ) def format_recovery_message(job: CronJob) -> str: job_label = job.name or job.id return f"✅ Cron 任务已恢复正常\n\n任务:{job_label}\n任务状态已恢复为 ACTIVE" async def send_alert( handler: DeliveryHandler | None, delivery_targets: list, job: CronJob, content: str, ) -> None: if handler is None: logger.warning("Cron alert for '%s' cannot be delivered: no delivery handler configured", job.id) return if not delivery_targets: logger.warning("Cron alert for '%s': no delivery targets configured", job.id) return for target in delivery_targets: if not target.enabled or not target.channel or not target.target_id: continue try: await handler(target.channel, target.target_id, content) except Exception: logger.exception("Cron alert delivery to %s:%s failed", target.channel, target.target_id) async def check_and_alert( handler: DeliveryHandler | None, job: CronJob, ) -> bool: alert = job.failure_alert now_ms = int(time.time() * 1000) delivery_targets = job.delivery if alert.should_alert(job.state.consecutive_errors, now_ms): if not _global_rate_check(now_ms): logger.warning("Cron alert for '%s' suppressed by global rate limit", job.id) return False content = format_alert_message(job, job.state.consecutive_errors, job.state.last_error) await send_alert(handler, delivery_targets, job, content) alert.mark_alerted(now_ms) logger.warning( "Cron failure alert sent for '%s' (consecutive errors: %d)", job.id, job.state.consecutive_errors, ) return True return False async def check_and_recover( handler: DeliveryHandler | None, job: CronJob, ) -> bool: alert = job.failure_alert now_ms = int(time.time() * 1000) delivery_targets = job.delivery if alert.should_recover(job.state.consecutive_errors, now_ms): content = format_recovery_message(job) await send_alert(handler, delivery_targets, job, content) alert.mark_recovered(now_ms) logger.info("Cron recovery notification sent for '%s'", job.id) return True return False async def evaluate_alerts( handler: DeliveryHandler | None, job: CronJob, ) -> dict: alert_sent = await check_and_alert(handler, job) recovery_sent = await check_and_recover(handler, job) return {"alert_sent": alert_sent, "recovery_sent": recovery_sent}