123 lines
3.8 KiB
Python
123 lines
3.8 KiB
Python
import asyncio
|
||
import logging
|
||
import time
|
||
|
||
from yuxi.channel.cron.types import CronFailureAlert, CronJob, DeliveryHandler
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
ALERT_COOLDOWN_MS = 600_000
|
||
RECOVERY_COOLDOWN_MS = 300_000
|
||
_GLOBAL_ALERT_MAX_PER_MINUTE = 10
|
||
_global_alert_timestamps: list[int] = []
|
||
|
||
|
||
def _global_rate_check(now_ms: int) -> bool:
|
||
cutoff = now_ms - 60_000
|
||
global _global_alert_timestamps
|
||
_global_alert_timestamps = [t for t in _global_alert_timestamps if t > cutoff]
|
||
if len(_global_alert_timestamps) >= _GLOBAL_ALERT_MAX_PER_MINUTE:
|
||
return False
|
||
_global_alert_timestamps.append(now_ms)
|
||
return True
|
||
|
||
|
||
def _format_cooldown_ms(ms: int) -> str:
|
||
if ms < 60_000:
|
||
return f"{ms // 1000}s"
|
||
if ms < 3_600_000:
|
||
return f"{ms // 60_000}min"
|
||
return f"{ms // 3_600_000}h"
|
||
|
||
|
||
def format_alert_message(job: CronJob, consecutive_errors: int, last_error: str | None) -> str:
|
||
job_label = job.name or job.id
|
||
return (
|
||
f"🚨 Cron 任务连续失败告警\n\n"
|
||
f"任务：{job_label}\n"
|
||
f"连续失败次数：{consecutive_errors}\n"
|
||
f"告警触发阈值：{job.failure_alert.threshold}\n"
|
||
f"最近错误：{last_error or '未知'}\n\n"
|
||
f"任务已自动标记为 FAILED，冷却期 {_format_cooldown_ms(job.failure_alert.cooldown_ms)} 内不再告警"
|
||
)
|
||
|
||
|
||
def format_recovery_message(job: CronJob) -> str:
|
||
job_label = job.name or job.id
|
||
return f"✅ Cron 任务已恢复正常\n\n任务：{job_label}\n任务状态已恢复为 ACTIVE"
|
||
|
||
|
||
async def send_alert(
|
||
handler: DeliveryHandler | None,
|
||
delivery_targets: list,
|
||
job: CronJob,
|
||
content: str,
|
||
) -> None:
|
||
if handler is None:
|
||
logger.warning("Cron alert for '%s' cannot be delivered: no delivery handler configured", job.id)
|
||
return
|
||
|
||
if not delivery_targets:
|
||
logger.warning("Cron alert for '%s': no delivery targets configured", job.id)
|
||
return
|
||
|
||
for target in delivery_targets:
|
||
if not target.enabled or not target.channel or not target.target_id:
|
||
continue
|
||
try:
|
||
await handler(target.channel, target.target_id, content)
|
||
except Exception:
|
||
logger.exception("Cron alert delivery to %s:%s failed", target.channel, target.target_id)
|
||
|
||
|
||
async def check_and_alert(
|
||
handler: DeliveryHandler | None,
|
||
job: CronJob,
|
||
) -> bool:
|
||
alert = job.failure_alert
|
||
now_ms = int(time.time() * 1000)
|
||
delivery_targets = job.delivery
|
||
|
||
if alert.should_alert(job.state.consecutive_errors, now_ms):
|
||
if not _global_rate_check(now_ms):
|
||
logger.warning("Cron alert for '%s' suppressed by global rate limit", job.id)
|
||
return False
|
||
content = format_alert_message(job, job.state.consecutive_errors, job.state.last_error)
|
||
await send_alert(handler, delivery_targets, job, content)
|
||
alert.mark_alerted(now_ms)
|
||
logger.warning(
|
||
"Cron failure alert sent for '%s' (consecutive errors: %d)",
|
||
job.id,
|
||
job.state.consecutive_errors,
|
||
)
|
||
return True
|
||
|
||
return False
|
||
|
||
|
||
async def check_and_recover(
|
||
handler: DeliveryHandler | None,
|
||
job: CronJob,
|
||
) -> bool:
|
||
alert = job.failure_alert
|
||
now_ms = int(time.time() * 1000)
|
||
delivery_targets = job.delivery
|
||
|
||
if alert.should_recover(job.state.consecutive_errors, now_ms):
|
||
content = format_recovery_message(job)
|
||
await send_alert(handler, delivery_targets, job, content)
|
||
alert.mark_recovered(now_ms)
|
||
logger.info("Cron recovery notification sent for '%s'", job.id)
|
||
return True
|
||
|
||
return False
|
||
|
||
|
||
async def evaluate_alerts(
|
||
handler: DeliveryHandler | None,
|
||
job: CronJob,
|
||
) -> dict:
|
||
alert_sent = await check_and_alert(handler, job)
|
||
recovery_sent = await check_and_recover(handler, job)
|
||
return {"alert_sent": alert_sent, "recovery_sent": recovery_sent}
|