ForcePilot/backend/package/yuxi/channel/cron/alert.py

import asyncio
import logging
import time

from yuxi.channel.cron.types import CronFailureAlert, CronJob, DeliveryHandler

logger = logging.getLogger(__name__)

ALERT_COOLDOWN_MS = 600_000
RECOVERY_COOLDOWN_MS = 300_000
_GLOBAL_ALERT_MAX_PER_MINUTE = 10
_global_alert_timestamps: list[int] = []


def _global_rate_check(now_ms: int) -> bool:
    cutoff = now_ms - 60_000
    global _global_alert_timestamps
    _global_alert_timestamps = [t for t in _global_alert_timestamps if t > cutoff]
    if len(_global_alert_timestamps) >= _GLOBAL_ALERT_MAX_PER_MINUTE:
        return False
    _global_alert_timestamps.append(now_ms)
    return True


def _format_cooldown_ms(ms: int) -> str:
    if ms < 60_000:
        return f"{ms // 1000}s"
    if ms < 3_600_000:
        return f"{ms // 60_000}min"
    return f"{ms // 3_600_000}h"


def format_alert_message(job: CronJob, consecutive_errors: int, last_error: str | None) -> str:
    job_label = job.name or job.id
    return (
        f"🚨 Cron 任务连续失败告警\n\n"
        f"任务：{job_label}\n"
        f"连续失败次数：{consecutive_errors}\n"
        f"告警触发阈值：{job.failure_alert.threshold}\n"
        f"最近错误：{last_error or '未知'}\n\n"
        f"任务已自动标记为 FAILED，冷却期 {_format_cooldown_ms(job.failure_alert.cooldown_ms)} 内不再告警"
    )


def format_recovery_message(job: CronJob) -> str:
    job_label = job.name or job.id
    return f"✅ Cron 任务已恢复正常\n\n任务：{job_label}\n任务状态已恢复为 ACTIVE"


async def send_alert(
    handler: DeliveryHandler | None,
    delivery_targets: list,
    job: CronJob,
    content: str,
) -> None:
    if handler is None:
        logger.warning("Cron alert for '%s' cannot be delivered: no delivery handler configured", job.id)
        return

    if not delivery_targets:
        logger.warning("Cron alert for '%s': no delivery targets configured", job.id)
        return

    for target in delivery_targets:
        if not target.enabled or not target.channel or not target.target_id:
            continue
        try:
            await handler(target.channel, target.target_id, content)
        except Exception:
            logger.exception("Cron alert delivery to %s:%s failed", target.channel, target.target_id)


async def check_and_alert(
    handler: DeliveryHandler | None,
    job: CronJob,
) -> bool:
    alert = job.failure_alert
    now_ms = int(time.time() * 1000)
    delivery_targets = job.delivery

    if alert.should_alert(job.state.consecutive_errors, now_ms):
        if not _global_rate_check(now_ms):
            logger.warning("Cron alert for '%s' suppressed by global rate limit", job.id)
            return False
        content = format_alert_message(job, job.state.consecutive_errors, job.state.last_error)
        await send_alert(handler, delivery_targets, job, content)
        alert.mark_alerted(now_ms)
        logger.warning(
            "Cron failure alert sent for '%s' (consecutive errors: %d)",
            job.id,
            job.state.consecutive_errors,
        )
        return True

    return False


async def check_and_recover(
    handler: DeliveryHandler | None,
    job: CronJob,
) -> bool:
    alert = job.failure_alert
    now_ms = int(time.time() * 1000)
    delivery_targets = job.delivery

    if alert.should_recover(job.state.consecutive_errors, now_ms):
        content = format_recovery_message(job)
        await send_alert(handler, delivery_targets, job, content)
        alert.mark_recovered(now_ms)
        logger.info("Cron recovery notification sent for '%s'", job.id)
        return True

    return False


async def evaluate_alerts(
    handler: DeliveryHandler | None,
    job: CronJob,
) -> dict:
    alert_sent = await check_and_alert(handler, job)
    recovery_sent = await check_and_recover(handler, job)
    return {"alert_sent": alert_sent, "recovery_sent": recovery_sent}