ForcePilot/backend/package/yuxi/external_systems/infrastructure/scheduler.py

473 lines
18 KiB
Python
Raw Normal View History

"""external_systems 模块的定时任务 handler 注册入口。
本模块是 external_systems 限界上下文接入 scheduler 限界上下文的唯一登记点
对齐 scheduler 上下文设计方案 §14.3 "业务域 container 主动注册"模式
``infrastructure/container.py`` 同级遵循 external_systems 的分层约定
装配逻辑集中在 ``infrastructure/``包根不出现装配代码
对外暴露
- ``register_scheduler_handlers(registry)``业务模块注册函数 worker
启动钩子``services/run_worker.py`` ``_worker_startup``统一调用
内部实例化本模块的 handler 并调用 ``registry.register(handler)``
边界规范
- 本模块是 external_systems 接入 scheduler **唯一装配点**
- handler 的依赖 ``session_factory``在函数内部构造不暴露给装配层
- 新增 handler 在本函数内追加 ``registry.register(...)`` 即可无需改
``run_worker.py``前提本模块已登记在 ``_BUSINESS_HANDLER_REGISTRARS``
"""
from __future__ import annotations
from collections.abc import Callable
from contextlib import AbstractAsyncContextManager
from datetime import datetime, timedelta
from typing import Any
from sqlalchemy.ext.asyncio import AsyncSession
from yuxi.external_systems.adapters.persistence import create_repositories
from yuxi.external_systems.framework.runtime import AlertManagerImpl
from yuxi.scheduler.core.contracts import TaskContext, TaskResult
from yuxi.scheduler.framework.runtime import HandlerRegistry
from yuxi.storage.postgres.manager import pg_manager
from yuxi.utils.datetime_utils import utc_now_naive
from yuxi.utils.logging_config import logger
__all__ = ["register_scheduler_handlers"]
# 默认事件保留天数,与 ExternalWebhookSubscription.retention_days 默认值对齐
_DEFAULT_RETENTION_DAYS = 30
class WebhookEventCleanupHandler:
"""清理超过保留期的 Webhook 事件。
通过 scheduled_tasks 表配置周期触发 retention_days 软删除
ext_webhook_events 表中过期的原始事件记录
payload 参数
retention_days: 保留天数未传入时使用构造函数默认值
依赖注入
session_factory: 调用返回 AsyncSession 的异步上下文管理器工厂
通常为 pg_manager.get_async_session_context每次执行创建独立
会话避免长生命周期会话问题
default_retention_days: 默认保留天数未在 payload 指定时使用
"""
name = "external_systems.webhook_event_cleanup"
description = "清理超过保留期的 Webhook 事件"
def __init__(
self,
session_factory: Callable[[], AbstractAsyncContextManager[AsyncSession]],
*,
default_retention_days: int = _DEFAULT_RETENTION_DAYS,
) -> None:
self._session_factory = session_factory
self._default_retention_days = default_retention_days
async def execute(self, ctx: TaskContext) -> TaskResult:
"""执行 Webhook 事件保留期清理。
payload 读取 retention_days计算 cutoff 后委托仓储执行软删除
异常隔离捕获所有异常并转换为 TaskResult(success=False)不向调度域传播
"""
retention_days = int(ctx.payload.get("retention_days", self._default_retention_days))
cutoff = utc_now_naive() - timedelta(days=retention_days)
try:
async with self._session_factory() as db:
repos = create_repositories(db)
deleted_count = await repos.webhook_event.delete_old_events(
cutoff,
updated_by="scheduler",
)
except Exception as exc:
logger.exception(
"webhook_event_cleanup_failed",
extra={
"retention_days": retention_days,
"cutoff": cutoff,
},
)
return TaskResult(success=False, error=str(exc))
if deleted_count > 0:
logger.info(
"webhook_event_cleanup",
extra={
"deleted_count": deleted_count,
"retention_days": retention_days,
"cutoff": cutoff,
},
)
return TaskResult(
success=True,
output={
"deleted_count": deleted_count,
"retention_days": retention_days,
},
)
class WebhookRenewalHandler:
"""自动续期即将过期的 Webhook 订阅。
通过 scheduled_tasks 表配置周期触发扫描 ``expires_at`` 在阈值内且
``auto_renew=True````status=active`` 的订阅延长其过期时间并记录结果
payload 参数
threshold_days: 提前扫描天数默认 7
当前实现为平台侧续期 ``retention_days`` 延长过期时间外部系统侧
真实续期逻辑需后续按 adapter 类型扩展
"""
name = "external_systems.webhook_renewal"
description = "自动续期即将过期的 Webhook 订阅"
def __init__(
self,
session_factory: Callable[[], AbstractAsyncContextManager[AsyncSession]],
) -> None:
self._session_factory = session_factory
async def execute(self, ctx: TaskContext) -> TaskResult:
"""执行续期扫描与续期。"""
threshold_days = int(ctx.payload.get("threshold_days", 7))
before = utc_now_naive() + timedelta(days=threshold_days)
renewed_count = 0
failed_count = 0
try:
async with self._session_factory() as db:
repos = create_repositories(db)
expiring = await repos.webhook_subscription.list_expiring(before=before)
for subscription in expiring:
try:
if subscription.id is None:
failed_count += 1
logger.warning(
"webhook_renewal_skip_missing_id",
extra={"subscription_slug": subscription.slug},
)
continue
new_expires_at = utc_now_naive() + timedelta(days=subscription.retention_days)
await repos.webhook_subscription.update_renewal_result(
subscription.id,
success=True,
new_expires_at=new_expires_at,
)
renewed_count += 1
except Exception: # noqa: BLE001 - 单个订阅续期失败不影响其他
failed_count += 1
logger.exception(
"webhook_renewal_single_failed",
extra={"subscription_slug": subscription.slug},
)
except Exception as exc:
logger.exception("webhook_renewal_failed", extra={"threshold_days": threshold_days})
return TaskResult(success=False, error=str(exc))
logger.info(
"webhook_renewal_completed",
extra={
"threshold_days": threshold_days,
"renewed_count": renewed_count,
"failed_count": failed_count,
},
)
return TaskResult(
success=True,
output={
"threshold_days": threshold_days,
"renewed_count": renewed_count,
"failed_count": failed_count,
},
)
class SecretRotationHandler:
"""检查到期密钥轮换策略并执行轮换/告警。
通过 scheduled_tasks 表配置周期触发调用 ``SecretRotationService``
``scheduled_rotation`` 方法完成到期策略扫描auto 模式轮换
manual 模式告警以及已过期策略 critical 告警
"""
name = "external_systems.secret_rotation"
description = "检查到期密钥轮换策略并执行轮换/告警"
def __init__(
self,
session_factory: Callable[[], AbstractAsyncContextManager[AsyncSession]],
) -> None:
self._session_factory = session_factory
async def execute(self, ctx: TaskContext) -> TaskResult:
"""执行密钥轮换调度。
异常隔离捕获所有异常并转换为 TaskResult(success=False)不向调度域传播
"""
_ = ctx # 本 handler 无需 payload 参数
try:
async with self._session_factory() as db:
# 延迟导入 create_use_cases_from_db避免在模块加载阶段触发
# infrastructure container 的装配container 依赖 app_config
# 在本地非容器测试环境中可能不存在)。
from yuxi.external_systems.infrastructure.container import (
create_use_cases_from_db,
)
use_cases = create_use_cases_from_db(db)
output = await use_cases.secret_rotation_service.scheduled_rotation()
except Exception as exc:
logger.exception("secret_rotation_failed")
return TaskResult(success=False, error=str(exc))
return TaskResult(
success=True,
output=output.model_dump(mode="json"),
)
def _compute_next_window(
quota_window: str,
window_start: datetime | None,
window_end: datetime | None,
now: datetime,
) -> tuple[datetime, datetime]:
"""根据配额窗口类型计算下一窗口起止时间。
规则
- rolling_24h: 以当前时间为起点向后滚动 24h
- daily/hourly/minute: 以上一窗口结束时间为起点按固定步长推进
- custom: 保持上一窗口长度无法推断时回退 1
"""
if quota_window == "rolling_24h":
return now, now + timedelta(hours=24)
last_end = window_end or now
if quota_window == "daily":
delta = timedelta(days=1)
elif quota_window == "hourly":
delta = timedelta(hours=1)
elif quota_window == "minute":
delta = timedelta(minutes=1)
elif window_start is not None and window_end is not None:
delta = window_end - window_start
if delta.total_seconds() <= 0:
delta = timedelta(days=1)
else:
delta = timedelta(days=1)
return last_end, last_end + delta
class QuotaThresholdAlertHandler:
"""扫描达到告警/临界阈值的外部系统配额并触发告警。
payload 参数
limit: 每次扫描最大记录数默认 100
"""
name = "external_systems.quota_threshold_alert"
description = "扫描达到告警/临界阈值的外部系统配额并触发告警"
def __init__(
self,
session_factory: Callable[[], AbstractAsyncContextManager[AsyncSession]],
*,
default_limit: int = 100,
) -> None:
self._session_factory = session_factory
self._default_limit = default_limit
async def execute(self, ctx: TaskContext) -> TaskResult:
"""执行配额阈值告警扫描。
先查询 critical 阈值记录高优先级再查询 warning 阈值记录并
排除已触达 critical 的记录避免同一配额同时产生两条不同严重级别告警
"""
limit = int(ctx.payload.get("limit", self._default_limit))
try:
async with self._session_factory() as db:
repos = create_repositories(db)
alert_manager = AlertManagerImpl(repos.alert)
critical_quotas = await repos.quota_usage.list_near_critical(limit=limit)
critical_ids = {q.id for q in critical_quotas if q.id is not None}
for quota in critical_quotas:
await self._fire_alert(alert_manager, quota, "critical")
warning_quotas = await repos.quota_usage.list_near_warning(limit=limit)
warning_count = 0
for quota in warning_quotas:
if quota.id in critical_ids:
continue
await self._fire_alert(alert_manager, quota, "warning")
warning_count += 1
except Exception as exc:
logger.exception("quota_threshold_alert_failed")
return TaskResult(success=False, error=str(exc))
return TaskResult(
success=True,
output={
"critical_count": len(critical_quotas),
"warning_count": warning_count,
},
)
async def _fire_alert(
self,
alert_manager: AlertManagerImpl,
quota: Any,
severity: str,
) -> None:
"""为单个配额触发告警。"""
title = f"配额接近上限: {quota.quota_name}" if severity == "warning" else f"配额即将耗尽: {quota.quota_name}"
description = (
f"system_id={quota.system_id}, env={quota.env_key}, "
f"key={quota.quota_key}, 已用 {quota.usage_ratio * 100:.1f}%"
)
await alert_manager.fire(
quota.system_id,
quota.env_key,
"quota_near_limit",
severity=severity, # type: ignore[arg-type]
title=title,
description=description,
resource_type="quota_usage",
resource_id=quota.quota_key,
metric_snapshot={
"limit_value": quota.limit_value,
"used_value": quota.used_value,
"usage_ratio": quota.usage_ratio,
"warning_threshold": quota.warning_threshold,
"critical_threshold": quota.critical_threshold,
},
)
class QuotaWindowResetHandler:
"""自动重置已到期外部系统配额窗口。
payload 参数
limit: 每次扫描最大记录数默认 100
"""
name = "external_systems.quota_window_reset"
description = "自动重置已到期的外部系统配额窗口"
def __init__(
self,
session_factory: Callable[[], AbstractAsyncContextManager[AsyncSession]],
*,
default_limit: int = 100,
) -> None:
self._session_factory = session_factory
self._default_limit = default_limit
async def execute(self, ctx: TaskContext) -> TaskResult:
"""执行配额窗口自动重置。
查询 ``window_end <= now`` 的配额记录按窗口类型计算下一窗口并
``used_value`` 清零单条失败不影响后续记录
"""
limit = int(ctx.payload.get("limit", self._default_limit))
now = utc_now_naive()
try:
async with self._session_factory() as db:
repos = create_repositories(db)
quotas = await repos.quota_usage.list(expiring_before=now, limit=limit)
reset_count = 0
failed_count = 0
errors: list[str] = []
for quota in quotas:
try:
window_start, window_end = _compute_next_window(
quota.quota_window,
quota.window_start,
quota.window_end,
now,
)
await repos.quota_usage.reset_window(
quota.system_id,
quota.env_key,
quota.quota_key,
window_start=window_start,
window_end=window_end,
used_value=0,
updated_by="scheduler",
commit=True,
)
reset_count += 1
except Exception as exc: # noqa: BLE001
failed_count += 1
error_msg = f"{quota.system_id}/{quota.env_key}/{quota.quota_key}: {exc}"
errors.append(error_msg)
logger.warning(
"quota_window_reset_failed",
extra={
"system_id": quota.system_id,
"env_key": quota.env_key,
"quota_key": quota.quota_key,
"error": error_msg,
},
)
except Exception as exc:
logger.exception("quota_window_reset_failed")
return TaskResult(success=False, error=str(exc))
return TaskResult(
success=True,
output={
"reset_count": reset_count,
"failed_count": failed_count,
"errors": errors,
},
)
def register_scheduler_handlers(registry: HandlerRegistry) -> None:
"""向 scheduler 注册 external_systems 模块的定时任务 handler。
worker 启动钩子在装配阶段统一调用新增 handler 时在此函数内追加
``registry.register(...)`` 即可
Args:
registry: worker 进程级 handler 注册表
"""
registry.register(
WebhookEventCleanupHandler(
session_factory=pg_manager.get_async_session_context,
),
)
registry.register(
QuotaThresholdAlertHandler(
session_factory=pg_manager.get_async_session_context,
),
)
registry.register(
QuotaWindowResetHandler(
session_factory=pg_manager.get_async_session_context,
),
)
registry.register(
WebhookRenewalHandler(
session_factory=pg_manager.get_async_session_context,
),
)
registry.register(
SecretRotationHandler(
session_factory=pg_manager.get_async_session_context,
),
)