Kris
00a5577aac
本次提交包含了大量跨模块的功能增强、bug修复与代码优化： 1. 清理冗余空行与导入格式，统一代码风格 2. 新增集成超时异常类，替换原生TimeoutError避免穿透核心层 3. 扩展审计日志DTO与用例，新增变更字段追踪能力 4. 完善各类仓储接口，新增批量操作、时间范围过滤支持 5. 重构健康检查返回格式，统一使用status字段替代冗余的reachable/healthy 6. 扩展仪表盘与各类服务端口，新增待办统计、系统dashboard等能力 7. 优化批量操作与事务处理，新增保存点支持隔离失败操作 8. 修复配额管理阈值计算逻辑，支持自定义告警阈值 9. 统一认证类型错误提示，优化插件注册校验逻辑 10. 扩展SOAP适配器预览能力，兼容bytes类型输入 11. 新增批量恢复、批量硬删除回收站资源的接口与DTO 12. 优化测试用例断言逻辑，兼容前端操作符别名与字段名差异
989 lines
37 KiB
Python
989 lines
37 KiB
Python
"""external_systems 模块的定时任务 handler 注册入口。
|
||
|
||
本模块是 external_systems 限界上下文接入 scheduler 限界上下文的唯一登记点，
|
||
对齐 scheduler 上下文设计方案 §14.3 的"业务域 container 主动注册"模式。
|
||
|
||
与 ``infrastructure/container.py`` 同级，遵循 external_systems 的分层约定：
|
||
装配逻辑集中在 ``infrastructure/``，包根不出现装配代码。
|
||
|
||
对外暴露：
|
||
|
||
- ``register_scheduler_handlers(registry)``：业务模块注册函数，由 worker
|
||
启动钩子（``services/run_worker.py`` 的 ``_worker_startup``）统一调用。
|
||
内部实例化本模块的 handler 并调用 ``registry.register(handler)``。
|
||
|
||
边界规范：
|
||
- 本模块是 external_systems 接入 scheduler 的**唯一装配点**
|
||
- handler 的依赖（如 ``session_factory``）在函数内部构造，不暴露给装配层
|
||
- 新增 handler 时，在本函数内追加 ``registry.register(...)`` 即可，无需改
|
||
``run_worker.py``（前提：本模块已登记在 ``_BUSINESS_HANDLER_REGISTRARS``）
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from collections.abc import Callable
|
||
from contextlib import AbstractAsyncContextManager
|
||
from datetime import datetime, timedelta
|
||
from typing import Any
|
||
|
||
from sqlalchemy.ext.asyncio import AsyncSession
|
||
|
||
from yuxi.config.app import config as app_config
|
||
from yuxi.external_systems.adapters.persistence import create_repositories
|
||
from yuxi.external_systems.framework.runtime import AlertManagerImpl
|
||
from yuxi.scheduler.core.contracts import TaskContext, TaskResult
|
||
from yuxi.scheduler.framework.runtime import HandlerRegistry
|
||
from yuxi.storage.postgres.manager import pg_manager
|
||
from yuxi.utils.datetime_utils import utc_now_naive
|
||
from yuxi.utils.logging_config import logger
|
||
|
||
__all__ = ["register_scheduler_handlers"]
|
||
|
||
# 默认事件保留天数，与 ExternalWebhookSubscription.retention_days 默认值对齐
|
||
_DEFAULT_RETENTION_DAYS = 30
|
||
|
||
# 默认审计日志保留天数，与 app_config.ext_system_audit_log_retention_days 对齐
|
||
_DEFAULT_AUDIT_LOG_RETENTION_DAYS = 90
|
||
|
||
|
||
class WebhookEventCleanupHandler:
|
||
"""清理超过保留期的 Webhook 事件。
|
||
|
||
通过 scheduled_tasks 表配置周期触发，按 retention_days 软删除
|
||
ext_webhook_events 表中过期的原始事件记录。
|
||
|
||
payload 参数：
|
||
retention_days: 保留天数，未传入时使用构造函数默认值。
|
||
|
||
依赖注入：
|
||
session_factory: 调用返回 AsyncSession 的异步上下文管理器工厂，
|
||
通常为 pg_manager.get_async_session_context。每次执行创建独立
|
||
会话，避免长生命周期会话问题。
|
||
default_retention_days: 默认保留天数，未在 payload 指定时使用。
|
||
"""
|
||
|
||
name = "external_systems.webhook_event_cleanup"
|
||
description = "清理超过保留期的 Webhook 事件"
|
||
|
||
def __init__(
|
||
self,
|
||
session_factory: Callable[[], AbstractAsyncContextManager[AsyncSession]],
|
||
*,
|
||
default_retention_days: int = _DEFAULT_RETENTION_DAYS,
|
||
) -> None:
|
||
self._session_factory = session_factory
|
||
self._default_retention_days = default_retention_days
|
||
|
||
async def execute(self, ctx: TaskContext) -> TaskResult:
|
||
"""执行 Webhook 事件保留期清理。
|
||
|
||
从 payload 读取 retention_days，计算 cutoff 后委托仓储执行软删除。
|
||
异常隔离：捕获所有异常并转换为 TaskResult(success=False)，不向调度域传播。
|
||
"""
|
||
retention_days = int(ctx.payload.get("retention_days", self._default_retention_days))
|
||
cutoff = utc_now_naive() - timedelta(days=retention_days)
|
||
|
||
try:
|
||
async with self._session_factory() as db:
|
||
repos = create_repositories(db)
|
||
deleted_count = await repos.webhook_event.delete_old_events(
|
||
cutoff,
|
||
updated_by="scheduler",
|
||
)
|
||
except Exception as exc:
|
||
logger.exception(
|
||
"webhook_event_cleanup_failed",
|
||
extra={
|
||
"retention_days": retention_days,
|
||
"cutoff": cutoff,
|
||
},
|
||
)
|
||
return TaskResult(success=False, error=str(exc))
|
||
|
||
if deleted_count > 0:
|
||
logger.info(
|
||
"webhook_event_cleanup",
|
||
extra={
|
||
"deleted_count": deleted_count,
|
||
"retention_days": retention_days,
|
||
"cutoff": cutoff,
|
||
},
|
||
)
|
||
return TaskResult(
|
||
success=True,
|
||
output={
|
||
"deleted_count": deleted_count,
|
||
"retention_days": retention_days,
|
||
},
|
||
)
|
||
|
||
|
||
class PendingWebhookEventConsumerHandler:
|
||
"""定时消费 pending 状态的 Webhook 事件。
|
||
|
||
通过 ``scheduled_tasks`` 表配置周期触发，扫描 ``processing_status=pending``
|
||
的事件并调用 ``webhook_event_handler.consume_pending_event`` 完成过滤/
|
||
转换/分派闭环。单事件失败不影响其他事件。
|
||
|
||
payload 参数：
|
||
batch_size: 每次扫描最大事件数，默认 100。
|
||
"""
|
||
|
||
name = "external_systems.webhook_pending_consumer"
|
||
description = "定时消费 pending 状态的 Webhook 事件"
|
||
|
||
def __init__(
|
||
self,
|
||
session_factory: Callable[[], AbstractAsyncContextManager[AsyncSession]],
|
||
*,
|
||
default_batch_size: int = 100,
|
||
) -> None:
|
||
self._session_factory = session_factory
|
||
self._default_batch_size = default_batch_size
|
||
|
||
async def execute(self, ctx: TaskContext) -> TaskResult:
|
||
"""执行 pending 事件消费。
|
||
|
||
异常隔离：整体异常转换为 TaskResult(success=False)；单事件失败记录
|
||
到 errors 列表并继续处理其他事件。
|
||
"""
|
||
batch_size = int(ctx.payload.get("batch_size", self._default_batch_size))
|
||
consumed_count = 0
|
||
failed_count = 0
|
||
errors: list[dict[str, Any]] = []
|
||
|
||
try:
|
||
async with self._session_factory() as db:
|
||
from yuxi.external_systems.infrastructure.container import (
|
||
create_use_cases_from_db,
|
||
)
|
||
|
||
use_cases = create_use_cases_from_db(db)
|
||
repos = create_repositories(db)
|
||
pending_events = await repos.webhook_event.list_pending(limit=batch_size)
|
||
for event in pending_events:
|
||
try:
|
||
if event.id is None:
|
||
failed_count += 1
|
||
logger.warning(
|
||
"webhook_pending_consumer_skip_missing_id",
|
||
extra={"subscription_slug": event.subscription_slug},
|
||
)
|
||
continue
|
||
await use_cases.webhook_event_handler.consume_pending_event(
|
||
event.id,
|
||
)
|
||
consumed_count += 1
|
||
except Exception as exc: # noqa: BLE001 - 单事件失败不影响其他
|
||
failed_count += 1
|
||
error_info = {
|
||
"event_id": event.id,
|
||
"error": str(exc),
|
||
}
|
||
errors.append(error_info)
|
||
logger.warning(
|
||
"webhook_pending_consumer_single_failed",
|
||
extra=error_info,
|
||
)
|
||
except Exception as exc:
|
||
logger.exception(
|
||
"webhook_pending_consumer_failed",
|
||
extra={"batch_size": batch_size},
|
||
)
|
||
return TaskResult(success=False, error=str(exc))
|
||
|
||
logger.info(
|
||
"webhook_pending_consumer_completed",
|
||
extra={
|
||
"batch_size": batch_size,
|
||
"consumed_count": consumed_count,
|
||
"failed_count": failed_count,
|
||
},
|
||
)
|
||
return TaskResult(
|
||
success=True,
|
||
output={
|
||
"batch_size": batch_size,
|
||
"consumed_count": consumed_count,
|
||
"failed_count": failed_count,
|
||
"errors": errors,
|
||
},
|
||
)
|
||
|
||
|
||
class ProcessingTimeoutRecoveryHandler:
|
||
"""将 processing 状态超时的 Webhook 事件重置为 pending。
|
||
|
||
通过 ``scheduled_tasks`` 表配置周期触发，扫描 ``processing_status=processing``
|
||
且 ``last_processed_at`` 早于超时阈值的事件，将其状态恢复为 ``pending``，
|
||
供 ``PendingWebhookEventConsumerHandler`` 下一次调度时消费。
|
||
|
||
payload 参数：
|
||
timeout_seconds: 超时阈值（秒），默认 300。
|
||
batch_size: 每次扫描最大事件数，默认 100。
|
||
"""
|
||
|
||
name = "external_systems.webhook_processing_recovery"
|
||
description = "将 processing 状态超时的 Webhook 事件重置为 pending"
|
||
|
||
def __init__(
|
||
self,
|
||
session_factory: Callable[[], AbstractAsyncContextManager[AsyncSession]],
|
||
*,
|
||
default_timeout_seconds: int = 300,
|
||
default_batch_size: int = 100,
|
||
) -> None:
|
||
self._session_factory = session_factory
|
||
self._default_timeout_seconds = default_timeout_seconds
|
||
self._default_batch_size = default_batch_size
|
||
|
||
async def execute(self, ctx: TaskContext) -> TaskResult:
|
||
"""执行 processing 超时恢复。
|
||
|
||
异常隔离：整体异常转换为 TaskResult(success=False)；单事件失败记录
|
||
到 errors 列表并继续处理其他事件。
|
||
"""
|
||
timeout_seconds = int(ctx.payload.get("timeout_seconds", self._default_timeout_seconds))
|
||
batch_size = int(ctx.payload.get("batch_size", self._default_batch_size))
|
||
recovered_count = 0
|
||
failed_count = 0
|
||
errors: list[dict[str, Any]] = []
|
||
|
||
try:
|
||
async with self._session_factory() as db:
|
||
repos = create_repositories(db)
|
||
stuck_events = await repos.webhook_event.list_processing_stuck(
|
||
timeout_seconds,
|
||
limit=batch_size,
|
||
)
|
||
for event in stuck_events:
|
||
try:
|
||
if event.id is None:
|
||
failed_count += 1
|
||
logger.warning(
|
||
"webhook_processing_recovery_skip_missing_id",
|
||
extra={"subscription_slug": event.subscription_slug},
|
||
)
|
||
continue
|
||
await repos.webhook_event.update_processing(
|
||
event.id,
|
||
status="pending",
|
||
error=f"processing 超时（>{timeout_seconds}s），由调度器恢复为 pending",
|
||
increment_attempts=False,
|
||
commit=True,
|
||
)
|
||
recovered_count += 1
|
||
except Exception as exc: # noqa: BLE001 - 单事件失败不影响其他
|
||
failed_count += 1
|
||
error_info = {
|
||
"event_id": event.id,
|
||
"error": str(exc),
|
||
}
|
||
errors.append(error_info)
|
||
logger.warning(
|
||
"webhook_processing_recovery_single_failed",
|
||
extra=error_info,
|
||
)
|
||
except Exception as exc:
|
||
logger.exception(
|
||
"webhook_processing_recovery_failed",
|
||
extra={
|
||
"timeout_seconds": timeout_seconds,
|
||
"batch_size": batch_size,
|
||
},
|
||
)
|
||
return TaskResult(success=False, error=str(exc))
|
||
|
||
logger.info(
|
||
"webhook_processing_recovery_completed",
|
||
extra={
|
||
"timeout_seconds": timeout_seconds,
|
||
"batch_size": batch_size,
|
||
"recovered_count": recovered_count,
|
||
"failed_count": failed_count,
|
||
},
|
||
)
|
||
return TaskResult(
|
||
success=True,
|
||
output={
|
||
"timeout_seconds": timeout_seconds,
|
||
"batch_size": batch_size,
|
||
"recovered_count": recovered_count,
|
||
"failed_count": failed_count,
|
||
"errors": errors,
|
||
},
|
||
)
|
||
|
||
|
||
class WebhookRenewalHandler:
|
||
"""自动续期即将过期的 Webhook 订阅。
|
||
|
||
通过 scheduled_tasks 表配置周期触发，扫描 ``expires_at`` 在阈值内且
|
||
``auto_renew=True``、``status=active`` 的订阅，延长其过期时间并记录结果。
|
||
|
||
payload 参数：
|
||
threshold_days: 提前扫描天数，默认 7 天。
|
||
|
||
当前实现为平台侧续期（按 ``retention_days`` 延长过期时间）。外部系统侧
|
||
真实续期逻辑需后续按 adapter 类型扩展。
|
||
"""
|
||
|
||
name = "external_systems.webhook_renewal"
|
||
description = "自动续期即将过期的 Webhook 订阅"
|
||
|
||
def __init__(
|
||
self,
|
||
session_factory: Callable[[], AbstractAsyncContextManager[AsyncSession]],
|
||
) -> None:
|
||
self._session_factory = session_factory
|
||
|
||
async def execute(self, ctx: TaskContext) -> TaskResult:
|
||
"""执行续期扫描与续期。"""
|
||
threshold_days = int(ctx.payload.get("threshold_days", 7))
|
||
before = utc_now_naive() + timedelta(days=threshold_days)
|
||
|
||
renewed_count = 0
|
||
failed_count = 0
|
||
try:
|
||
async with self._session_factory() as db:
|
||
repos = create_repositories(db)
|
||
expiring = await repos.webhook_subscription.list_expiring(before=before)
|
||
for subscription in expiring:
|
||
try:
|
||
if subscription.id is None:
|
||
failed_count += 1
|
||
logger.warning(
|
||
"webhook_renewal_skip_missing_id",
|
||
extra={"subscription_slug": subscription.slug},
|
||
)
|
||
continue
|
||
# TODO: 按系统 adapter 类型调用外部系统真实续期接口；
|
||
# 当前为平台侧续期占位，默认视为外部续期成功。
|
||
external_renewed = True
|
||
external_error: str | None = None
|
||
if not external_renewed:
|
||
await repos.webhook_subscription.update_renewal_result(
|
||
subscription.id,
|
||
success=False,
|
||
error=external_error or "外部系统续期未实现或失败",
|
||
)
|
||
failed_count += 1
|
||
continue
|
||
new_expires_at = utc_now_naive() + timedelta(days=subscription.retention_days)
|
||
await repos.webhook_subscription.update_renewal_result(
|
||
subscription.id,
|
||
success=True,
|
||
new_expires_at=new_expires_at,
|
||
)
|
||
renewed_count += 1
|
||
except Exception: # noqa: BLE001 - 单个订阅续期失败不影响其他
|
||
failed_count += 1
|
||
logger.exception(
|
||
"webhook_renewal_single_failed",
|
||
extra={"subscription_slug": subscription.slug},
|
||
)
|
||
except Exception as exc:
|
||
logger.exception("webhook_renewal_failed", extra={"threshold_days": threshold_days})
|
||
return TaskResult(success=False, error=str(exc))
|
||
|
||
logger.info(
|
||
"webhook_renewal_completed",
|
||
extra={
|
||
"threshold_days": threshold_days,
|
||
"renewed_count": renewed_count,
|
||
"failed_count": failed_count,
|
||
},
|
||
)
|
||
return TaskResult(
|
||
success=True,
|
||
output={
|
||
"threshold_days": threshold_days,
|
||
"renewed_count": renewed_count,
|
||
"failed_count": failed_count,
|
||
},
|
||
)
|
||
|
||
|
||
class SecretRotationHandler:
|
||
"""检查到期密钥轮换策略并执行轮换/告警。
|
||
|
||
通过 scheduled_tasks 表配置周期触发，调用 ``SecretRotationService``
|
||
的 ``scheduled_rotation`` 方法完成到期策略扫描、auto 模式轮换、
|
||
manual 模式告警以及已过期策略 critical 告警。
|
||
"""
|
||
|
||
name = "external_systems.secret_rotation"
|
||
description = "检查到期密钥轮换策略并执行轮换/告警"
|
||
|
||
def __init__(
|
||
self,
|
||
session_factory: Callable[[], AbstractAsyncContextManager[AsyncSession]],
|
||
) -> None:
|
||
self._session_factory = session_factory
|
||
|
||
async def execute(self, ctx: TaskContext) -> TaskResult:
|
||
"""执行密钥轮换调度。
|
||
|
||
异常隔离：捕获所有异常并转换为 TaskResult(success=False)，不向调度域传播。
|
||
"""
|
||
_ = ctx # 本 handler 无需 payload 参数
|
||
try:
|
||
async with self._session_factory() as db:
|
||
# 延迟导入 create_use_cases_from_db：避免在模块加载阶段触发
|
||
# infrastructure container 的装配（container 依赖 app_config，
|
||
# 在本地非容器测试环境中可能不存在）。
|
||
from yuxi.external_systems.infrastructure.container import (
|
||
create_use_cases_from_db,
|
||
)
|
||
|
||
use_cases = create_use_cases_from_db(db)
|
||
output = await use_cases.secret_rotation_service.scheduled_rotation()
|
||
except Exception as exc:
|
||
logger.exception("secret_rotation_failed")
|
||
return TaskResult(success=False, error=str(exc))
|
||
|
||
return TaskResult(
|
||
success=True,
|
||
output=output.model_dump(mode="json"),
|
||
)
|
||
|
||
|
||
def _compute_next_window(
|
||
quota_window: str,
|
||
window_start: datetime | None,
|
||
window_end: datetime | None,
|
||
now: datetime,
|
||
) -> tuple[datetime, datetime]:
|
||
"""根据配额窗口类型计算下一窗口起止时间。
|
||
|
||
规则：
|
||
- rolling_24h: 以当前时间为起点，向后滚动 24h。
|
||
- daily/hourly/minute: 以上一窗口结束时间为起点，按固定步长推进。
|
||
- custom: 保持上一窗口长度；无法推断时回退 1 天。
|
||
"""
|
||
if quota_window == "rolling_24h":
|
||
return now, now + timedelta(hours=24)
|
||
|
||
last_end = window_end or now
|
||
if quota_window == "daily":
|
||
delta = timedelta(days=1)
|
||
elif quota_window == "hourly":
|
||
delta = timedelta(hours=1)
|
||
elif quota_window == "minute":
|
||
delta = timedelta(minutes=1)
|
||
elif window_start is not None and window_end is not None:
|
||
delta = window_end - window_start
|
||
if delta.total_seconds() <= 0:
|
||
delta = timedelta(days=1)
|
||
else:
|
||
delta = timedelta(days=1)
|
||
return last_end, last_end + delta
|
||
|
||
|
||
class QuotaThresholdAlertHandler:
|
||
"""扫描达到告警/临界阈值的外部系统配额并触发告警。
|
||
|
||
payload 参数：
|
||
limit: 每次扫描最大记录数，默认 100。
|
||
"""
|
||
|
||
name = "external_systems.quota_threshold_alert"
|
||
description = "扫描达到告警/临界阈值的外部系统配额并触发告警"
|
||
|
||
def __init__(
|
||
self,
|
||
session_factory: Callable[[], AbstractAsyncContextManager[AsyncSession]],
|
||
*,
|
||
default_limit: int = 100,
|
||
) -> None:
|
||
self._session_factory = session_factory
|
||
self._default_limit = default_limit
|
||
|
||
async def execute(self, ctx: TaskContext) -> TaskResult:
|
||
"""执行配额阈值告警扫描。
|
||
|
||
先查询 critical 阈值记录（高优先级），再查询 warning 阈值记录并
|
||
排除已触达 critical 的记录，避免同一配额同时产生两条不同严重级别告警。
|
||
"""
|
||
limit = int(ctx.payload.get("limit", self._default_limit))
|
||
|
||
try:
|
||
async with self._session_factory() as db:
|
||
repos = create_repositories(db)
|
||
alert_manager = AlertManagerImpl(repos.alert)
|
||
|
||
critical_quotas = await repos.quota_usage.list_near_critical(limit=limit)
|
||
critical_ids = {q.id for q in critical_quotas if q.id is not None}
|
||
for quota in critical_quotas:
|
||
await self._fire_alert(alert_manager, quota, "critical")
|
||
|
||
warning_quotas = await repos.quota_usage.list_near_warning(limit=limit)
|
||
warning_count = 0
|
||
for quota in warning_quotas:
|
||
if quota.id in critical_ids:
|
||
continue
|
||
await self._fire_alert(alert_manager, quota, "warning")
|
||
warning_count += 1
|
||
except Exception as exc:
|
||
logger.exception("quota_threshold_alert_failed")
|
||
return TaskResult(success=False, error=str(exc))
|
||
|
||
return TaskResult(
|
||
success=True,
|
||
output={
|
||
"critical_count": len(critical_quotas),
|
||
"warning_count": warning_count,
|
||
},
|
||
)
|
||
|
||
async def _fire_alert(
|
||
self,
|
||
alert_manager: AlertManagerImpl,
|
||
quota: Any,
|
||
severity: str,
|
||
) -> None:
|
||
"""为单个配额触发告警。"""
|
||
title = f"配额接近上限: {quota.quota_name}" if severity == "warning" else f"配额即将耗尽: {quota.quota_name}"
|
||
description = (
|
||
f"system_id={quota.system_id}, env={quota.env_key}, "
|
||
f"key={quota.quota_key}, 已用 {quota.usage_ratio * 100:.1f}%"
|
||
)
|
||
await alert_manager.fire(
|
||
quota.system_id,
|
||
quota.env_key,
|
||
"quota_near_limit",
|
||
severity=severity, # type: ignore[arg-type]
|
||
title=title,
|
||
description=description,
|
||
resource_type="quota_usage",
|
||
resource_id=quota.quota_key,
|
||
metric_snapshot={
|
||
"limit_value": quota.limit_value,
|
||
"used_value": quota.used_value,
|
||
"usage_ratio": quota.usage_ratio,
|
||
"warning_threshold": quota.warning_threshold,
|
||
"critical_threshold": quota.critical_threshold,
|
||
},
|
||
)
|
||
|
||
|
||
class QuotaWindowResetHandler:
|
||
"""自动重置已到期外部系统配额窗口。
|
||
|
||
payload 参数：
|
||
limit: 每次扫描最大记录数，默认 100。
|
||
"""
|
||
|
||
name = "external_systems.quota_window_reset"
|
||
description = "自动重置已到期的外部系统配额窗口"
|
||
|
||
def __init__(
|
||
self,
|
||
session_factory: Callable[[], AbstractAsyncContextManager[AsyncSession]],
|
||
*,
|
||
default_limit: int = 100,
|
||
) -> None:
|
||
self._session_factory = session_factory
|
||
self._default_limit = default_limit
|
||
|
||
async def execute(self, ctx: TaskContext) -> TaskResult:
|
||
"""执行配额窗口自动重置。
|
||
|
||
查询 ``window_end <= now`` 的配额记录，按窗口类型计算下一窗口并
|
||
将 ``used_value`` 清零。单条失败不影响后续记录。
|
||
"""
|
||
limit = int(ctx.payload.get("limit", self._default_limit))
|
||
now = utc_now_naive()
|
||
|
||
try:
|
||
async with self._session_factory() as db:
|
||
repos = create_repositories(db)
|
||
quotas = await repos.quota_usage.list(expiring_before=now, limit=limit)
|
||
|
||
reset_count = 0
|
||
failed_count = 0
|
||
errors: list[str] = []
|
||
for quota in quotas:
|
||
try:
|
||
window_start, window_end = _compute_next_window(
|
||
quota.quota_window,
|
||
quota.window_start,
|
||
quota.window_end,
|
||
now,
|
||
)
|
||
await repos.quota_usage.reset_window(
|
||
quota.system_id,
|
||
quota.env_key,
|
||
quota.quota_key,
|
||
window_start=window_start,
|
||
window_end=window_end,
|
||
used_value=0,
|
||
updated_by="scheduler",
|
||
commit=True,
|
||
)
|
||
reset_count += 1
|
||
except Exception as exc: # noqa: BLE001
|
||
failed_count += 1
|
||
error_msg = f"{quota.system_id}/{quota.env_key}/{quota.quota_key}: {exc}"
|
||
errors.append(error_msg)
|
||
logger.warning(
|
||
"quota_window_reset_failed",
|
||
extra={
|
||
"system_id": quota.system_id,
|
||
"env_key": quota.env_key,
|
||
"quota_key": quota.quota_key,
|
||
"error": error_msg,
|
||
},
|
||
)
|
||
except Exception as exc:
|
||
logger.exception("quota_window_reset_failed")
|
||
return TaskResult(success=False, error=str(exc))
|
||
|
||
return TaskResult(
|
||
success=True,
|
||
output={
|
||
"reset_count": reset_count,
|
||
"failed_count": failed_count,
|
||
"errors": errors,
|
||
},
|
||
)
|
||
|
||
|
||
class AuditLogRetentionHandler:
|
||
"""清理超过保留期的外部系统审计日志。
|
||
|
||
通过 ``scheduled_tasks`` 表配置周期触发，软删除 ``ext_system_audit_logs``
|
||
表中早于保留期的记录。对齐 channels 模块的 ``ChannelAuditLogRetentionHandler``。
|
||
|
||
payload 参数：
|
||
retention_days: 保留天数，默认取 app_config.ext_system_audit_log_retention_days。
|
||
"""
|
||
|
||
name = "external_systems.audit_log_retention"
|
||
description = "清理超过保留期的外部系统审计日志"
|
||
|
||
def __init__(
|
||
self,
|
||
session_factory: Callable[[], AbstractAsyncContextManager[AsyncSession]],
|
||
*,
|
||
default_retention_days: int = _DEFAULT_AUDIT_LOG_RETENTION_DAYS,
|
||
) -> None:
|
||
self._session_factory = session_factory
|
||
self._default_retention_days = default_retention_days
|
||
|
||
async def execute(self, ctx: TaskContext) -> TaskResult:
|
||
"""执行审计日志保留期清理。
|
||
|
||
异常隔离：捕获所有异常并转换为 TaskResult(success=False)，不向调度域传播。
|
||
"""
|
||
retention_days = int(ctx.payload.get("retention_days", self._default_retention_days))
|
||
cutoff = utc_now_naive() - timedelta(days=retention_days)
|
||
|
||
try:
|
||
async with self._session_factory() as db:
|
||
repos = create_repositories(db)
|
||
deleted_count = await repos.audit_log.delete_old_logs(
|
||
cutoff,
|
||
updated_by="scheduler",
|
||
)
|
||
except Exception as exc:
|
||
logger.exception(
|
||
"ext_audit_log_retention_failed",
|
||
extra={
|
||
"retention_days": retention_days,
|
||
"cutoff": cutoff,
|
||
},
|
||
)
|
||
return TaskResult(success=False, error=str(exc))
|
||
|
||
if deleted_count > 0:
|
||
logger.info(
|
||
"ext_audit_log_retention",
|
||
extra={
|
||
"deleted_count": deleted_count,
|
||
"retention_days": retention_days,
|
||
"cutoff": cutoff,
|
||
},
|
||
)
|
||
return TaskResult(
|
||
success=True,
|
||
output={
|
||
"deleted_count": deleted_count,
|
||
"retention_days": retention_days,
|
||
},
|
||
)
|
||
|
||
|
||
# 回收站定时清理默认保留天数（软删除超过此天数的记录物理删除）
|
||
_DEFAULT_TRASH_RETENTION_DAYS = 90
|
||
|
||
|
||
class TrashPurgeHandler:
|
||
"""定时清理回收站中超过保留期的软删除资源。
|
||
|
||
通过 ``scheduled_tasks`` 表配置周期触发，调用 ``TrashService.purge`` 物理删除
|
||
超过 ``older_than_days`` 天的软删除记录。对齐手动清理（POST /trash/purge）
|
||
的语义，复用同一 service 实现保证事务边界 / 审计 / 事件一致。
|
||
|
||
payload 参数：
|
||
older_than_days: 保留天数，默认 90。
|
||
"""
|
||
|
||
name = "external_systems.trash_purge"
|
||
description = "定时清理回收站中超过保留期的软删除资源"
|
||
|
||
def __init__(
|
||
self,
|
||
session_factory: Callable[[], AbstractAsyncContextManager[AsyncSession]],
|
||
*,
|
||
default_older_than_days: int = _DEFAULT_TRASH_RETENTION_DAYS,
|
||
) -> None:
|
||
self._session_factory = session_factory
|
||
self._default_older_than_days = default_older_than_days
|
||
|
||
async def execute(self, ctx: TaskContext) -> TaskResult:
|
||
"""执行回收站定时清理。
|
||
|
||
异常隔离：捕获所有异常并转换为 TaskResult(success=False)，不向调度域传播。
|
||
"""
|
||
older_than_days = int(ctx.payload.get("older_than_days", self._default_older_than_days))
|
||
|
||
try:
|
||
async with self._session_factory() as db:
|
||
# 延迟导入：避免模块加载阶段触发 infrastructure container 装配
|
||
from yuxi.external_systems.infrastructure.container import (
|
||
create_use_cases_from_db,
|
||
)
|
||
from yuxi.external_systems.use_cases.dto.trash import PurgeTrashInput
|
||
|
||
use_cases = create_use_cases_from_db(db)
|
||
output = await use_cases.trash_service.purge(
|
||
PurgeTrashInput(
|
||
older_than_days=older_than_days,
|
||
purged_by="scheduler",
|
||
)
|
||
)
|
||
except Exception as exc:
|
||
logger.exception(
|
||
"trash_purge_failed",
|
||
extra={"older_than_days": older_than_days},
|
||
)
|
||
return TaskResult(success=False, error=str(exc))
|
||
|
||
if output.purged_count > 0:
|
||
logger.info(
|
||
"trash_purge_completed",
|
||
extra={
|
||
"purged_count": output.purged_count,
|
||
"by_type": output.by_type,
|
||
"older_than_days": older_than_days,
|
||
"cutoff_at": output.cutoff_at.isoformat(),
|
||
},
|
||
)
|
||
return TaskResult(
|
||
success=True,
|
||
output=output.model_dump(mode="json"),
|
||
)
|
||
|
||
|
||
class HealthCheckSchedulerHandler:
|
||
"""定时健康检查探测。
|
||
|
||
通过 ``scheduled_tasks`` 表配置周期触发，扫描所有启用的外部系统，
|
||
对每个系统调用 ``tool_service.trigger_system_health_check`` 执行探测。
|
||
单个系统失败不影响其他系统。
|
||
|
||
payload 参数：
|
||
batch_size: 每次扫描最大系统数，默认 50。
|
||
"""
|
||
|
||
name = "external_systems.health_check"
|
||
description = "定时健康检查探测"
|
||
|
||
def __init__(
|
||
self,
|
||
session_factory: Callable[[], AbstractAsyncContextManager[AsyncSession]],
|
||
*,
|
||
default_batch_size: int = 50,
|
||
) -> None:
|
||
self._session_factory = session_factory
|
||
self._default_batch_size = default_batch_size
|
||
|
||
async def execute(self, ctx: TaskContext) -> TaskResult:
|
||
"""执行定时健康检查。
|
||
|
||
异常隔离：捕获所有异常并转换为 TaskResult(success=False)，不向调度域传播。
|
||
单个系统失败记录到 errors 列表，不影响整体任务成功。
|
||
"""
|
||
batch_size = int(ctx.payload.get("batch_size", self._default_batch_size))
|
||
checked_count = 0
|
||
failed_count = 0
|
||
errors: list[dict[str, Any]] = []
|
||
|
||
try:
|
||
async with self._session_factory() as db:
|
||
from yuxi.external_systems.infrastructure.container import (
|
||
create_use_cases_from_db,
|
||
)
|
||
from yuxi.external_systems.use_cases.dto.health_check import (
|
||
TriggerSystemHealthCheckInput,
|
||
)
|
||
|
||
repos = create_repositories(db)
|
||
use_cases = create_use_cases_from_db(db)
|
||
system_ids = await repos.system.list_ids(
|
||
enabled=True,
|
||
sort_by="created_at",
|
||
sort_order="asc",
|
||
)
|
||
for system_id in system_ids[:batch_size]:
|
||
try:
|
||
await use_cases.tool_service.trigger_system_health_check(
|
||
TriggerSystemHealthCheckInput(
|
||
system_id=system_id,
|
||
triggered_by="scheduler",
|
||
),
|
||
)
|
||
checked_count += 1
|
||
except Exception as exc: # noqa: BLE001 - 单系统失败不影响其他
|
||
failed_count += 1
|
||
errors.append({"system_id": system_id, "error": str(exc)})
|
||
logger.warning(
|
||
"health_check_single_failed",
|
||
extra={"system_id": system_id, "error": str(exc)},
|
||
)
|
||
except Exception as exc:
|
||
logger.exception("health_check_scheduler_failed")
|
||
return TaskResult(success=False, error=str(exc))
|
||
|
||
logger.info(
|
||
"health_check_scheduler_completed",
|
||
extra={
|
||
"checked_count": checked_count,
|
||
"failed_count": failed_count,
|
||
"batch_size": batch_size,
|
||
},
|
||
)
|
||
return TaskResult(
|
||
success=True,
|
||
output={
|
||
"checked_count": checked_count,
|
||
"failed_count": failed_count,
|
||
"batch_size": batch_size,
|
||
"errors": errors,
|
||
},
|
||
)
|
||
|
||
|
||
class TestRegressionSchedulerHandler:
|
||
"""定时执行到期的测试用例回归。
|
||
|
||
通过 ``scheduled_tasks`` 表配置周期触发，调用 ``TestRegressionService``
|
||
的 ``scheduled_regression`` 方法执行所有到期用例。执行结果中的单条异常
|
||
已由 service 层隔离，本 handler 仅做整体异常转换。
|
||
|
||
payload 参数：
|
||
当前无需 payload 参数（服务层内部决定到期用例）。
|
||
"""
|
||
|
||
name = "external_systems.test_regression"
|
||
description = "定时执行到期的外部系统工具测试用例回归"
|
||
|
||
def __init__(
|
||
self,
|
||
session_factory: Callable[[], AbstractAsyncContextManager[AsyncSession]],
|
||
) -> None:
|
||
self._session_factory = session_factory
|
||
|
||
async def execute(self, ctx: TaskContext) -> TaskResult:
|
||
"""执行到期测试用例回归。
|
||
|
||
异常隔离：捕获所有异常并转换为 TaskResult(success=False)，不向调度域传播。
|
||
"""
|
||
_ = ctx
|
||
try:
|
||
async with self._session_factory() as db:
|
||
from yuxi.external_systems.infrastructure.container import (
|
||
create_use_cases_from_db,
|
||
)
|
||
|
||
use_cases = create_use_cases_from_db(db)
|
||
output = await use_cases.test_regression_service.scheduled_regression()
|
||
except Exception as exc:
|
||
logger.exception("test_regression_scheduler_failed")
|
||
return TaskResult(success=False, error=str(exc))
|
||
|
||
return TaskResult(
|
||
success=True,
|
||
output=output.model_dump(mode="json"),
|
||
)
|
||
|
||
|
||
def register_scheduler_handlers(registry: HandlerRegistry) -> None:
|
||
"""向 scheduler 注册 external_systems 模块的定时任务 handler。
|
||
|
||
由 worker 启动钩子在装配阶段统一调用。新增 handler 时在此函数内追加
|
||
``registry.register(...)`` 即可。
|
||
|
||
Args:
|
||
registry: worker 进程级 handler 注册表。
|
||
"""
|
||
registry.register(
|
||
WebhookEventCleanupHandler(
|
||
session_factory=pg_manager.get_async_session_context,
|
||
),
|
||
)
|
||
registry.register(
|
||
PendingWebhookEventConsumerHandler(
|
||
session_factory=pg_manager.get_async_session_context,
|
||
),
|
||
)
|
||
registry.register(
|
||
ProcessingTimeoutRecoveryHandler(
|
||
session_factory=pg_manager.get_async_session_context,
|
||
),
|
||
)
|
||
registry.register(
|
||
QuotaThresholdAlertHandler(
|
||
session_factory=pg_manager.get_async_session_context,
|
||
),
|
||
)
|
||
registry.register(
|
||
QuotaWindowResetHandler(
|
||
session_factory=pg_manager.get_async_session_context,
|
||
),
|
||
)
|
||
registry.register(
|
||
WebhookRenewalHandler(
|
||
session_factory=pg_manager.get_async_session_context,
|
||
),
|
||
)
|
||
registry.register(
|
||
SecretRotationHandler(
|
||
session_factory=pg_manager.get_async_session_context,
|
||
),
|
||
)
|
||
registry.register(
|
||
AuditLogRetentionHandler(
|
||
session_factory=pg_manager.get_async_session_context,
|
||
default_retention_days=getattr(
|
||
app_config, "ext_system_audit_log_retention_days", _DEFAULT_AUDIT_LOG_RETENTION_DAYS
|
||
),
|
||
),
|
||
)
|
||
registry.register(
|
||
TrashPurgeHandler(
|
||
session_factory=pg_manager.get_async_session_context,
|
||
),
|
||
)
|
||
registry.register(
|
||
HealthCheckSchedulerHandler(
|
||
session_factory=pg_manager.get_async_session_context,
|
||
),
|
||
)
|
||
registry.register(
|
||
TestRegressionSchedulerHandler(
|
||
session_factory=pg_manager.get_async_session_context,
|
||
),
|
||
)
|