97 lines
3.7 KiB
Python
97 lines
3.7 KiB
Python
"""渠道出站消息补偿任务。"""
|
|
|
|
import asyncio
|
|
from datetime import timedelta
|
|
|
|
from yuxi.channel.constants import (
|
|
CHANNEL_COMPENSATE_LOCK_KEY,
|
|
CHANNEL_COMPENSATE_LOCK_TTL_SECONDS,
|
|
CHANNEL_MAX_COMPENSATE_ATTEMPTS,
|
|
CHANNEL_PROCESSING_LOCK_RENEW_INTERVAL_SECONDS,
|
|
DeliveryStatus,
|
|
)
|
|
from yuxi.channel.outbound.lock import RedisLockRenewer
|
|
from yuxi.channel.outbound.publisher import publish_channel_message_event
|
|
from yuxi.repositories.conversation_repository import ConversationRepository
|
|
from yuxi.repositories.message_repository import MessageRepository
|
|
from yuxi.services.run_queue_service import get_redis_client
|
|
from yuxi.storage.postgres.manager import pg_manager
|
|
from yuxi.utils.logging_config import logger
|
|
|
|
|
|
async def compensate_channel_messages() -> None:
|
|
"""扫描未投递完成的渠道消息并重投事件。
|
|
|
|
由 API 容器周期性调用，兜底处理 Redis Stream 投递失败或消息丢失的场景。
|
|
使用 Redis 分布式锁避免多实例并发补偿同一条消息；锁内置续期防止长扫描超时。
|
|
compensate_attempts 与 dispatcher 的 attempts 解耦，单独计数。
|
|
"""
|
|
redis = await get_redis_client()
|
|
lock_value = "1"
|
|
locked = await redis.set(CHANNEL_COMPENSATE_LOCK_KEY, lock_value, nx=True, ex=CHANNEL_COMPENSATE_LOCK_TTL_SECONDS)
|
|
if not locked:
|
|
logger.debug("Compensate task is already running on another instance")
|
|
return
|
|
|
|
async with RedisLockRenewer(
|
|
redis,
|
|
CHANNEL_COMPENSATE_LOCK_KEY,
|
|
ttl=CHANNEL_COMPENSATE_LOCK_TTL_SECONDS,
|
|
interval=CHANNEL_PROCESSING_LOCK_RENEW_INTERVAL_SECONDS,
|
|
):
|
|
await _do_compensate()
|
|
|
|
try:
|
|
await redis.delete(CHANNEL_COMPENSATE_LOCK_KEY)
|
|
except Exception:
|
|
logger.warning("Failed to release compensate lock %s", CHANNEL_COMPENSATE_LOCK_KEY)
|
|
|
|
|
|
async def _do_compensate() -> None:
|
|
async with pg_manager.get_async_session_context() as session:
|
|
message_repo = MessageRepository(session)
|
|
conversation_repo = ConversationRepository(session)
|
|
|
|
pending_messages = await message_repo.list_pending_channel_messages(
|
|
older_than=timedelta(minutes=1),
|
|
limit=50,
|
|
)
|
|
|
|
for message in pending_messages:
|
|
conversation = await conversation_repo.get_conversation_by_id(message.conversation_id)
|
|
if not conversation:
|
|
continue
|
|
|
|
session_key = conversation.channel_session_id
|
|
if not session_key:
|
|
continue
|
|
|
|
compensate_attempts = (message.channel_metadata or {}).get("compensate_attempts", 0) + 1
|
|
message.update_channel_metadata({"compensate_attempts": compensate_attempts})
|
|
await session.commit()
|
|
|
|
if compensate_attempts > CHANNEL_MAX_COMPENSATE_ATTEMPTS:
|
|
logger.warning(
|
|
"Channel message %s exceeded max compensate attempts (%s), marking dead_letter",
|
|
message.id,
|
|
CHANNEL_MAX_COMPENSATE_ATTEMPTS,
|
|
)
|
|
message.delivery_status = DeliveryStatus.DEAD_LETTER
|
|
await session.commit()
|
|
continue
|
|
|
|
try:
|
|
await publish_channel_message_event(
|
|
message=message,
|
|
conversation=conversation,
|
|
session_key=session_key,
|
|
)
|
|
logger.info(
|
|
"Compensated channel message (message_id=%s, conversation_id=%s, compensate_attempt=%s)",
|
|
message.id,
|
|
conversation.id,
|
|
compensate_attempts,
|
|
)
|
|
except Exception:
|
|
logger.exception("Failed to compensate channel message (message_id=%s)", message.id)
|