Kris
7a972055b7
新增大量渠道适配器相关的协议、策略、工具类与基础设施代码，包括： 1. 多协议定义：认证、消息、配置、网关等核心接口 2. 策略模块：上下文、群聊、去重、防抖等业务策略 3. 工具集：重试、去重、文本分块、消息格式化等SDK工具 4. 基础设施：外部进程管理、事件广播、熔断机制等 5. 账户与管道系统：账户管理、消息处理管道实现 6. 运行时服务：状态收集、维护任务、日志等后台服务
80 lines
2.6 KiB
Python
80 lines
2.6 KiB
Python
import asyncio
|
|
import hashlib
|
|
|
|
from cachetools import TTLCache
|
|
|
|
from yuxi.channels.models import ChannelMessage
|
|
from yuxi.utils.logging_config import logger
|
|
|
|
|
|
class DedupPolicy:
|
|
def __init__(self, ttl: int = 300, maxsize: int = 10000):
|
|
self._seen: TTLCache = TTLCache(maxsize=maxsize, ttl=ttl)
|
|
self._lock = asyncio.Lock()
|
|
|
|
async def is_duplicate(self, message: ChannelMessage) -> bool:
|
|
msg_id = message.identity.channel_message_id
|
|
if not msg_id:
|
|
return False
|
|
key = f"{message.identity.channel_id}:{msg_id}"
|
|
async with self._lock:
|
|
if key in self._seen:
|
|
logger.debug(f"Duplicate message filtered: {key}")
|
|
return True
|
|
self._seen[key] = True
|
|
return False
|
|
|
|
async def check_and_remember(self, message: ChannelMessage) -> bool:
|
|
async with self._lock:
|
|
if self._check_identity_key(message):
|
|
return True
|
|
if self._check_fingerprint(message):
|
|
return True
|
|
self._record(message)
|
|
return False
|
|
|
|
def _check_identity_key(self, message: ChannelMessage) -> bool:
|
|
msg_id = message.identity.channel_message_id
|
|
if not msg_id:
|
|
return False
|
|
key = f"{message.identity.channel_id}:{msg_id}"
|
|
if key in self._seen:
|
|
logger.debug(f"Duplicate message filtered (identity): {key}")
|
|
return True
|
|
return False
|
|
|
|
def _check_fingerprint(self, message: ChannelMessage) -> bool:
|
|
fp = self._build_fingerprint(message)
|
|
if not fp:
|
|
return False
|
|
key = f"fp:{message.identity.channel_id}:{fp}"
|
|
if key in self._seen:
|
|
logger.debug(f"Duplicate message filtered (fingerprint): {key}")
|
|
return True
|
|
return False
|
|
|
|
def _record(self, message: ChannelMessage) -> None:
|
|
msg_id = message.identity.channel_message_id
|
|
if msg_id:
|
|
self._seen[f"{message.identity.channel_id}:{msg_id}"] = True
|
|
fp = self._build_fingerprint(message)
|
|
if fp:
|
|
self._seen[f"fp:{message.identity.channel_id}:{fp}"] = True
|
|
|
|
@staticmethod
|
|
def _build_fingerprint(message: ChannelMessage) -> str | None:
|
|
if not message.content:
|
|
return None
|
|
raw = "|".join(
|
|
[
|
|
message.identity.channel_id,
|
|
message.identity.channel_chat_id,
|
|
message.content,
|
|
]
|
|
)
|
|
return hashlib.sha256(raw.encode()).hexdigest()[:32]
|
|
|
|
async def clear(self) -> None:
|
|
async with self._lock:
|
|
self._seen.clear()
|