80 lines
2.6 KiB
Python
80 lines
2.6 KiB
Python
|
import asyncio
|
||
|
|
import hashlib
|
||
|
|
|
||
|
|
from cachetools import TTLCache
|
||
|
|
|
||
|
|
from yuxi.channels.models import ChannelMessage
|
||
|
|
from yuxi.utils.logging_config import logger
|
||
|
|
|
||
|
|
|
||
|
|
class DedupPolicy:
|
||
|
|
def __init__(self, ttl: int = 300, maxsize: int = 10000):
|
||
|
|
self._seen: TTLCache = TTLCache(maxsize=maxsize, ttl=ttl)
|
||
|
|
self._lock = asyncio.Lock()
|
||
|
|
|
||
|
|
async def is_duplicate(self, message: ChannelMessage) -> bool:
|
||
|
|
msg_id = message.identity.channel_message_id
|
||
|
|
if not msg_id:
|
||
|
|
return False
|
||
|
|
key = f"{message.identity.channel_id}:{msg_id}"
|
||
|
|
async with self._lock:
|
||
|
|
if key in self._seen:
|
||
|
|
logger.debug(f"Duplicate message filtered: {key}")
|
||
|
|
return True
|
||
|
|
self._seen[key] = True
|
||
|
|
return False
|
||
|
|
|
||
|
|
async def check_and_remember(self, message: ChannelMessage) -> bool:
|
||
|
|
async with self._lock:
|
||
|
|
if self._check_identity_key(message):
|
||
|
|
return True
|
||
|
|
if self._check_fingerprint(message):
|
||
|
|
return True
|
||
|
|
self._record(message)
|
||
|
|
return False
|
||
|
|
|
||
|
|
def _check_identity_key(self, message: ChannelMessage) -> bool:
|
||
|
|
msg_id = message.identity.channel_message_id
|
||
|
|
if not msg_id:
|
||
|
|
return False
|
||
|
|
key = f"{message.identity.channel_id}:{msg_id}"
|
||
|
|
if key in self._seen:
|
||
|
|
logger.debug(f"Duplicate message filtered (identity): {key}")
|
||
|
|
return True
|
||
|
|
return False
|
||
|
|
|
||
|
|
def _check_fingerprint(self, message: ChannelMessage) -> bool:
|
||
|
|
fp = self._build_fingerprint(message)
|
||
|
|
if not fp:
|
||
|
|
return False
|
||
|
|
key = f"fp:{message.identity.channel_id}:{fp}"
|
||
|
|
if key in self._seen:
|
||
|
|
logger.debug(f"Duplicate message filtered (fingerprint): {key}")
|
||
|
|
return True
|
||
|
|
return False
|
||
|
|
|
||
|
|
def _record(self, message: ChannelMessage) -> None:
|
||
|
|
msg_id = message.identity.channel_message_id
|
||
|
|
if msg_id:
|
||
|
|
self._seen[f"{message.identity.channel_id}:{msg_id}"] = True
|
||
|
|
fp = self._build_fingerprint(message)
|
||
|
|
if fp:
|
||
|
|
self._seen[f"fp:{message.identity.channel_id}:{fp}"] = True
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def _build_fingerprint(message: ChannelMessage) -> str | None:
|
||
|
|
if not message.content:
|
||
|
|
return None
|
||
|
|
raw = "|".join(
|
||
|
|
[
|
||
|
|
message.identity.channel_id,
|
||
|
|
message.identity.channel_chat_id,
|
||
|
|
message.content,
|
||
|
|
]
|
||
|
|
)
|
||
|
|
return hashlib.sha256(raw.encode()).hexdigest()[:32]
|
||
|
|
|
||
|
|
async def clear(self) -> None:
|
||
|
|
async with self._lock:
|
||
|
|
self._seen.clear()
|