Kris
3ad218e537
新增群晖Chat渠道适配器的全套实现，包括： 1. 基础适配器与导出接口定义 2. DSM API认证、探测与会话管理 3. 轮询与Webhook两种消息接收方式 4. 消息去重、格式化与规范化处理 5. 多账号支持与权限安全策略 6. 目录用户/群组发现功能 7. 审批配对与流量控制机制 8. 安全审计与配置检查功能
130 lines
4.3 KiB
Python
130 lines
4.3 KiB
Python
"""Message deduplication for Synology Chat inbound events.
|
|
|
|
Prevents duplicate message processing using an LRU-style in-memory cache
|
|
keyed by message_id. Also deduplicates by content hash within short time
|
|
windows for additional safety in polling mode.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import time
|
|
from collections import OrderedDict
|
|
from typing import Any
|
|
|
|
from yuxi.utils.logging_config import logger
|
|
|
|
_DEFAULT_MAX_ENTRIES = 1000
|
|
_DEFAULT_TTL_SECONDS = 300
|
|
_DEFAULT_CONTENT_TTL_SECONDS = 60
|
|
|
|
|
|
class MessageDeduplicator:
|
|
"""LRU-based message deduplication cache.
|
|
|
|
Tracks processed message_ids and content hashes, expiring entries
|
|
after configurable TTL. Thread-compatible (not thread-safe).
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
max_entries: int = _DEFAULT_MAX_ENTRIES,
|
|
ttl_seconds: int = _DEFAULT_TTL_SECONDS,
|
|
content_ttl_seconds: int = _DEFAULT_CONTENT_TTL_SECONDS,
|
|
):
|
|
self._max_entries = max_entries
|
|
self._ttl = ttl_seconds
|
|
self._content_ttl = content_ttl_seconds
|
|
self._message_ids: OrderedDict[str, float] = OrderedDict()
|
|
self._content_hashes: OrderedDict[str, float] = OrderedDict()
|
|
|
|
def is_duplicate(self, message_id: str) -> bool:
|
|
"""Check if a message_id has already been processed."""
|
|
if not message_id:
|
|
return False
|
|
self._expire_message_ids()
|
|
return message_id in self._message_ids
|
|
|
|
def mark_processed(self, message_id: str) -> None:
|
|
"""Record a message_id as processed."""
|
|
if not message_id:
|
|
return
|
|
self._expire_message_ids()
|
|
if message_id in self._message_ids:
|
|
self._message_ids.move_to_end(message_id)
|
|
else:
|
|
if len(self._message_ids) >= self._max_entries:
|
|
self._message_ids.popitem(last=False)
|
|
self._message_ids[message_id] = time.time()
|
|
|
|
def is_content_duplicate(self, content: str, chat_id: str = "") -> bool:
|
|
"""Check if the same content from the same chat was recently processed."""
|
|
if not content:
|
|
return False
|
|
hash_key = _content_hash(content, chat_id)
|
|
self._expire_content_hashes()
|
|
return hash_key in self._content_hashes
|
|
|
|
def mark_content_processed(self, content: str, chat_id: str = "") -> None:
|
|
"""Record content as processed for a chat."""
|
|
if not content:
|
|
return
|
|
hash_key = _content_hash(content, chat_id)
|
|
self._expire_content_hashes()
|
|
if len(self._content_hashes) >= self._max_entries:
|
|
self._content_hashes.popitem(last=False)
|
|
self._content_hashes[hash_key] = time.time()
|
|
|
|
def check_and_mark(self, message_id: str) -> bool:
|
|
"""Check duplicate and mark if new. Returns True if duplicate."""
|
|
if self.is_duplicate(message_id):
|
|
return True
|
|
self.mark_processed(message_id)
|
|
return False
|
|
|
|
def _expire_message_ids(self) -> None:
|
|
now = time.time()
|
|
expired = [k for k, ts in self._message_ids.items() if now - ts > self._ttl]
|
|
for k in expired:
|
|
del self._message_ids[k]
|
|
|
|
def _expire_content_hashes(self) -> None:
|
|
now = time.time()
|
|
expired = [k for k, ts in self._content_hashes.items() if now - ts > self._content_ttl]
|
|
for k in expired:
|
|
del self._content_hashes[k]
|
|
|
|
def __len__(self) -> int:
|
|
return len(self._message_ids)
|
|
|
|
def clear(self) -> None:
|
|
self._message_ids.clear()
|
|
self._content_hashes.clear()
|
|
|
|
|
|
def _content_hash(content: str, chat_id: str) -> str:
|
|
raw = f"{chat_id}:{content}"
|
|
return hashlib.sha256(raw.encode()).hexdigest()
|
|
|
|
|
|
def normalize_message_id(raw: Any) -> str:
|
|
"""Normalize various message_id formats to a consistent string."""
|
|
if raw is None:
|
|
return ""
|
|
return str(raw).strip()
|
|
|
|
|
|
def should_process_event(
|
|
event: dict[str, Any],
|
|
dedup: MessageDeduplicator,
|
|
) -> bool:
|
|
"""Convenience check: returns True if the event should be processed (not a duplicate)."""
|
|
message_id = normalize_message_id(event.get("message_id"))
|
|
if not message_id:
|
|
return True
|
|
if dedup.is_duplicate(message_id):
|
|
logger.debug(f"[SynologyChat] Duplicate event skipped: msg_id={message_id}")
|
|
return False
|
|
dedup.mark_processed(message_id)
|
|
return True
|