"""Message deduplication for Synology Chat inbound events. Prevents duplicate message processing using an LRU-style in-memory cache keyed by message_id. Also deduplicates by content hash within short time windows for additional safety in polling mode. """ from __future__ import annotations import hashlib import time from collections import OrderedDict from typing import Any from yuxi.utils.logging_config import logger _DEFAULT_MAX_ENTRIES = 1000 _DEFAULT_TTL_SECONDS = 300 _DEFAULT_CONTENT_TTL_SECONDS = 60 class MessageDeduplicator: """LRU-based message deduplication cache. Tracks processed message_ids and content hashes, expiring entries after configurable TTL. Thread-compatible (not thread-safe). """ def __init__( self, max_entries: int = _DEFAULT_MAX_ENTRIES, ttl_seconds: int = _DEFAULT_TTL_SECONDS, content_ttl_seconds: int = _DEFAULT_CONTENT_TTL_SECONDS, ): self._max_entries = max_entries self._ttl = ttl_seconds self._content_ttl = content_ttl_seconds self._message_ids: OrderedDict[str, float] = OrderedDict() self._content_hashes: OrderedDict[str, float] = OrderedDict() def is_duplicate(self, message_id: str) -> bool: """Check if a message_id has already been processed.""" if not message_id: return False self._expire_message_ids() return message_id in self._message_ids def mark_processed(self, message_id: str) -> None: """Record a message_id as processed.""" if not message_id: return self._expire_message_ids() if message_id in self._message_ids: self._message_ids.move_to_end(message_id) else: if len(self._message_ids) >= self._max_entries: self._message_ids.popitem(last=False) self._message_ids[message_id] = time.time() def is_content_duplicate(self, content: str, chat_id: str = "") -> bool: """Check if the same content from the same chat was recently processed.""" if not content: return False hash_key = _content_hash(content, chat_id) self._expire_content_hashes() return hash_key in self._content_hashes def mark_content_processed(self, content: str, chat_id: str = "") -> None: """Record content as processed for a chat.""" if not content: return hash_key = _content_hash(content, chat_id) self._expire_content_hashes() if len(self._content_hashes) >= self._max_entries: self._content_hashes.popitem(last=False) self._content_hashes[hash_key] = time.time() def check_and_mark(self, message_id: str) -> bool: """Check duplicate and mark if new. Returns True if duplicate.""" if self.is_duplicate(message_id): return True self.mark_processed(message_id) return False def _expire_message_ids(self) -> None: now = time.time() expired = [k for k, ts in self._message_ids.items() if now - ts > self._ttl] for k in expired: del self._message_ids[k] def _expire_content_hashes(self) -> None: now = time.time() expired = [k for k, ts in self._content_hashes.items() if now - ts > self._content_ttl] for k in expired: del self._content_hashes[k] def __len__(self) -> int: return len(self._message_ids) def clear(self) -> None: self._message_ids.clear() self._content_hashes.clear() def _content_hash(content: str, chat_id: str) -> str: raw = f"{chat_id}:{content}" return hashlib.sha256(raw.encode()).hexdigest() def normalize_message_id(raw: Any) -> str: """Normalize various message_id formats to a consistent string.""" if raw is None: return "" return str(raw).strip() def should_process_event( event: dict[str, Any], dedup: MessageDeduplicator, ) -> bool: """Convenience check: returns True if the event should be processed (not a duplicate).""" message_id = normalize_message_id(event.get("message_id")) if not message_id: return True if dedup.is_duplicate(message_id): logger.debug(f"[SynologyChat] Duplicate event skipped: msg_id={message_id}") return False dedup.mark_processed(message_id) return True