from __future__ import annotations import logging import time from collections import OrderedDict logger = logging.getLogger(__name__) class TwitterDeduplicator: def __init__(self, max_size: int = 5000, ttl_seconds: int = 86400): self._cache: OrderedDict[str, float] = OrderedDict() self._max_size = max_size self._ttl = ttl_seconds def is_duplicate(self, msg_id: str) -> bool: if not msg_id: return False now = time.monotonic() self._evict_expired(now) if msg_id in self._cache: return True self._cache[msg_id] = now while len(self._cache) > self._max_size: self._cache.popitem(last=False) return False def mark_seen(self, msg_id: str) -> None: if not msg_id: return now = time.monotonic() self._evict_expired(now) self._cache[msg_id] = now def _evict_expired(self, now: float): expired = [k for k, v in self._cache.items() if now - v > self._ttl] for k in expired: del self._cache[k] def reset(self): self._cache.clear() _deduplicators: dict[str, TwitterDeduplicator] = {} def get_deduplicator(account_id: str) -> TwitterDeduplicator: if account_id not in _deduplicators: _deduplicators[account_id] = TwitterDeduplicator() return _deduplicators[account_id]