40 lines
1.2 KiB
Python
40 lines
1.2 KiB
Python
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import logging
|
||
|
|
import time
|
||
|
|
|
||
|
|
logger = logging.getLogger(__name__)
|
||
|
|
|
||
|
|
|
||
|
|
class MessageDeduplicator:
|
||
|
|
def __init__(self, ttl_seconds: int = 300, max_entries: int = 10000):
|
||
|
|
self._seen: dict[str, float] = {}
|
||
|
|
self._ttl = ttl_seconds
|
||
|
|
self._max_entries = max_entries
|
||
|
|
|
||
|
|
def is_duplicate(self, msg_id: str) -> bool:
|
||
|
|
self._cleanup()
|
||
|
|
if msg_id in self._seen:
|
||
|
|
return True
|
||
|
|
self._seen[msg_id] = time.time()
|
||
|
|
return False
|
||
|
|
|
||
|
|
def mark_seen(self, msg_id: str) -> None:
|
||
|
|
self._seen[msg_id] = time.time()
|
||
|
|
if len(self._seen) > self._max_entries:
|
||
|
|
self._cleanup(aggressive=True)
|
||
|
|
|
||
|
|
def clear(self) -> None:
|
||
|
|
self._seen.clear()
|
||
|
|
|
||
|
|
def _cleanup(self, aggressive: bool = False) -> None:
|
||
|
|
now = time.time()
|
||
|
|
expired = [k for k, v in self._seen.items() if now - v > self._ttl]
|
||
|
|
for k in expired:
|
||
|
|
del self._seen[k]
|
||
|
|
|
||
|
|
if aggressive and len(self._seen) > self._max_entries:
|
||
|
|
sorted_entries = sorted(self._seen.items(), key=lambda x: x[1])
|
||
|
|
to_remove = len(self._seen) - self._max_entries // 2
|
||
|
|
for k, _ in sorted_entries[:to_remove]:
|
||
|
|
del self._seen[k]
|