30 lines
939 B
Python
30 lines
939 B
Python
import time
|
|
from collections import OrderedDict
|
|
|
|
|
|
class DedupeTracker:
|
|
def __init__(self, max_entries: int = 50_000, ttl_s: int = 3600):
|
|
self.max_entries = max_entries
|
|
self.ttl_s = ttl_s
|
|
self._store: OrderedDict[str, float] = OrderedDict()
|
|
self._last_prune = time.monotonic()
|
|
|
|
def has(self, event_id: str) -> bool:
|
|
self._maybe_prune()
|
|
if event_id in self._store:
|
|
self._store.move_to_end(event_id)
|
|
return True
|
|
self._store[event_id] = time.monotonic()
|
|
if len(self._store) > self.max_entries:
|
|
self._store.popitem(last=False)
|
|
return False
|
|
|
|
def _maybe_prune(self):
|
|
now = time.monotonic()
|
|
if now - self._last_prune < 600:
|
|
return
|
|
self._last_prune = now
|
|
expired = [k for k, ts in self._store.items() if now - ts > self.ttl_s]
|
|
for k in expired:
|
|
del self._store[k]
|