from __future__ import annotations import hashlib import logging import time from collections import OrderedDict logger = logging.getLogger(__name__) class GitLabDeduplicator: def __init__(self, max_size: int = 10000, ttl_seconds: float = 600): self._seen: OrderedDict[str, float] = OrderedDict() self._max_size = max_size self._ttl = ttl_seconds def is_duplicate(self, payload: dict, event_uuid: str | None) -> bool: now = time.monotonic() self._evict_expired(now) if event_uuid: key = f"uuid:{event_uuid}" else: key = f"fp:{self._create_fingerprint(payload)}" if key in self._seen: logger.debug("GitLab dedupe: duplicate %s", key[:60]) self._seen.move_to_end(key) return True self._seen[key] = now self._seen.move_to_end(key) while len(self._seen) > self._max_size: self._seen.popitem(last=False) return False def _evict_expired(self, now: float) -> None: cutoff = now - self._ttl expired = [k for k, ts in self._seen.items() if ts < cutoff] for k in expired: del self._seen[k] @staticmethod def _create_fingerprint(payload: dict) -> str: key_parts = [ str(payload.get("object_kind")), str(payload.get("event_type")), str(payload.get("object_attributes", {}).get("id")), str(payload.get("object_attributes", {}).get("updated_at")), str(payload.get("user", {}).get("id")), ] raw = "|".join(key_parts) return hashlib.sha256(raw.encode()).hexdigest()[:16] def reset(self): self._seen.clear()