import json import logging import os import time from collections import OrderedDict from pathlib import Path logger = logging.getLogger(__name__) class FeishuDeduplicator: def __init__(self, max_size: int = 10000, ttl_seconds: int = 25560, storage_dir: str | None = None): self._cache: OrderedDict[str, float] = OrderedDict() self._max_size = max_size self._ttl = ttl_seconds self._storage_dir = storage_dir self._flush_pending: set[str] = set() def is_duplicate(self, msg_id: str) -> bool: if not msg_id: return False now = time.monotonic() self._evict_expired(now) if msg_id in self._cache: return True self._cache[msg_id] = now while len(self._cache) > self._max_size: self._cache.popitem(last=False) return False def record(self, msg_id: str) -> None: if not msg_id: return now = time.monotonic() self._evict_expired(now) self._cache[msg_id] = now while len(self._cache) > self._max_size: self._cache.popitem(last=False) def finalize(self, msg_id: str) -> None: if not msg_id: return self._flush_pending.add(msg_id) self._persist() def _persist(self) -> None: if not self._storage_dir: return try: store_path = Path(self._storage_dir) / "feishu_dedup.json" store_path.parent.mkdir(parents=True, exist_ok=True) ids = list(self._flush_pending) store_path.write_text(json.dumps(ids, ensure_ascii=False), encoding="utf-8") except OSError: logger.warning("Failed to persist dedup state", exc_info=True) def _evict_expired(self, now: float): expired = [k for k, v in self._cache.items() if now - v > self._ttl] for k in expired: del self._cache[k] def warmup(self, stored_ids: set[str] | None = None) -> None: now = time.monotonic() if stored_ids: for msg_id in stored_ids: self._cache[msg_id] = now self._flush_pending.add(msg_id) elif self._storage_dir: store_path = Path(self._storage_dir) / "feishu_dedup.json" if store_path.exists(): try: content = store_path.read_text(encoding="utf-8") ids = json.loads(content) for msg_id in ids: if now - self._ttl < now: self._cache[msg_id] = now self._flush_pending.add(msg_id) # 清理超过 TTL 的持久化条目 self._persist() except (OSError, json.JSONDecodeError): logger.warning("Failed to load dedup state", exc_info=True) def reset(self): self._cache.clear() self._flush_pending.clear() if self._storage_dir: store_path = Path(self._storage_dir) / "feishu_dedup.json" try: if store_path.exists(): os.remove(store_path) except OSError: pass _deduplicators: dict[str, FeishuDeduplicator] = {} def get_deduplicator(account_id: str) -> FeishuDeduplicator: if account_id not in _deduplicators: _deduplicators[account_id] = FeishuDeduplicator() return _deduplicators[account_id]