from __future__ import annotations import json import os import time from collections import OrderedDict from typing import Any import hashlib DEFAULT_MAX_ENTRIES = 10000 DEFAULT_TTL_S = 300 PERSIST_BATCH_INTERVAL_S = 5 class FeishuDedupStore: """内存级飞书事件去重,可选磁盘持久化。 通过 event_id / message_id 去重,支持 TTL 过期自动驱逐。 当提供 persist_path 时,去重记录会在进程重启后保留。 """ def __init__( self, max_entries: int = DEFAULT_MAX_ENTRIES, ttl_s: int = DEFAULT_TTL_S, persist_path: str = "", ): self._max_entries = max_entries self._ttl_s = ttl_s self._store: OrderedDict[str, float] = OrderedDict() self._processing: set[str] = set() self._persist_path = persist_path self._last_persist_at = 0.0 self._dirty = False if self._persist_path: self._load_from_disk() @staticmethod def _make_key(raw_event: dict[str, Any]) -> str: event = raw_event.get("event", {}) message = event.get("message", {}) event_id = event.get("event_id", "") or "" message_id = message.get("message_id", "") or "" key_raw = f"{event_id}:{message_id}" if event_id or message_id else str(raw_event) return hashlib.sha256(key_raw.encode()).hexdigest()[:32] def has_processed(self, raw_event: dict[str, Any]) -> bool: self._evict_expired() key = self._make_key(raw_event) if key in self._store: return True if key in self._processing: return True return False def record_processed(self, raw_event: dict[str, Any]) -> None: key = self._make_key(raw_event) self._processing.add(key) def finalize_processing(self, raw_event: dict[str, Any]) -> None: key = self._make_key(raw_event) self._processing.discard(key) self._store[key] = time.monotonic() self._evict_expired() self._trim_to_max() self._dirty = True self._maybe_persist() def _evict_expired(self) -> None: now = time.monotonic() expired = [k for k, ts in self._store.items() if now - ts > self._ttl_s] if expired: self._dirty = True for k in expired: self._store.pop(k, None) def _trim_to_max(self) -> None: while len(self._store) > self._max_entries: self._dirty = True self._store.popitem(last=False) def clear(self) -> None: self._store.clear() self._processing.clear() self._dirty = True self._maybe_persist() def __len__(self) -> int: return len(self._store) + len(self._processing) def _maybe_persist(self) -> None: if not self._persist_path: return now = time.monotonic() if now - self._last_persist_at < PERSIST_BATCH_INTERVAL_S: return self._persist_to_disk() self._last_persist_at = now def _persist_to_disk(self) -> None: if not self._persist_path or not self._dirty: return try: entries = [{"k": k, "t": ts} for k, ts in self._store.items() if time.monotonic() - ts <= self._ttl_s] tmp_path = self._persist_path + ".tmp" with open(tmp_path, "w", encoding="utf-8") as f: json.dump({"ttl_s": self._ttl_s, "entries": entries}, f) os.replace(tmp_path, self._persist_path) self._dirty = False except OSError: pass def _load_from_disk(self) -> None: if not os.path.exists(self._persist_path): return try: with open(self._persist_path, encoding="utf-8") as f: data = json.load(f) except (json.JSONDecodeError, OSError): return stored_ttl = data.get("ttl_s", self._ttl_s) now = time.monotonic() loaded = 0 for entry in data.get("entries", []): key = entry.get("k", "") ts = entry.get("t", 0) if not key: continue age = now - ts if age > stored_ttl: continue self._store[key] = ts - stored_ttl + self._ttl_s loaded += 1 if loaded > 0: self._dirty = False self._last_persist_at = now