106 lines
3.4 KiB
Python
106 lines
3.4 KiB
Python
import json
|
|
import logging
|
|
import os
|
|
import time
|
|
from collections import OrderedDict
|
|
from pathlib import Path
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class FeishuDeduplicator:
|
|
def __init__(self, max_size: int = 10000, ttl_seconds: int = 25560, storage_dir: str | None = None):
|
|
self._cache: OrderedDict[str, float] = OrderedDict()
|
|
self._max_size = max_size
|
|
self._ttl = ttl_seconds
|
|
self._storage_dir = storage_dir
|
|
self._flush_pending: set[str] = set()
|
|
|
|
def is_duplicate(self, msg_id: str) -> bool:
|
|
if not msg_id:
|
|
return False
|
|
|
|
now = time.monotonic()
|
|
self._evict_expired(now)
|
|
|
|
if msg_id in self._cache:
|
|
return True
|
|
|
|
self._cache[msg_id] = now
|
|
while len(self._cache) > self._max_size:
|
|
self._cache.popitem(last=False)
|
|
|
|
return False
|
|
|
|
def record(self, msg_id: str) -> None:
|
|
if not msg_id:
|
|
return
|
|
now = time.monotonic()
|
|
self._evict_expired(now)
|
|
self._cache[msg_id] = now
|
|
while len(self._cache) > self._max_size:
|
|
self._cache.popitem(last=False)
|
|
|
|
def finalize(self, msg_id: str) -> None:
|
|
if not msg_id:
|
|
return
|
|
self._flush_pending.add(msg_id)
|
|
self._persist()
|
|
|
|
def _persist(self) -> None:
|
|
if not self._storage_dir:
|
|
return
|
|
try:
|
|
store_path = Path(self._storage_dir) / "feishu_dedup.json"
|
|
store_path.parent.mkdir(parents=True, exist_ok=True)
|
|
ids = list(self._flush_pending)
|
|
store_path.write_text(json.dumps(ids, ensure_ascii=False), encoding="utf-8")
|
|
except OSError:
|
|
logger.warning("Failed to persist dedup state", exc_info=True)
|
|
|
|
def _evict_expired(self, now: float):
|
|
expired = [k for k, v in self._cache.items() if now - v > self._ttl]
|
|
for k in expired:
|
|
del self._cache[k]
|
|
|
|
def warmup(self, stored_ids: set[str] | None = None) -> None:
|
|
now = time.monotonic()
|
|
if stored_ids:
|
|
for msg_id in stored_ids:
|
|
self._cache[msg_id] = now
|
|
self._flush_pending.add(msg_id)
|
|
elif self._storage_dir:
|
|
store_path = Path(self._storage_dir) / "feishu_dedup.json"
|
|
if store_path.exists():
|
|
try:
|
|
content = store_path.read_text(encoding="utf-8")
|
|
ids = json.loads(content)
|
|
for msg_id in ids:
|
|
if now - self._ttl < now:
|
|
self._cache[msg_id] = now
|
|
self._flush_pending.add(msg_id)
|
|
# 清理超过 TTL 的持久化条目
|
|
self._persist()
|
|
except (OSError, json.JSONDecodeError):
|
|
logger.warning("Failed to load dedup state", exc_info=True)
|
|
|
|
def reset(self):
|
|
self._cache.clear()
|
|
self._flush_pending.clear()
|
|
if self._storage_dir:
|
|
store_path = Path(self._storage_dir) / "feishu_dedup.json"
|
|
try:
|
|
if store_path.exists():
|
|
os.remove(store_path)
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
_deduplicators: dict[str, FeishuDeduplicator] = {}
|
|
|
|
|
|
def get_deduplicator(account_id: str) -> FeishuDeduplicator:
|
|
if account_id not in _deduplicators:
|
|
_deduplicators[account_id] = FeishuDeduplicator()
|
|
return _deduplicators[account_id]
|