Kris
1f78c44b03
这是一个批量整理提交，包含以下主要改动： 1. 删除多处冗余的空行和未使用的导入 2. 修复文件末尾缺少换行符的问题 3. 调整部分模块的导入顺序与代码排版 4. 修复部分配置默认值与策略逻辑 5. 新增多个功能模块与辅助工具 6. 完善异常处理与日志记录 7. 修复速率限制、消息缓存、权限校验等逻辑bug 8. 废弃部分旧有API与配置项并添加警告提示
141 lines
4.3 KiB
Python
141 lines
4.3 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import time
|
|
from collections import OrderedDict
|
|
from typing import Any
|
|
|
|
import hashlib
|
|
|
|
DEFAULT_MAX_ENTRIES = 10000
|
|
DEFAULT_TTL_S = 300
|
|
PERSIST_BATCH_INTERVAL_S = 5
|
|
|
|
|
|
class FeishuDedupStore:
|
|
|
|
def __init__(
|
|
self,
|
|
max_entries: int = DEFAULT_MAX_ENTRIES,
|
|
ttl_s: int = DEFAULT_TTL_S,
|
|
persist_path: str = "",
|
|
):
|
|
self._max_entries = max_entries
|
|
self._ttl_s = ttl_s
|
|
self._store: OrderedDict[str, float] = OrderedDict()
|
|
self._processing: set[str] = set()
|
|
self._persist_path = persist_path
|
|
self._last_persist_at = 0.0
|
|
self._dirty = False
|
|
|
|
if self._persist_path:
|
|
self._load_from_disk()
|
|
|
|
@staticmethod
|
|
def _make_key(raw_event: dict[str, Any]) -> str:
|
|
event = raw_event.get("event", {})
|
|
message = event.get("message", {})
|
|
event_id = event.get("event_id", "") or ""
|
|
message_id = message.get("message_id", "") or ""
|
|
|
|
if event_id or message_id:
|
|
key_raw = f"{event_id}:{message_id}"
|
|
else:
|
|
raw_copy = {k: v for k, v in raw_event.items() if k != "timestamp"}
|
|
key_raw = json.dumps(raw_copy, sort_keys=True, ensure_ascii=False)
|
|
|
|
return hashlib.sha256(key_raw.encode()).hexdigest()[:32]
|
|
|
|
def has_processed(self, raw_event: dict[str, Any]) -> bool:
|
|
self._evict_expired()
|
|
key = self._make_key(raw_event)
|
|
if key in self._store:
|
|
return True
|
|
if key in self._processing:
|
|
return True
|
|
return False
|
|
|
|
def record_processed(self, raw_event: dict[str, Any]) -> None:
|
|
key = self._make_key(raw_event)
|
|
self._processing.add(key)
|
|
|
|
def finalize_processing(self, raw_event: dict[str, Any]) -> None:
|
|
key = self._make_key(raw_event)
|
|
self._processing.discard(key)
|
|
self._store[key] = time.monotonic()
|
|
self._evict_expired()
|
|
self._trim_to_max()
|
|
self._dirty = True
|
|
self._maybe_persist()
|
|
|
|
def _evict_expired(self) -> None:
|
|
now = time.monotonic()
|
|
expired = [k for k, ts in self._store.items() if now - ts > self._ttl_s]
|
|
if expired:
|
|
self._dirty = True
|
|
for k in expired:
|
|
self._store.pop(k, None)
|
|
|
|
def _trim_to_max(self) -> None:
|
|
while len(self._store) > self._max_entries:
|
|
self._dirty = True
|
|
self._store.popitem(last=False)
|
|
|
|
def clear(self) -> None:
|
|
self._store.clear()
|
|
self._processing.clear()
|
|
self._dirty = True
|
|
self._maybe_persist()
|
|
|
|
def __len__(self) -> int:
|
|
return len(self._store) + len(self._processing)
|
|
|
|
def _maybe_persist(self) -> None:
|
|
if not self._persist_path:
|
|
return
|
|
now = time.monotonic()
|
|
if now - self._last_persist_at < PERSIST_BATCH_INTERVAL_S:
|
|
return
|
|
self._persist_to_disk()
|
|
self._last_persist_at = now
|
|
|
|
def _persist_to_disk(self) -> None:
|
|
if not self._persist_path or not self._dirty:
|
|
return
|
|
try:
|
|
entries = [{"k": k, "t": ts} for k, ts in self._store.items() if time.monotonic() - ts <= self._ttl_s]
|
|
tmp_path = self._persist_path + ".tmp"
|
|
with open(tmp_path, "w", encoding="utf-8") as f:
|
|
json.dump({"ttl_s": self._ttl_s, "entries": entries}, f)
|
|
os.replace(tmp_path, self._persist_path)
|
|
self._dirty = False
|
|
except OSError:
|
|
pass
|
|
|
|
def _load_from_disk(self) -> None:
|
|
if not os.path.exists(self._persist_path):
|
|
return
|
|
try:
|
|
with open(self._persist_path, encoding="utf-8") as f:
|
|
data = json.load(f)
|
|
except (json.JSONDecodeError, OSError):
|
|
return
|
|
|
|
stored_ttl = data.get("ttl_s", self._ttl_s)
|
|
now = time.monotonic()
|
|
loaded = 0
|
|
for entry in data.get("entries", []):
|
|
key = entry.get("k", "")
|
|
ts = entry.get("t", 0)
|
|
if not key:
|
|
continue
|
|
age = now - ts
|
|
if age > stored_ttl:
|
|
continue
|
|
self._store[key] = ts - stored_ttl + self._ttl_s
|
|
loaded += 1
|
|
|
|
if loaded > 0:
|
|
self._dirty = False
|
|
self._last_persist_at = now |