552 lines
18 KiB
Python
552 lines
18 KiB
Python
import hashlib
|
||
import json
|
||
import logging
|
||
import re
|
||
import threading
|
||
from dataclasses import dataclass, field
|
||
from datetime import UTC, datetime
|
||
from itertools import count
|
||
from typing import Any
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
_SENSITIVE_KEY_PATTERNS = [
|
||
re.compile(r"token$", re.IGNORECASE),
|
||
re.compile(r"password", re.IGNORECASE),
|
||
re.compile(r"secret", re.IGNORECASE),
|
||
re.compile(r"api[\W_]?key", re.IGNORECASE),
|
||
re.compile(r"encrypt[\W_]?key", re.IGNORECASE),
|
||
re.compile(r"private[\W_]?key", re.IGNORECASE),
|
||
re.compile(r"serviceaccount(?:ref)?$", re.IGNORECASE),
|
||
]
|
||
|
||
_SENSITIVE_KEY_WHITELIST_SUFFIXES = [
|
||
"maxtokens",
|
||
"maxoutputtokens",
|
||
"maxinputtokens",
|
||
"maxcompletiontokens",
|
||
"contexttokens",
|
||
"totaltokens",
|
||
"tokencount",
|
||
"tokenlimit",
|
||
"tokenbudget",
|
||
"passwordfile",
|
||
]
|
||
|
||
REDACTED_SENTINEL = "__FORCEPILOT_REDACTED__"
|
||
|
||
_ENV_VAR_PLACEHOLDER_PATTERN = re.compile(r"^\$\{[^}]*\}$")
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# AccountConfigEntry – 渠道账户配置与运行时状态
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@dataclass
|
||
class AccountConfigEntry:
|
||
channel_type: str
|
||
account_id: str
|
||
enabled: bool
|
||
configured: bool
|
||
config_hash: str
|
||
|
||
name: str | None = None
|
||
connected: bool | None = None
|
||
running: bool | None = None
|
||
linked: bool | None = None
|
||
restart_pending: bool | None = None
|
||
status_state: str | None = None
|
||
health_state: str | None = None
|
||
mode: str | None = None
|
||
dm_policy: str | None = None
|
||
allow_from: list[str] | None = None
|
||
reconnect_attempts: int | None = None
|
||
last_connected_at: float | None = None
|
||
last_message_at: float | None = None
|
||
last_inbound_at: float | None = None
|
||
last_outbound_at: float | None = None
|
||
last_event_at: float | None = None
|
||
last_transport_activity_at: float | None = None
|
||
last_error: str | None = None
|
||
last_start_at: float | None = None
|
||
last_stop_at: float | None = None
|
||
busy: bool | None = None
|
||
active_runs: int | None = None
|
||
last_run_activity_at: float | None = None
|
||
token_status: str | None = None
|
||
bot_token_status: str | None = None
|
||
app_token_status: str | None = None
|
||
signing_secret_status: str | None = None
|
||
user_token_status: str | None = None
|
||
base_url: str | None = None
|
||
port: int | None = None
|
||
allow_unmentioned_groups: bool | None = None
|
||
cli_path: str | None = None
|
||
db_path: str | None = None
|
||
|
||
|
||
_SENSITIVE_ACCOUNT_FIELDS: set[str] = {
|
||
"webhook_url",
|
||
"webhook_path",
|
||
"audience",
|
||
"public_key",
|
||
"channel_access_token",
|
||
"channel_secret",
|
||
"token",
|
||
"bot_token",
|
||
"app_token",
|
||
"signing_secret",
|
||
"user_token",
|
||
}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 安全投影 – 剥离敏感字段
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def project_safe_account_fields(account: dict[str, Any] | Any) -> dict[str, Any]:
|
||
"""从账户对象中提取对外安全的快照字段，剥离敏感信息"""
|
||
if not isinstance(account, dict):
|
||
return {}
|
||
|
||
record: dict[str, Any] = account
|
||
result: dict[str, Any] = {}
|
||
|
||
for key in ("name", "status_state", "health_state", "mode", "dm_policy"):
|
||
val = record.get(key)
|
||
if isinstance(val, str) and val:
|
||
result[key] = val
|
||
|
||
for key in ("linked", "running", "connected", "restart_pending", "busy"):
|
||
val = record.get(key)
|
||
if isinstance(val, bool):
|
||
result[key] = val
|
||
|
||
for key in (
|
||
"reconnect_attempts",
|
||
"active_runs",
|
||
"last_connected_at",
|
||
"last_message_at",
|
||
"last_inbound_at",
|
||
"last_outbound_at",
|
||
"last_event_at",
|
||
"last_transport_activity_at",
|
||
"last_start_at",
|
||
"last_stop_at",
|
||
"last_run_activity_at",
|
||
):
|
||
val = record.get(key)
|
||
if isinstance(val, (int, float)):
|
||
result[key] = val
|
||
elif val is None and key in record:
|
||
result[key] = None
|
||
|
||
for key in ("token_status", "bot_token_status", "app_token_status", "signing_secret_status", "user_token_status"):
|
||
val = record.get(key)
|
||
if val in ("available", "configured_unavailable", "missing"):
|
||
result[key] = val
|
||
|
||
base_url = record.get("base_url")
|
||
if isinstance(base_url, str) and base_url:
|
||
result["base_url"] = _strip_url_userinfo(base_url)
|
||
|
||
allow_from = record.get("allow_from")
|
||
if isinstance(allow_from, list):
|
||
normalized = [str(e).strip() for e in allow_from if isinstance(e, (str, int, float)) and str(e).strip()]
|
||
if normalized:
|
||
result["allow_from"] = normalized
|
||
|
||
cli_path = record.get("cli_path")
|
||
if isinstance(cli_path, str) and cli_path:
|
||
result["cli_path"] = cli_path
|
||
|
||
db_path = record.get("db_path")
|
||
if isinstance(db_path, str) and db_path:
|
||
result["db_path"] = db_path
|
||
|
||
port = record.get("port")
|
||
if isinstance(port, (int, float)):
|
||
result["port"] = int(port)
|
||
elif port is None and "port" in record:
|
||
result["port"] = None
|
||
|
||
allow_unmentioned = record.get("allow_unmentioned_groups")
|
||
if isinstance(allow_unmentioned, bool):
|
||
result["allow_unmentioned_groups"] = allow_unmentioned
|
||
|
||
account_id = record.get("account_id")
|
||
if isinstance(account_id, str):
|
||
result["account_id"] = account_id
|
||
|
||
enabled = record.get("enabled")
|
||
if isinstance(enabled, bool):
|
||
result["enabled"] = enabled
|
||
|
||
configured = record.get("configured")
|
||
if isinstance(configured, bool):
|
||
result["configured"] = configured
|
||
|
||
return result
|
||
|
||
|
||
def _strip_url_userinfo(url: str) -> str:
|
||
"""剥离 URL 中的用户信息部分 (https://user:pass@host -> https://host)"""
|
||
import re as _re
|
||
|
||
return _re.sub(r"://[^@/]+@", "://", url)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 敏感路径检测
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def _is_whitelisted_sensitive_path(path: str) -> bool:
|
||
lower = path.lower()
|
||
return any(lower.endswith(suffix) for suffix in _SENSITIVE_KEY_WHITELIST_SUFFIXES)
|
||
|
||
|
||
def is_sensitive_config_path(path: str) -> bool:
|
||
"""检测配置路径是否指向敏感字段 (token/password/secret/api_key 等)"""
|
||
if _is_whitelisted_sensitive_path(path):
|
||
return False
|
||
return any(pattern.search(path) for pattern in _SENSITIVE_KEY_PATTERNS)
|
||
|
||
|
||
def _is_env_var_placeholder(value: str) -> bool:
|
||
return bool(_ENV_VAR_PLACEHOLDER_PATTERN.match(value.strip()))
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 脱敏 & 还原 (Redact / Restore)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def _is_object_record(value: Any) -> bool:
|
||
return isinstance(value, dict)
|
||
|
||
|
||
def _collect_sensitive_strings(value: Any, values: list[str]) -> None:
|
||
if isinstance(value, str):
|
||
if not _is_env_var_placeholder(value):
|
||
values.append(value)
|
||
return
|
||
if isinstance(value, list):
|
||
for item in value:
|
||
_collect_sensitive_strings(item, values)
|
||
return
|
||
if _is_object_record(value):
|
||
for item in value.values():
|
||
_collect_sensitive_strings(item, values)
|
||
|
||
|
||
def redact_config_object(obj: Any, ui_hints: dict[str, Any] | None = None) -> Any:
|
||
"""深度遍历对象，将敏感路径上的字符串值替换为 REDACTED_SENTINEL"""
|
||
if ui_hints:
|
||
lookup = _build_redaction_lookup(ui_hints)
|
||
if "" in lookup:
|
||
return _redact_with_lookup(obj, lookup, "")
|
||
return _redact_guessing(obj, "", ui_hints)
|
||
return _redact_guessing(obj, "")
|
||
|
||
|
||
def restore_redacted_values(incoming: Any, original: Any, ui_hints: dict[str, Any] | None = None) -> Any:
|
||
"""深度遍历 incoming，将 REDACTED_SENTINEL 替换回 original 中的真实值"""
|
||
if incoming is None:
|
||
return incoming
|
||
if not isinstance(incoming, dict):
|
||
return incoming
|
||
|
||
if ui_hints:
|
||
lookup = _build_redaction_lookup(ui_hints)
|
||
if "" in lookup:
|
||
return _restore_with_lookup(incoming, original, lookup, "", ui_hints)
|
||
return _restore_guessing(incoming, original, "", ui_hints)
|
||
return _restore_guessing(incoming, original, "")
|
||
|
||
|
||
def _build_redaction_lookup(hints: dict[str, Any]) -> set[str]:
|
||
result: set[str] = set()
|
||
for path, hint in hints.items():
|
||
if not hint.get("sensitive"):
|
||
continue
|
||
parts = path.split(".")
|
||
joined = parts[0] if parts else ""
|
||
result.add(joined)
|
||
if joined.endswith("[]"):
|
||
result.add(joined[:-2])
|
||
for part in parts[1:]:
|
||
if part.endswith("[]"):
|
||
result.add(f"{joined}.{part[:-2]}")
|
||
joined = f"{joined}.{part}"
|
||
result.add(joined)
|
||
if result:
|
||
result.add("")
|
||
return result
|
||
|
||
|
||
def _redact_with_lookup(obj: Any, lookup: set[str], prefix: str) -> Any:
|
||
if obj is None:
|
||
return obj
|
||
if isinstance(obj, list):
|
||
path = f"{prefix}[]"
|
||
if path not in lookup:
|
||
return [_redact_guessing(item, prefix) for item in obj]
|
||
return [
|
||
REDACTED_SENTINEL
|
||
if isinstance(item, str) and not _is_env_var_placeholder(item)
|
||
else _redact_with_lookup(item, lookup, path)
|
||
for item in obj
|
||
]
|
||
if _is_object_record(obj):
|
||
result: dict[str, Any] = {}
|
||
for key, value in obj.items():
|
||
path = f"{prefix}.{key}" if prefix else key
|
||
wildcard_path = f"{prefix}.*" if prefix else "*"
|
||
matched = False
|
||
for candidate in (path, wildcard_path):
|
||
if candidate in lookup:
|
||
matched = True
|
||
if isinstance(value, str) and not _is_env_var_placeholder(value):
|
||
result[key] = REDACTED_SENTINEL
|
||
elif isinstance(value, dict):
|
||
result[key] = _redact_with_lookup(value, lookup, candidate)
|
||
elif isinstance(value, list):
|
||
result[key] = _redact_with_lookup(value, lookup, candidate)
|
||
else:
|
||
result[key] = value
|
||
break
|
||
if not matched:
|
||
result[key] = _redact_guessing(value, path)
|
||
return result
|
||
return obj
|
||
|
||
|
||
def _redact_guessing(obj: Any, prefix: str, hints: dict[str, Any] | None = None) -> Any:
|
||
if obj is None:
|
||
return obj
|
||
if isinstance(obj, list):
|
||
return [_redact_guessing(item, f"{prefix}[]", hints) for item in obj]
|
||
if _is_object_record(obj):
|
||
result: dict[str, Any] = {}
|
||
for key, value in obj.items():
|
||
dot_path = f"{prefix}.{key}" if prefix else key
|
||
if isinstance(value, str) and not _is_env_var_placeholder(value) and is_sensitive_config_path(dot_path):
|
||
result[key] = REDACTED_SENTINEL
|
||
elif isinstance(value, dict):
|
||
result[key] = _redact_guessing(value, dot_path, hints)
|
||
elif isinstance(value, list):
|
||
result[key] = _redact_guessing(value, dot_path, hints)
|
||
else:
|
||
result[key] = value
|
||
return result
|
||
return obj
|
||
|
||
|
||
def _restore_with_lookup(incoming: Any, original: Any, lookup: set[str], prefix: str, hints: dict[str, Any]) -> Any:
|
||
if incoming is None or not isinstance(incoming, (dict, list)):
|
||
return incoming
|
||
|
||
if isinstance(incoming, list):
|
||
path = f"{prefix}[]"
|
||
if path not in lookup:
|
||
return [_restore_guessing(item, original, prefix, hints) for item in incoming]
|
||
orig_list = original if isinstance(original, list) else []
|
||
return [
|
||
orig_list[i]
|
||
if item == REDACTED_SENTINEL and i < len(orig_list)
|
||
else _restore_with_lookup(item, orig_list[i] if i < len(orig_list) else None, lookup, path, hints)
|
||
for i, item in enumerate(incoming)
|
||
]
|
||
|
||
if _is_object_record(incoming):
|
||
orig = original if _is_object_record(original) else {}
|
||
result: dict[str, Any] = {}
|
||
for key, value in incoming.items():
|
||
path = f"{prefix}.{key}" if prefix else key
|
||
wildcard_path = f"{prefix}.*" if prefix else "*"
|
||
matched = False
|
||
for candidate in (path, wildcard_path):
|
||
if candidate in lookup:
|
||
matched = True
|
||
if value == REDACTED_SENTINEL:
|
||
result[key] = orig.get(key, value)
|
||
elif isinstance(value, (dict, list)):
|
||
result[key] = _restore_with_lookup(value, orig.get(key), lookup, candidate, hints)
|
||
else:
|
||
result[key] = value
|
||
break
|
||
if not matched:
|
||
result[key] = _restore_guessing(value, orig, prefix, hints)
|
||
return result
|
||
return incoming
|
||
|
||
|
||
def _restore_guessing(incoming: Any, original: Any, prefix: str, hints: dict[str, Any] | None = None) -> Any:
|
||
if incoming is None or not isinstance(incoming, (dict, list)):
|
||
return incoming
|
||
|
||
if isinstance(incoming, list):
|
||
orig_list = original if isinstance(original, list) else []
|
||
path = f"{prefix}[]"
|
||
return [
|
||
orig_list[i]
|
||
if item == REDACTED_SENTINEL and is_sensitive_config_path(path) and i < len(orig_list)
|
||
else _restore_guessing(item, orig_list[i] if i < len(orig_list) else None, path, hints)
|
||
for i, item in enumerate(incoming)
|
||
]
|
||
|
||
if _is_object_record(incoming):
|
||
orig = original if _is_object_record(original) else {}
|
||
result: dict[str, Any] = {}
|
||
for key, value in incoming.items():
|
||
dot_path = f"{prefix}.{key}" if prefix else key
|
||
if value == REDACTED_SENTINEL and is_sensitive_config_path(dot_path):
|
||
result[key] = orig.get(key, value)
|
||
elif isinstance(value, (dict, list)):
|
||
result[key] = _restore_guessing(value, orig.get(key), dot_path, hints)
|
||
else:
|
||
result[key] = value
|
||
return result
|
||
return incoming
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# ConfigFileSnapshot – 配置文件全量快照
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@dataclass
|
||
class ConfigFileSnapshot:
|
||
path: str
|
||
exists: bool
|
||
raw: str | None
|
||
parsed: Any
|
||
source_config: dict[str, Any] = field(default_factory=dict)
|
||
resolved: dict[str, Any] = field(default_factory=dict)
|
||
runtime_config: dict[str, Any] = field(default_factory=dict)
|
||
valid: bool = False
|
||
hash: str = ""
|
||
issues: list[dict[str, Any]] = field(default_factory=list)
|
||
warnings: list[dict[str, Any]] = field(default_factory=list)
|
||
|
||
def __post_init__(self):
|
||
if not self.hash and self.runtime_config:
|
||
self.hash = hash_config_value(self.runtime_config)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# ConfigSnapshot – 渠道配置运行时快照
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@dataclass
|
||
class ConfigSnapshot:
|
||
version: str
|
||
accounts: list[AccountConfigEntry]
|
||
route_bindings_hash: str
|
||
created_at: datetime = field(default_factory=lambda: datetime.now(UTC))
|
||
revision: int = 0
|
||
source_fingerprint: str = ""
|
||
runtime_fingerprint: str = ""
|
||
|
||
@property
|
||
def stable_hash(self) -> str:
|
||
if self.source_fingerprint:
|
||
return self.source_fingerprint
|
||
return _compute_snapshot_stable_hash(self.accounts, self.route_bindings_hash)
|
||
|
||
|
||
def _compute_snapshot_stable_hash(accounts: list[AccountConfigEntry], routes_hash: str) -> str:
|
||
sorted_accounts = sorted(
|
||
[
|
||
{
|
||
"t": a.channel_type,
|
||
"id": a.account_id,
|
||
"e": a.enabled,
|
||
"c": a.configured,
|
||
"h": a.config_hash,
|
||
}
|
||
for a in accounts
|
||
],
|
||
key=lambda x: (x["t"], x["id"]),
|
||
)
|
||
payload = {"accounts": sorted_accounts, "routes_hash": routes_hash}
|
||
raw = json.dumps(payload, sort_keys=True, ensure_ascii=False)
|
||
return hashlib.sha256(raw.encode()).hexdigest()[:16]
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 通用哈希 & 快照比较
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def _stable_config_stringify(value: Any) -> str:
|
||
if value is None:
|
||
return "null"
|
||
if isinstance(value, bool):
|
||
return "true" if value else "false"
|
||
if isinstance(value, (int, float)):
|
||
return json.dumps(value)
|
||
if isinstance(value, str):
|
||
return json.dumps(value, ensure_ascii=False)
|
||
if isinstance(value, (list, tuple)):
|
||
return f"[{','.join(_stable_config_stringify(v) for v in value)}]"
|
||
if isinstance(value, dict):
|
||
keys = sorted(value.keys())
|
||
return (
|
||
"{"
|
||
+ ",".join(f"{json.dumps(k, ensure_ascii=False)}:{_stable_config_stringify(value[k])}" for k in keys)
|
||
+ "}"
|
||
)
|
||
return json.dumps(value, default=str, ensure_ascii=False)
|
||
|
||
|
||
def hash_config_value(value: dict[str, Any]) -> str:
|
||
return hashlib.sha256(_stable_config_stringify(value).encode()).hexdigest()[:16]
|
||
|
||
|
||
def compute_source_fingerprint(config: dict) -> str:
|
||
raw = json.dumps(config, sort_keys=True, default=str, ensure_ascii=False)
|
||
return hashlib.sha256(raw.encode()).hexdigest()[:16]
|
||
|
||
|
||
def config_snapshots_match(left: dict[str, Any], right: dict[str, Any]) -> bool:
|
||
if left is right:
|
||
return True
|
||
try:
|
||
return _stable_config_stringify(left) == _stable_config_stringify(right)
|
||
except Exception as e:
|
||
logger.warning("Config snapshot comparison failed: %s", e)
|
||
return False
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# RevisionCounter – 全局递增计数器
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
class RevisionCounter:
|
||
_revision = count(1)
|
||
_current: int = 0
|
||
_lock = threading.Lock()
|
||
|
||
@classmethod
|
||
def next(cls) -> int:
|
||
with cls._lock:
|
||
cls._current = next(cls._revision)
|
||
return cls._current
|
||
|
||
@classmethod
|
||
def current(cls) -> int:
|
||
with cls._lock:
|
||
return cls._current
|
||
|
||
@classmethod
|
||
def reset(cls) -> None:
|
||
with cls._lock:
|
||
cls._revision = count(1)
|
||
cls._current = 0
|