refactor(utils): 统一敏感字段脱敏逻辑,新增trace_id上下文工具

1.  新增请求级trace_id上下文共享模块,通过ContextVar存储trace_id供跨层使用
2.  重构gRPC和SOAP执行器的脱敏逻辑,统一使用crypto模块的敏感头集合
3.  扩展crypto模块的敏感字段匹配规则,新增全量遮蔽的强敏感字段类型
4.  新增mask_full和mask_partial两种脱敏模式,完善脱敏工具链
5.  定义SENSITIVE_HTTP_HEADERS作为HTTP敏感头的单一事实源
This commit is contained in:
Kris 2026-07-01 19:48:25 +08:00
parent 0993a721b6
commit 2c190b2a5c
4 changed files with 197 additions and 30 deletions

View File

@ -42,14 +42,12 @@ from yuxi.external_systems.framework.auth_plugins.credentials import (
)
from yuxi.external_systems.framework.auth_plugins.registry import resolve_credential
from yuxi.utils import logger
from yuxi.utils.crypto import SENSITIVE_HTTP_HEADERS
from yuxi.utils.net_security import SSRFValidator
if TYPE_CHECKING:
from yuxi.external_systems.core.contracts import ExecutableTool, RuntimeContext
# 敏感 metadata 键名(大小写不敏感)
_SENSITIVE_METADATA_KEYS = {"authorization", "cookie", "token", "x-api-key", "api-key"}
def _load_grpc():
"""懒加载 grpc缺失时抛出 DomainValidationError。"""
@ -163,10 +161,13 @@ def _endpoint_to_url(endpoint: str) -> str:
def _sanitize_metadata(metadata: dict[str, str]) -> dict[str, str]:
"""对敏感 metadata 值进行脱敏。"""
"""对敏感 metadata 值进行脱敏。
敏感键名集合统一引用 ``yuxi.utils.crypto.SENSITIVE_HTTP_HEADERS``单一事实源
"""
result: dict[str, str] = {}
for key, value in metadata.items():
if key.lower() in _SENSITIVE_METADATA_KEYS:
if key.lower() in SENSITIVE_HTTP_HEADERS:
result[key] = "***"
else:
result[key] = value

View File

@ -39,6 +39,7 @@ from yuxi.external_systems.framework.auth_plugins.credentials import (
)
from yuxi.external_systems.framework.auth_plugins.registry import resolve_credential
from yuxi.utils import logger
from yuxi.utils.crypto import SENSITIVE_HTTP_HEADERS
from yuxi.utils.net_security import SSRFValidator
if TYPE_CHECKING:
@ -46,9 +47,6 @@ if TYPE_CHECKING:
from zeep.transports import AsyncTransport
from zeep.wsse.username import UsernameToken
# 敏感 header 键名(大小写不敏感)
_SENSITIVE_HEADER_KEYS = {"authorization", "cookie", "token", "x-api-key", "api-key", "password"}
def _load_zeep():
"""懒加载 zeep 及其子模块,缺失时抛出 DomainValidationError。"""
@ -63,10 +61,13 @@ def _load_zeep():
def _sanitize_headers(headers: dict[str, str]) -> dict[str, str]:
"""对敏感 header 值进行脱敏。"""
"""对敏感 header 值进行脱敏。
敏感键名集合统一引用 ``yuxi.utils.crypto.SENSITIVE_HTTP_HEADERS``单一事实源
"""
result: dict[str, str] = {}
for key, value in headers.items():
if key.lower() in _SENSITIVE_HEADER_KEYS:
if key.lower() in SENSITIVE_HTTP_HEADERS:
result[key] = "***"
else:
result[key] = value

View File

@ -59,8 +59,12 @@ class CryptoHelper:
"id_token",
"bearer",
"authorization",
"jwt",
"verification_token",
# 密钥类
"secret",
"app_secret",
"encrypt_key",
"api_key",
"apikey",
"api_secret",
@ -76,6 +80,31 @@ class CryptoHelper:
# 凭证类
"credential",
"credentials",
# URL 内嵌 secret 类
"webhook_url",
"webhook_secret",
)
# 强敏感字段子集:即使 partial 模式也全量遮蔽(不保留前 4 位)。
# 依据password 前缀是常用密码、api_key 前缀可被利用、webhook_url 内嵌
# secretSlack/Discord/Teams任何片段泄露都等于泄露完整凭证。
_FULL_MASK_KEYWORDS = (
"password",
"passwd",
"pwd",
"api_key",
"apikey",
"api_secret",
"client_secret",
"client_key",
"private_key",
"webhook_url",
"webhook_secret",
"ca_cert",
"client_cert",
"certificate",
"credential",
"credentials",
)
def __init__(self, key: str | None = None, fernet: Fernet | None = None) -> None:
@ -206,9 +235,13 @@ class CryptoHelper:
return self._transform_config(config, "decrypt")
def mask_sensitive_fields(self, config: dict[str, Any] | None) -> dict[str, Any]:
"""返回配置字典的脱敏副本,用于 API 响应。"""
result = self._transform_config(config, "mask")
return result if result is not None else {}
"""返回配置字典的脱敏副本(全量遮蔽),用于 API 响应。
等价于 ``mask_full(config)``None 输入返回 ``{}``向后兼容
"""
if config is None:
return {}
return self.mask_full(config)
def mask_all_string_values(self, obj: Any) -> Any:
"""递归将对象中的所有字符串值替换为脱敏占位符。"""
@ -220,6 +253,62 @@ class CryptoHelper:
return "***"
return obj
def mask_full(self, value: Any, *, parent_key: str = "") -> Any:
"""全量遮蔽脱敏(对外 HTTP 响应、审计日志)。
敏感字段统一替换为 ``"***"``不保留任何片段递归语义
- dict按子键判定敏感性
- list按父键判定敏感性修复原 ``_mask_nested`` list 项不脱敏漏洞
- str按父键判定敏感性
- 其他类型原样返回
Args:
value: 待脱敏的任意值dict / list / str / 其他
parent_key: 父级键名用于 list 项的敏感性判定顶层调用留空
Returns:
脱敏后的值新对象不修改原对象
"""
if isinstance(value, dict):
return {k: self.mask_full(v, parent_key=k) for k, v in value.items()}
if isinstance(value, list):
return [self.mask_full(item, parent_key=parent_key) for item in value]
if isinstance(value, str):
if self._is_sensitive_key(parent_key):
return "***"
return value
return value
def mask_partial(self, value: Any, *, parent_key: str = "") -> Any:
"""部分遮蔽脱敏(日志、诊断包)。
- ``_FULL_MASK_KEYWORDS`` 命中 ``"***"``强敏感字段不保留前缀
- 其他 ``_SENSITIVE_KEYWORDS`` 命中 4 + ``"***"``len<=4 时全量
- 非敏感字段原样返回
递归语义同 ``mask_full``
Args:
value: 待脱敏的任意值
parent_key: 父级键名用于 list 项的敏感性判定
Returns:
脱敏后的值新对象不修改原对象
"""
if isinstance(value, dict):
return {k: self.mask_partial(v, parent_key=k) for k, v in value.items()}
if isinstance(value, list):
return [self.mask_partial(item, parent_key=parent_key) for item in value]
if isinstance(value, str):
if not value:
return value
if self._is_full_mask_key(parent_key):
return "***"
if self._is_sensitive_key(parent_key):
return value[:4] + "***" if len(value) > 4 else "***"
return value
return value
# ------------------------------------------------------------------
# 内部实现
# ------------------------------------------------------------------
@ -230,6 +319,12 @@ class CryptoHelper:
normalized = re.sub(r"[^a-z0-9_]+", "_", key.lower())
return any(keyword in normalized for keyword in cls._SENSITIVE_KEYWORDS)
@classmethod
def _is_full_mask_key(cls, key: str) -> bool:
"""判断字段名是否属于强敏感字段partial 模式也全量遮蔽)。"""
normalized = re.sub(r"[^a-z0-9_]+", "_", key.lower())
return any(keyword in normalized for keyword in cls._FULL_MASK_KEYWORDS)
def _transform_config(
self,
config: dict[str, Any] | None,
@ -244,7 +339,7 @@ class CryptoHelper:
if mode == "decrypt":
return self._decrypt_nested(config)
# mask
return self._mask_nested(config)
return self.mask_full(config)
def _encrypt_nested(self, obj: Any) -> Any:
if isinstance(obj, dict):
@ -266,21 +361,6 @@ class CryptoHelper:
return [self._decrypt_nested(item) for item in obj]
return obj
def _mask_nested(self, obj: Any) -> Any:
if isinstance(obj, dict):
return {
k: self._mask_string(v) if self._is_sensitive_key(k) and isinstance(v, str) else self._mask_nested(v)
for k, v in obj.items()
}
if isinstance(obj, list):
return [self._mask_nested(item) for item in obj]
return obj
@staticmethod
def _mask_string(value: str) -> str:
"""对字符串进行脱敏,返回统一占位符。"""
return "***"
# ----------------------------------------------------------------------
# 模块级默认实例与便捷函数(向后兼容现有调用)
@ -339,10 +419,55 @@ def decrypt(value: str) -> str:
def mask_sensitive_fields(config: dict[str, Any] | None) -> dict[str, Any]:
"""返回配置字典的脱敏副本(委托给默认实例)。"""
"""返回配置字典的脱敏副本(全量遮蔽,委托给默认实例)。"""
return _default_helper.mask_sensitive_fields(config)
def mask_all_string_values(obj: Any) -> Any:
"""递归将对象中的所有字符串值替换为脱敏占位符(委托给默认实例)。"""
return _default_helper.mask_all_string_values(obj)
def mask_full(value: Any, *, parent_key: str = "") -> Any:
"""全量遮蔽脱敏(委托给默认实例)。
供对外 HTTP 响应审计日志使用敏感字段统一替换为 ``"***"``
"""
return _default_helper.mask_full(value, parent_key=parent_key)
def mask_partial(value: Any, *, parent_key: str = "") -> Any:
"""部分遮蔽脱敏(委托给默认实例)。
供日志诊断包使用强敏感字段全量遮蔽普通敏感字段保留前 4
"""
return _default_helper.mask_partial(value, parent_key=parent_key)
def is_sensitive_key(key: str) -> bool:
"""判断字段名是否属于敏感字段(委托给默认实例)。"""
return _default_helper._is_sensitive_key(key)
# ----------------------------------------------------------------------
# HTTP header 敏感字段集合(单一事实源)
# ----------------------------------------------------------------------
# 用于过滤 / 脱敏日志与诊断包中的 HTTP header。统一收口到 crypto 模块,
# 避免在 channels/__init__.py / webhook_router.py / grpc/executor.py /
# soap/executor.py 等多处重复定义发散(统一脱敏重构)。
# 取值是 4 处历史定义的并集authorization / cookie / set-cookie /
# x-api-key / x-auth-token / proxy-authorization / token / api-key /
# password。调用方使用 ``key.lower() in SENSITIVE_HTTP_HEADERS`` 判定。
SENSITIVE_HTTP_HEADERS: frozenset[str] = frozenset(
{
"authorization",
"proxy-authorization",
"cookie",
"set-cookie",
"x-api-key",
"x-auth-token",
"token",
"api-key",
"password",
}
)

View File

@ -0,0 +1,40 @@
"""请求级 trace_id 上下文(共享模块)。
通过 ContextVar 存储每个请求的 trace_id供异常处理器日志适配器
等跨层读取中间件入口生成或从 X-Request-Id header 读取
本模块为 server 层与 channels 层共享的 trace_id 机制确保两套
ContextVarserver ``_trace_id_var`` channels ``_current_span``
保持同步
"""
from __future__ import annotations
import uuid
from contextvars import ContextVar
_trace_id_var: ContextVar[str | None] = ContextVar(
"yuxi_request_trace_id", default=None
)
def set_trace_id(trace_id: str) -> None:
"""设置当前请求的 trace_id。"""
_trace_id_var.set(trace_id)
def get_trace_id() -> str | None:
"""获取当前请求的 trace_id未设置时返回 None。"""
return _trace_id_var.get()
def generate_trace_id() -> str:
"""生成新的 trace_iduuid4 十六进制)。"""
return uuid.uuid4().hex
def reset_trace_id() -> None:
"""请求结束时重置 trace_id。
ContextVar 在请求结束后自动回收显式重置作为保险
"""
_trace_id_var.set(None)