ForcePilot/backend/package/yuxi/external_systems/use_cases/mappers.py
Kris 00a5577aac chore: 整理并合并近期多模块迭代变更
本次提交包含了大量跨模块的功能增强、bug修复与代码优化:
1.  清理冗余空行与导入格式,统一代码风格
2.  新增集成超时异常类,替换原生TimeoutError避免穿透核心层
3.  扩展审计日志DTO与用例,新增变更字段追踪能力
4.  完善各类仓储接口,新增批量操作、时间范围过滤支持
5.  重构健康检查返回格式,统一使用status字段替代冗余的reachable/healthy
6.  扩展仪表盘与各类服务端口,新增待办统计、系统dashboard等能力
7.  优化批量操作与事务处理,新增保存点支持隔离失败操作
8.  修复配额管理阈值计算逻辑,支持自定义告警阈值
9.  统一认证类型错误提示,优化插件注册校验逻辑
10. 扩展SOAP适配器预览能力,兼容bytes类型输入
11. 新增批量恢复、批量硬删除回收站资源的接口与DTO
12. 优化测试用例断言逻辑,兼容前端操作符别名与字段名差异
2026-07-11 21:38:08 +08:00

675 lines
25 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""领域模型(dataclass) ↔ Output DTO(Pydantic) 转换 + 集中脱敏。
规范 §6.1/§7.3
- persistence 返回领域模型 dataclassservice 返回 Output DTO
- 脱敏集中在 mapper调 crypto_service.mask_sensitive_fields
- use_cases 层不调用 encrypt / decrypt
脱敏策略:
- dict 型敏感字段(``connection_config`` / ``auth_config`` / ``secret_refs`` /
``adapter_config``)调用 ``crypto_service.mask_sensitive_fields`` 递归脱敏
- 字符串型敏感字段(``access_token`` / ``refresh_token`` / ``secret``
非空时替换为 ``"***"``
审计字段(``created_by`` / ``updated_by``)由 service 在调用 mapper 前补充,
mapper 仅做字段映射。
"""
from __future__ import annotations
import logging
from typing import Any
from yuxi.external_systems.core.models import (
AdapterAsset,
Alert,
AuditLog,
Environment,
Execution,
ExecutionTrace,
ExternalSystem,
ExternalTool,
NotificationChannel,
QuotaUsage,
SecretRotationPolicy,
SystemToken,
ToolAccessRule,
ToolTestCase,
WebhookEvent,
WebhookSubscription,
)
from yuxi.external_systems.framework.runtime import crypto_service
from yuxi.external_systems.use_cases.dto.access_rule import AccessRuleOutput
from yuxi.external_systems.use_cases.dto.alert import AlertOutput
from yuxi.external_systems.use_cases.dto.asset import AssetOutput
from yuxi.external_systems.use_cases.dto.audit_log import AuditLogOutput
from yuxi.external_systems.use_cases.dto.environment import EnvironmentOutput
from yuxi.external_systems.use_cases.dto.execution import (
ExecutionDetailOutput,
ExecutionOutput,
ExecutionTraceOutput,
)
from yuxi.external_systems.use_cases.dto.notification_channel import (
NotificationChannelOutput,
)
from yuxi.external_systems.use_cases.dto.quota import QuotaOutput
from yuxi.external_systems.use_cases.dto.secret_rotation import (
SecretRotationPolicyOutput,
)
from yuxi.external_systems.use_cases.dto.system import (
GovernanceConfigOutput,
SystemOutput,
)
from yuxi.external_systems.use_cases.dto.test_case import TestCaseOutput
from yuxi.external_systems.use_cases.dto.token import TokenOutput
from yuxi.external_systems.use_cases.dto.tool import ToolOutput
from yuxi.external_systems.use_cases.dto.webhook import (
WebhookEventOutput,
WebhookSubscriptionOutput,
)
# 敏感字符串字段的统一脱敏占位符
_MASKED_VALUE = "***"
logger = logging.getLogger(__name__)
def _safe_mask(data: Any, record_label: str, field_name: str) -> Any:
"""安全脱敏:对非 dict 输入或脱敏失败场景降级,避免单条脏数据拖垮列表查询。
脱敏失败时记录日志并返回原值dict 型字段返回空 dict保证列表可用性。
``record_label`` 为调用方提供的记录标识(如 "系统 5" / "工具 12"),用于日志定位。
"""
if data is None:
return {}
if not isinstance(data, dict):
logger.warning(
"%s%s 字段类型为 %s,无法脱敏,原样返回",
record_label,
field_name,
type(data).__name__,
)
return data
try:
return crypto_service.mask_sensitive_fields(data)
except Exception as exc: # noqa: BLE001
logger.exception(
"%s%s 字段脱敏失败: %s",
record_label,
field_name,
exc,
)
return data
def _to_iso_str(dt: Any) -> str | None:
"""将 datetime 或 None 转为 ISO 8601 字符串None 透传。"""
if dt is None:
return None
if hasattr(dt, "isoformat"):
return dt.isoformat()
return str(dt)
# ═══════════════════════════════════════════════════════════════════════════
# 含敏感字段的子域mask: bool = True
# ═══════════════════════════════════════════════════════════════════════════
def to_system_output(system: ExternalSystem, *, mask: bool = True) -> SystemOutput:
"""ExternalSystem dataclass → SystemOutput DTO含脱敏
dict 型敏感字段:``connection_config`` / ``auth_config`` / ``secret_refs``。
脱敏失败时按字段降级,避免单条脏数据导致整页 500。
"""
connection_config = system.connection_config
auth_config = system.auth_config
secret_refs = system.secret_refs
if mask:
label = f"系统 {system.id}"
connection_config = _safe_mask(connection_config, label, "connection_config")
auth_config = _safe_mask(auth_config, label, "auth_config")
secret_refs = _safe_mask(secret_refs, label, "secret_refs")
return SystemOutput(
id=system.id,
slug=system.slug,
name=system.name,
description=system.description,
category=system.category,
adapter_type=system.adapter_type,
source_type=system.source_type,
enabled=system.enabled,
connection_config=connection_config,
auth_type=system.auth_type,
auth_config=auth_config,
secret_refs=secret_refs,
rate_limit=system.rate_limit,
circuit_breaker=system.circuit_breaker,
pool_config=system.pool_config,
observability=system.observability,
timeout=system.timeout,
retry_policy=system.retry_policy,
created_by=system.created_by,
updated_by=system.updated_by,
version=system.version,
env_count=system.env_count,
)
def to_tool_output(
tool: ExternalTool,
*,
system_name: str | None = None,
mask: bool = True,
) -> ToolOutput:
"""ExternalTool dataclass → ToolOutput DTO含脱敏
dict 型敏感字段:``adapter_config`` / ``auth_config``。
``system_name`` 由调用方批量查询后注入,避免 mapper 引入仓储依赖。
脱敏失败时按字段降级,避免单条脏数据导致整页 500。
"""
adapter_config = tool.adapter_config
auth_config = tool.auth_config
if mask:
label = f"工具 {tool.id}"
adapter_config = _safe_mask(adapter_config, label, "adapter_config")
auth_config = _safe_mask(auth_config, label, "auth_config")
return ToolOutput(
id=tool.id,
slug=tool.slug,
name=tool.name,
description=tool.description,
category=tool.category,
adapter_type=tool.adapter_type,
auth_type=tool.auth_type,
adapter_config=adapter_config,
auth_config=auth_config,
timeout=tool.timeout,
retry_policy=tool.retry_policy,
enabled=tool.enabled,
system_id=tool.system_id,
system_name=system_name,
source_type=tool.source_type,
created_by=tool.created_by,
updated_by=tool.updated_by,
created_at=_to_iso_str(tool.created_at),
updated_at=_to_iso_str(tool.updated_at),
)
def to_environment_output(env: Environment, *, mask: bool = True) -> EnvironmentOutput:
"""Environment dataclass → EnvironmentOutput DTO含脱敏
dict 型敏感字段:``connection_config`` / ``auth_config``。
整合旧 ``environment_service._mask_secrets`` 的脱敏职责(非 decrypt 部分)。
审计字段(``created_by`` / ``updated_by`` / ``created_at`` / ``updated_at``
透传,``datetime`` 序列化为 ISO 字符串。
脱敏失败时按字段降级,避免单条脏数据导致整页 500。
"""
connection_config = env.connection_config
auth_config = env.auth_config
if mask:
label = f"环境 {env.id}"
connection_config = _safe_mask(connection_config, label, "connection_config")
auth_config = _safe_mask(auth_config, label, "auth_config")
return EnvironmentOutput(
id=env.id,
system_id=env.system_id,
env_key=env.env_key,
name=env.name,
connection_config=connection_config,
auth_config=auth_config,
is_default=env.is_default,
enabled=env.enabled,
created_by=env.created_by,
updated_by=env.updated_by,
created_at=_to_iso_str(env.created_at),
updated_at=_to_iso_str(env.updated_at),
)
def to_token_output(token: SystemToken, *, mask: bool = True) -> TokenOutput:
"""SystemToken dataclass → TokenOutput DTO含脱敏
字符串型敏感字段:``access_token`` / ``refresh_token``,非空时替换为 ``"***"``。
审计字段(``created_by`` / ``updated_by``)透传,``datetime`` 序列化为 ISO 字符串。
"""
access_token = token.access_token
refresh_token = token.refresh_token
if mask:
access_token = _MASKED_VALUE if access_token else access_token
refresh_token = _MASKED_VALUE if refresh_token else refresh_token
return TokenOutput(
id=token.id,
system_id=token.system_id,
env_key=token.env_key,
token_type=token.token_type,
access_token=access_token,
refresh_token=refresh_token,
expires_at=token.expires_at,
extra=token.extra,
is_invalidated=token.is_invalidated,
created_by=token.created_by,
updated_by=token.updated_by,
created_at=_to_iso_str(token.created_at),
updated_at=_to_iso_str(token.updated_at),
)
def to_webhook_subscription_output(
subscription: WebhookSubscription,
*,
mask: bool = True,
) -> WebhookSubscriptionOutput:
"""WebhookSubscription dataclass → WebhookSubscriptionOutput DTO含脱敏
字符串型敏感字段:``secret``,非空时替换为 ``"***"``。
"""
secret = subscription.secret
if mask:
secret = _MASKED_VALUE if secret else secret
return WebhookSubscriptionOutput(
id=subscription.id,
slug=subscription.slug,
name=subscription.name,
system_id=subscription.system_id,
source_event_type=subscription.source_event_type,
target_handler_type=subscription.target_handler_type,
callback_path=subscription.callback_path,
description=subscription.description,
env_key=subscription.env_key,
subscription_id=subscription.subscription_id,
target_handler_id=subscription.target_handler_id,
secret=secret,
secret_algorithm=subscription.secret_algorithm,
filter_rules=subscription.filter_rules,
transform_rules=subscription.transform_rules,
status=subscription.status,
enabled=subscription.enabled,
expires_at=subscription.expires_at,
auto_renew=subscription.auto_renew,
last_renewed_at=subscription.last_renewed_at,
last_renew_error=subscription.last_renew_error,
renewal_attempts=subscription.renewal_attempts,
last_received_at=subscription.last_received_at,
received_count=subscription.received_count,
retention_days=subscription.retention_days,
created_by=subscription.created_by,
updated_by=subscription.updated_by,
)
# ═══════════════════════════════════════════════════════════════════════════
# 无敏感字段的子域(直接字段映射)
# ═══════════════════════════════════════════════════════════════════════════
def to_webhook_event_output(event: WebhookEvent) -> WebhookEventOutput:
"""将 WebhookEvent dataclass 转换为 WebhookEventOutput DTO。"""
return WebhookEventOutput(
id=event.id,
subscription_id=event.subscription_id,
subscription_slug=event.subscription_slug,
external_event_id=event.external_event_id,
event_type=event.event_type,
payload=event.payload,
payload_hash=event.payload_hash,
headers=event.headers,
signature=event.signature,
signature_verified=event.signature_verified,
received_at=event.received_at,
source_ip=event.source_ip,
processing_status=event.processing_status,
processing_attempts=event.processing_attempts,
last_processed_at=event.last_processed_at,
processing_error=event.processing_error,
correlation_id=event.correlation_id,
triggered_handler_type=event.triggered_handler_type,
triggered_handler_id=event.triggered_handler_id,
triggered_execution_id=event.triggered_execution_id,
)
def to_execution_output(execution: Execution) -> ExecutionOutput:
"""Execution dataclass → ExecutionOutput DTO无敏感字段直接映射"""
return ExecutionOutput(
id=execution.id,
execution_id=execution.execution_id,
trace_id=execution.trace_id,
system_id=execution.system_id,
env_key=execution.env_key,
tool_slug=execution.tool_slug,
caller=execution.caller,
caller_id=execution.caller_id,
status=execution.status,
started_at=execution.started_at,
operation=execution.operation,
request_summary=execution.request_summary,
response_summary=execution.response_summary,
error=execution.error,
ended_at=execution.ended_at,
duration_ms=execution.duration_ms,
correlation_id=execution.correlation_id,
retry_of=execution.retry_of,
retry_count=execution.retry_count,
tags=execution.tags,
)
def to_execution_detail_output(
execution: Execution,
traces: list[ExecutionTrace],
) -> ExecutionDetailOutput:
"""Execution + traces → ExecutionDetailOutput DTO。
注意:当前 ``ExecutionDetailOutput`` 未包含 traces 字段,``traces`` 参数
预留用于后续扩展service 层可单独调用 ``to_execution_trace_output``
处理 traces。``system_slug`` / ``system_name`` 由 service 层补充。
"""
_ = traces # 预留参数,当前 DTO 未包含 traces 字段
return ExecutionDetailOutput(
execution=to_execution_output(execution),
)
def to_execution_trace_output(trace: ExecutionTrace) -> ExecutionTraceOutput:
"""ExecutionTrace dataclass → ExecutionTraceOutput DTO无敏感字段直接映射"""
return ExecutionTraceOutput(
id=trace.id,
execution_id=trace.execution_id,
trace_id=trace.trace_id,
spans=trace.spans,
total_duration_ms=trace.total_duration_ms,
created_at=trace.created_at,
updated_at=trace.updated_at,
)
def compute_changed_fields(
before: dict[str, Any] | None,
after: dict[str, Any] | None,
) -> list[str]:
"""对比 snapshot_before 与 snapshot_after返回变更字段列表。
变更定义:
- before 中存在但 after 中不存在(删除字段)
- after 中存在但 before 中不存在(新增字段)
- before 与 after 中均存在但值不同(修改字段)
Note:
仅对比顶层 keys不递归对比嵌套 dict避免过度复杂
"""
before = before or {}
after = after or {}
changed: list[str] = []
for key in set(before.keys()) | set(after.keys()):
if key not in before:
changed.append(key)
elif key not in after:
changed.append(key)
elif before[key] != after[key]:
changed.append(key)
return sorted(changed)
def to_audit_log_output(
audit_log: AuditLog,
system: ExternalSystem | None = None,
) -> AuditLogOutput:
"""AuditLog dataclass → AuditLogOutput DTO无敏感字段直接映射
Args:
audit_log: 审计日志数据类。
system: 可选的所属系统数据类,用于填充 ``system_name`` / ``system_slug``。
列表/详情场景由 service 从 LEFT JOIN 结果中传入。
"""
return AuditLogOutput(
id=audit_log.id,
system_id=audit_log.system_id,
system_name=system.name if system else None,
system_slug=system.slug if system else None,
env_key=audit_log.env_key,
event_type=audit_log.event_type,
action_type=audit_log.action_type,
resource=audit_log.resource,
user=audit_log.user,
detail=audit_log.detail,
snapshot_before=audit_log.snapshot_before,
snapshot_after=audit_log.snapshot_after,
description=audit_log.description,
created_by=audit_log.created_by,
updated_by=audit_log.updated_by,
created_at=audit_log.created_at,
changed_fields=compute_changed_fields(
audit_log.snapshot_before,
audit_log.snapshot_after,
),
)
def to_asset_output(asset: AdapterAsset) -> AssetOutput:
"""AdapterAsset dataclass → AssetOutput DTO不含二进制 content"""
return AssetOutput(
id=asset.id,
adapter_type=asset.adapter_type,
asset_type=asset.asset_type,
name=asset.name,
size=asset.size,
checksum=asset.checksum,
status=asset.status,
status_message=asset.status_message,
created_by=asset.created_by,
updated_by=asset.updated_by,
created_at=asset.created_at,
updated_at=asset.updated_at,
)
def to_quota_output(
quota: QuotaUsage,
*,
system_name: str | None = None,
) -> QuotaOutput:
"""QuotaUsage dataclass → QuotaOutput DTO无敏感字段直接映射
``system_name`` 由调用方批量查询后注入,避免 mapper 引入仓储依赖,
对齐 ``to_tool_output`` / ``to_alert_output`` 约定。
"""
return QuotaOutput(
id=quota.id,
system_id=quota.system_id,
system_name=system_name,
quota_key=quota.quota_key,
quota_name=quota.quota_name,
quota_window=quota.quota_window,
env_key=quota.env_key,
window_start=quota.window_start,
window_end=quota.window_end,
limit_value=quota.limit_value,
used_value=quota.used_value,
remaining_value=quota.remaining_value,
usage_ratio=quota.usage_ratio,
is_warning=quota.is_warning,
is_critical=quota.is_critical,
unit=quota.unit,
source=quota.source,
source_header_name=quota.source_header_name,
source_header_strategy=quota.source_header_strategy,
warning_threshold=quota.warning_threshold,
critical_threshold=quota.critical_threshold,
last_updated_at=quota.last_updated_at,
last_response_at=quota.last_response_at,
)
def to_alert_output(alert: Alert) -> AlertOutput:
"""Alert dataclass → AlertOutput DTO无敏感字段直接映射
审计字段与 ``system_name`` / ``system_slug`` 为可选字段,由调用方
按需填充(列表查询时由 JOIN 结果填充 system 信息)。
"""
return AlertOutput(
id=alert.id,
system_id=alert.system_id,
alert_type=alert.alert_type,
title=alert.title,
triggered_at=alert.triggered_at,
env_key=alert.env_key,
severity=alert.severity,
status=alert.status,
description=alert.description,
detail=alert.detail,
resource_type=alert.resource_type,
resource_id=alert.resource_id,
related_execution_id=alert.related_execution_id,
related_trace_id=alert.related_trace_id,
metric_snapshot=alert.metric_snapshot,
dedup_key=alert.dedup_key,
resolved_at=alert.resolved_at,
acknowledged_by=alert.acknowledged_by,
acknowledged_at=alert.acknowledged_at,
resolution_note=alert.resolution_note,
notification_status=alert.notification_status,
notification_sent_at=alert.notification_sent_at,
notification_channels=alert.notification_channels,
created_by=alert.created_by,
updated_by=alert.updated_by,
created_at=alert.created_at,
updated_at=alert.updated_at,
system_name=alert.system_name,
system_slug=alert.system_slug,
)
def to_secret_rotation_policy_output(
policy: SecretRotationPolicy,
) -> SecretRotationPolicyOutput:
"""SecretRotationPolicy dataclass → SecretRotationPolicyOutput DTO无敏感字段直接映射
注意:``secret_ref`` 仅是引用 key指向 ``ExternalSystem.secret_refs`` 中的
具名 secret非敏感值本身不脱敏。
datetime 字段统一序列化为 ISO 8601 字符串,与 DTO 类型对齐。
"""
return SecretRotationPolicyOutput(
id=policy.id,
system_id=policy.system_id,
secret_type=policy.secret_type,
secret_name=policy.secret_name,
secret_ref=policy.secret_ref,
env_key=policy.env_key,
rotation_mode=policy.rotation_mode,
rotation_interval_days=policy.rotation_interval_days,
notify_before_days=policy.notify_before_days,
notify_channels=policy.notify_channels,
last_rotated_at=_to_iso_str(policy.last_rotated_at),
next_rotation_at=_to_iso_str(policy.next_rotation_at),
last_rotation_at=_to_iso_str(policy.last_rotation_at),
last_rotation_status=policy.last_rotation_status,
last_rotation_error=policy.last_rotation_error,
rotation_count=policy.rotation_count,
max_rotation_retries=policy.max_rotation_retries,
current_retry_count=policy.current_retry_count,
status=policy.status,
version=policy.version,
created_by=policy.created_by,
updated_by=policy.updated_by,
created_at=_to_iso_str(policy.created_at),
updated_at=_to_iso_str(policy.updated_at),
)
def to_access_rule_output(rule: ToolAccessRule) -> AccessRuleOutput:
"""ToolAccessRule dataclass → AccessRuleOutput DTO无敏感字段直接映射"""
return AccessRuleOutput(
id=rule.id,
principal_type=rule.principal_type,
principal_id=rule.principal_id,
created_by=rule.created_by,
tool_slug=rule.tool_slug,
system_id=rule.system_id,
principal_name=rule.principal_name,
env_key=rule.env_key,
effect=rule.effect,
priority=rule.priority,
conditions=rule.conditions,
expires_at=rule.expires_at,
enabled=rule.enabled,
description=rule.description,
updated_by=rule.updated_by,
created_at=rule.created_at,
updated_at=rule.updated_at,
)
def to_test_case_output(test_case: ToolTestCase) -> TestCaseOutput:
"""ToolTestCase dataclass → TestCaseOutput DTO无敏感字段直接映射"""
return TestCaseOutput(
id=test_case.id,
tool_slug=test_case.tool_slug,
name=test_case.name,
request_fixture=test_case.request_fixture,
created_by=test_case.created_by,
description=test_case.description,
env_key=test_case.env_key,
expected_status=test_case.expected_status,
assertions=test_case.assertions,
timeout=test_case.timeout,
enabled=test_case.enabled,
schedule_cron=test_case.schedule_cron,
alert_on_failure=test_case.alert_on_failure,
alert_threshold=test_case.alert_threshold,
last_run_at=test_case.last_run_at,
last_run_status=test_case.last_run_status,
last_run_duration_ms=test_case.last_run_duration_ms,
last_run_response=test_case.last_run_response,
last_run_error=test_case.last_run_error,
last_run_assertions_result=test_case.last_run_assertions_result,
consecutive_failures=test_case.consecutive_failures,
updated_by=test_case.updated_by,
)
def to_notification_channel_output(
channel: NotificationChannel,
*,
mask: bool = True,
) -> NotificationChannelOutput:
"""NotificationChannel dataclass → NotificationChannelOutput DTO含脱敏
dict 型敏感字段:``config``(可能含 webhook URL、SMTP 密码等)。
脱敏失败时按字段降级,避免单条脏数据导致整页 500。
"""
config = channel.config
if mask:
config = _safe_mask(config, f"通知渠道 {channel.id}", "config")
return NotificationChannelOutput(
id=channel.id,
channel_type=channel.channel_type,
name=channel.name,
config=config,
enabled=channel.enabled,
description=channel.description,
created_by=channel.created_by,
updated_by=channel.updated_by,
created_at=channel.created_at,
updated_at=channel.updated_at,
)
def to_secret_refs_governance_output(
system: ExternalSystem,
*,
mask: bool = True,
) -> GovernanceConfigOutput:
"""ExternalSystem.secret_refs → GovernanceConfigOutput DTO含脱敏
dict 型敏感字段:``secret_refs``,调用 ``crypto_service.mask_sensitive_fields``
递归脱敏。整合旧 ``system_service.get_secret_refs`` / ``update_secret_refs``
的脱敏职责,使 service 层不再直接调用 crypto_service。
脱敏失败时按字段降级,避免单条脏数据导致整页 500。
"""
secret_refs = system.secret_refs
if mask:
secret_refs = _safe_mask(secret_refs, f"系统 {system.id}", "secret_refs")
return GovernanceConfigOutput(system_id=system.id, config=secret_refs)