ForcePilot/backend/package/yuxi/external_systems/use_cases/mappers.py
Kris 830a3c1458 refactor: 完成多模块代码优化与功能迭代
本次提交包含多项核心改进:
1. 统一列表查询分页范式,替换page/page_size为limit/offset标准分页
2. 新增多项仓库方法与服务接口,完善统计与测试能力
3. 优化审计日志事务处理逻辑,保证业务与审计数据原子性
4. 统一适配器类型与源类型排序契约,添加排序注释说明
5. 完善通知渠道、健康检查、告警等模块的DTO与实体字段
6. 优化测试用例执行回调,透传调用方标识
7. 修复资产导入时的校验逻辑,重新计算size与checksum
8. 拆分UpsertQuotaInput的审计字段,区分创建人与修改人
2026-07-11 06:55:19 +08:00

574 lines
22 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""领域模型(dataclass) ↔ Output DTO(Pydantic) 转换 + 集中脱敏。
规范 §6.1/§7.3
- persistence 返回领域模型 dataclassservice 返回 Output DTO
- 脱敏集中在 mapper调 crypto_service.mask_sensitive_fields
- use_cases 层不调用 encrypt / decrypt
脱敏策略:
- dict 型敏感字段(``connection_config`` / ``auth_config`` / ``secret_refs`` /
``adapter_config``)调用 ``crypto_service.mask_sensitive_fields`` 递归脱敏
- 字符串型敏感字段(``access_token`` / ``refresh_token`` / ``secret``
非空时替换为 ``"***"``
审计字段(``created_by`` / ``updated_by``)由 service 在调用 mapper 前补充,
mapper 仅做字段映射。
"""
from __future__ import annotations
from typing import Any
from yuxi.external_systems.core.models import (
AdapterAsset,
Alert,
AuditLog,
Environment,
Execution,
ExecutionTrace,
ExternalSystem,
ExternalTool,
NotificationChannel,
QuotaUsage,
SecretRotationPolicy,
SystemToken,
ToolAccessRule,
ToolTestCase,
WebhookEvent,
WebhookSubscription,
)
from yuxi.external_systems.framework.runtime import crypto_service
from yuxi.external_systems.use_cases.dto.access_rule import AccessRuleOutput
from yuxi.external_systems.use_cases.dto.alert import AlertOutput
from yuxi.external_systems.use_cases.dto.asset import AssetOutput
from yuxi.external_systems.use_cases.dto.audit_log import AuditLogOutput
from yuxi.external_systems.use_cases.dto.environment import EnvironmentOutput
from yuxi.external_systems.use_cases.dto.execution import (
ExecutionDetailOutput,
ExecutionOutput,
ExecutionTraceOutput,
)
from yuxi.external_systems.use_cases.dto.notification_channel import (
NotificationChannelOutput,
)
from yuxi.external_systems.use_cases.dto.quota import QuotaOutput
from yuxi.external_systems.use_cases.dto.secret_rotation import (
SecretRotationPolicyOutput,
)
from yuxi.external_systems.use_cases.dto.system import (
GovernanceConfigOutput,
SystemOutput,
)
from yuxi.external_systems.use_cases.dto.test_case import TestCaseOutput
from yuxi.external_systems.use_cases.dto.token import TokenOutput
from yuxi.external_systems.use_cases.dto.tool import ToolOutput
from yuxi.external_systems.use_cases.dto.webhook import (
WebhookEventOutput,
WebhookSubscriptionOutput,
)
# 敏感字符串字段的统一脱敏占位符
_MASKED_VALUE = "***"
def _to_iso_str(dt: Any) -> str | None:
"""将 datetime 或 None 转为 ISO 8601 字符串None 透传。"""
if dt is None:
return None
if hasattr(dt, "isoformat"):
return dt.isoformat()
return str(dt)
# ═══════════════════════════════════════════════════════════════════════════
# 含敏感字段的子域mask: bool = True
# ═══════════════════════════════════════════════════════════════════════════
def to_system_output(system: ExternalSystem, *, mask: bool = True) -> SystemOutput:
"""ExternalSystem dataclass → SystemOutput DTO含脱敏
dict 型敏感字段:``connection_config`` / ``auth_config`` / ``secret_refs``。
"""
connection_config = system.connection_config
auth_config = system.auth_config
secret_refs = system.secret_refs
if mask:
connection_config = crypto_service.mask_sensitive_fields(connection_config)
auth_config = crypto_service.mask_sensitive_fields(auth_config)
secret_refs = crypto_service.mask_sensitive_fields(secret_refs)
return SystemOutput(
id=system.id,
slug=system.slug,
name=system.name,
description=system.description,
category=system.category,
adapter_type=system.adapter_type,
source_type=system.source_type,
enabled=system.enabled,
connection_config=connection_config,
auth_type=system.auth_type,
auth_config=auth_config,
secret_refs=secret_refs,
rate_limit=system.rate_limit,
circuit_breaker=system.circuit_breaker,
pool_config=system.pool_config,
observability=system.observability,
timeout=system.timeout,
retry_policy=system.retry_policy,
created_by=system.created_by,
updated_by=system.updated_by,
env_count=system.env_count,
)
def to_tool_output(tool: ExternalTool, *, mask: bool = True) -> ToolOutput:
"""ExternalTool dataclass → ToolOutput DTO含脱敏
dict 型敏感字段:``adapter_config`` / ``auth_config``。
"""
adapter_config = tool.adapter_config
auth_config = tool.auth_config
if mask:
adapter_config = crypto_service.mask_sensitive_fields(adapter_config)
auth_config = crypto_service.mask_sensitive_fields(auth_config)
return ToolOutput(
id=tool.id,
slug=tool.slug,
name=tool.name,
description=tool.description,
category=tool.category,
adapter_type=tool.adapter_type,
auth_type=tool.auth_type,
adapter_config=adapter_config,
auth_config=auth_config,
timeout=tool.timeout,
retry_policy=tool.retry_policy,
enabled=tool.enabled,
system_id=tool.system_id,
created_by=tool.created_by,
updated_by=tool.updated_by,
)
def to_environment_output(env: Environment, *, mask: bool = True) -> EnvironmentOutput:
"""Environment dataclass → EnvironmentOutput DTO含脱敏
dict 型敏感字段:``connection_config`` / ``auth_config``。
整合旧 ``environment_service._mask_secrets`` 的脱敏职责(非 decrypt 部分)。
审计字段(``created_by`` / ``updated_by`` / ``created_at`` / ``updated_at``
透传,``datetime`` 序列化为 ISO 字符串。
"""
connection_config = env.connection_config
auth_config = env.auth_config
if mask:
connection_config = crypto_service.mask_sensitive_fields(connection_config)
auth_config = crypto_service.mask_sensitive_fields(auth_config)
return EnvironmentOutput(
id=env.id,
system_id=env.system_id,
env_key=env.env_key,
name=env.name,
connection_config=connection_config,
auth_config=auth_config,
is_default=env.is_default,
enabled=env.enabled,
created_by=env.created_by,
updated_by=env.updated_by,
created_at=_to_iso_str(env.created_at),
updated_at=_to_iso_str(env.updated_at),
)
def to_token_output(token: SystemToken, *, mask: bool = True) -> TokenOutput:
"""SystemToken dataclass → TokenOutput DTO含脱敏
字符串型敏感字段:``access_token`` / ``refresh_token``,非空时替换为 ``"***"``。
"""
access_token = token.access_token
refresh_token = token.refresh_token
if mask:
access_token = _MASKED_VALUE if access_token else access_token
refresh_token = _MASKED_VALUE if refresh_token else refresh_token
return TokenOutput(
id=token.id,
system_id=token.system_id,
env_key=token.env_key,
token_type=token.token_type,
access_token=access_token,
refresh_token=refresh_token,
expires_at=token.expires_at,
extra=token.extra,
is_invalidated=token.is_invalidated,
)
def to_webhook_subscription_output(
subscription: WebhookSubscription,
*,
mask: bool = True,
) -> WebhookSubscriptionOutput:
"""WebhookSubscription dataclass → WebhookSubscriptionOutput DTO含脱敏
字符串型敏感字段:``secret``,非空时替换为 ``"***"``。
"""
secret = subscription.secret
if mask:
secret = _MASKED_VALUE if secret else secret
return WebhookSubscriptionOutput(
id=subscription.id,
slug=subscription.slug,
name=subscription.name,
system_id=subscription.system_id,
source_event_type=subscription.source_event_type,
target_handler_type=subscription.target_handler_type,
callback_path=subscription.callback_path,
description=subscription.description,
env_key=subscription.env_key,
subscription_id=subscription.subscription_id,
target_handler_id=subscription.target_handler_id,
secret=secret,
secret_algorithm=subscription.secret_algorithm,
filter_rules=subscription.filter_rules,
transform_rules=subscription.transform_rules,
status=subscription.status,
enabled=subscription.enabled,
expires_at=subscription.expires_at,
auto_renew=subscription.auto_renew,
last_renewed_at=subscription.last_renewed_at,
last_renew_error=subscription.last_renew_error,
renewal_attempts=subscription.renewal_attempts,
last_received_at=subscription.last_received_at,
received_count=subscription.received_count,
retention_days=subscription.retention_days,
created_by=subscription.created_by,
updated_by=subscription.updated_by,
)
# ═══════════════════════════════════════════════════════════════════════════
# 无敏感字段的子域(直接字段映射)
# ═══════════════════════════════════════════════════════════════════════════
def to_webhook_event_output(event: WebhookEvent) -> WebhookEventOutput:
"""将 WebhookEvent dataclass 转换为 WebhookEventOutput DTO。"""
return WebhookEventOutput(
id=event.id,
subscription_id=event.subscription_id,
subscription_slug=event.subscription_slug,
external_event_id=event.external_event_id,
event_type=event.event_type,
payload=event.payload,
payload_hash=event.payload_hash,
headers=event.headers,
signature=event.signature,
signature_verified=event.signature_verified,
received_at=event.received_at,
source_ip=event.source_ip,
processing_status=event.processing_status,
processing_attempts=event.processing_attempts,
last_processed_at=event.last_processed_at,
processing_error=event.processing_error,
correlation_id=event.correlation_id,
triggered_handler_type=event.triggered_handler_type,
triggered_handler_id=event.triggered_handler_id,
triggered_execution_id=event.triggered_execution_id,
)
def to_execution_output(execution: Execution) -> ExecutionOutput:
"""Execution dataclass → ExecutionOutput DTO无敏感字段直接映射"""
return ExecutionOutput(
id=execution.id,
execution_id=execution.execution_id,
trace_id=execution.trace_id,
system_id=execution.system_id,
env_key=execution.env_key,
tool_slug=execution.tool_slug,
caller=execution.caller,
caller_id=execution.caller_id,
status=execution.status,
started_at=execution.started_at,
operation=execution.operation,
request_summary=execution.request_summary,
response_summary=execution.response_summary,
error=execution.error,
ended_at=execution.ended_at,
duration_ms=execution.duration_ms,
correlation_id=execution.correlation_id,
retry_of=execution.retry_of,
retry_count=execution.retry_count,
tags=execution.tags,
)
def to_execution_detail_output(
execution: Execution,
traces: list[ExecutionTrace],
) -> ExecutionDetailOutput:
"""Execution + traces → ExecutionDetailOutput DTO。
注意:当前 ``ExecutionDetailOutput`` 未包含 traces 字段,``traces`` 参数
预留用于后续扩展service 层可单独调用 ``to_execution_trace_output``
处理 traces。``system_slug`` / ``system_name`` 由 service 层补充。
"""
_ = traces # 预留参数,当前 DTO 未包含 traces 字段
return ExecutionDetailOutput(
execution=to_execution_output(execution),
)
def to_execution_trace_output(trace: ExecutionTrace) -> ExecutionTraceOutput:
"""ExecutionTrace dataclass → ExecutionTraceOutput DTO无敏感字段直接映射"""
return ExecutionTraceOutput(
id=trace.id,
execution_id=trace.execution_id,
trace_id=trace.trace_id,
spans=trace.spans,
total_duration_ms=trace.total_duration_ms,
created_at=trace.created_at,
updated_at=trace.updated_at,
)
def to_audit_log_output(
audit_log: AuditLog,
system: ExternalSystem | None = None,
) -> AuditLogOutput:
"""AuditLog dataclass → AuditLogOutput DTO无敏感字段直接映射
Args:
audit_log: 审计日志数据类。
system: 可选的所属系统数据类,用于填充 ``system_name`` / ``system_slug``。
列表/详情场景由 service 从 LEFT JOIN 结果中传入。
"""
return AuditLogOutput(
id=audit_log.id,
system_id=audit_log.system_id,
system_name=system.name if system else None,
system_slug=system.slug if system else None,
env_key=audit_log.env_key,
event_type=audit_log.event_type,
action_type=audit_log.action_type,
resource=audit_log.resource,
user=audit_log.user,
detail=audit_log.detail,
snapshot_before=audit_log.snapshot_before,
snapshot_after=audit_log.snapshot_after,
description=audit_log.description,
created_by=audit_log.created_by,
updated_by=audit_log.updated_by,
created_at=audit_log.created_at,
)
def to_asset_output(asset: AdapterAsset) -> AssetOutput:
"""AdapterAsset dataclass → AssetOutput DTO不含二进制 content"""
return AssetOutput(
id=asset.id,
adapter_type=asset.adapter_type,
asset_type=asset.asset_type,
name=asset.name,
size=asset.size,
checksum=asset.checksum,
status=asset.status,
status_message=asset.status_message,
created_by=asset.created_by,
updated_by=asset.updated_by,
created_at=asset.created_at,
updated_at=asset.updated_at,
)
def to_quota_output(quota: QuotaUsage) -> QuotaOutput:
"""QuotaUsage dataclass → QuotaOutput DTO无敏感字段直接映射"""
return QuotaOutput(
id=quota.id,
system_id=quota.system_id,
quota_key=quota.quota_key,
quota_name=quota.quota_name,
quota_window=quota.quota_window,
env_key=quota.env_key,
window_start=quota.window_start,
window_end=quota.window_end,
limit_value=quota.limit_value,
used_value=quota.used_value,
remaining_value=quota.remaining_value,
usage_ratio=quota.usage_ratio,
is_warning=quota.is_warning,
is_critical=quota.is_critical,
unit=quota.unit,
source=quota.source,
source_header_name=quota.source_header_name,
source_header_strategy=quota.source_header_strategy,
warning_threshold=quota.warning_threshold,
critical_threshold=quota.critical_threshold,
last_updated_at=quota.last_updated_at,
last_response_at=quota.last_response_at,
)
def to_alert_output(alert: Alert) -> AlertOutput:
"""Alert dataclass → AlertOutput DTO无敏感字段直接映射
审计字段与 ``system_name`` / ``system_slug`` 为可选字段,由调用方
按需填充(列表查询时由 JOIN 结果填充 system 信息)。
"""
return AlertOutput(
id=alert.id,
system_id=alert.system_id,
alert_type=alert.alert_type,
title=alert.title,
triggered_at=alert.triggered_at,
env_key=alert.env_key,
severity=alert.severity,
status=alert.status,
description=alert.description,
detail=alert.detail,
resource_type=alert.resource_type,
resource_id=alert.resource_id,
related_execution_id=alert.related_execution_id,
related_trace_id=alert.related_trace_id,
metric_snapshot=alert.metric_snapshot,
dedup_key=alert.dedup_key,
resolved_at=alert.resolved_at,
acknowledged_by=alert.acknowledged_by,
acknowledged_at=alert.acknowledged_at,
resolution_note=alert.resolution_note,
notification_status=alert.notification_status,
notification_sent_at=alert.notification_sent_at,
notification_channels=alert.notification_channels,
created_by=alert.created_by,
updated_by=alert.updated_by,
created_at=alert.created_at,
updated_at=alert.updated_at,
system_name=alert.system_name,
system_slug=alert.system_slug,
)
def to_secret_rotation_policy_output(
policy: SecretRotationPolicy,
) -> SecretRotationPolicyOutput:
"""SecretRotationPolicy dataclass → SecretRotationPolicyOutput DTO无敏感字段直接映射
注意:``secret_ref`` 仅是引用 key指向 ``ExternalSystem.secret_refs`` 中的
具名 secret非敏感值本身不脱敏。
"""
return SecretRotationPolicyOutput(
id=policy.id,
system_id=policy.system_id,
secret_type=policy.secret_type,
secret_name=policy.secret_name,
secret_ref=policy.secret_ref,
env_key=policy.env_key,
rotation_mode=policy.rotation_mode,
rotation_interval_days=policy.rotation_interval_days,
notify_before_days=policy.notify_before_days,
notify_channels=policy.notify_channels,
last_rotated_at=policy.last_rotated_at,
next_rotation_at=policy.next_rotation_at,
last_rotation_at=policy.last_rotation_at,
last_rotation_status=policy.last_rotation_status,
last_rotation_error=policy.last_rotation_error,
rotation_count=policy.rotation_count,
max_rotation_retries=policy.max_rotation_retries,
current_retry_count=policy.current_retry_count,
status=policy.status,
)
def to_access_rule_output(rule: ToolAccessRule) -> AccessRuleOutput:
"""ToolAccessRule dataclass → AccessRuleOutput DTO无敏感字段直接映射"""
return AccessRuleOutput(
id=rule.id,
principal_type=rule.principal_type,
principal_id=rule.principal_id,
created_by=rule.created_by,
tool_slug=rule.tool_slug,
system_id=rule.system_id,
principal_name=rule.principal_name,
env_key=rule.env_key,
effect=rule.effect,
priority=rule.priority,
conditions=rule.conditions,
expires_at=rule.expires_at,
enabled=rule.enabled,
description=rule.description,
updated_by=rule.updated_by,
created_at=rule.created_at,
updated_at=rule.updated_at,
)
def to_test_case_output(test_case: ToolTestCase) -> TestCaseOutput:
"""ToolTestCase dataclass → TestCaseOutput DTO无敏感字段直接映射"""
return TestCaseOutput(
id=test_case.id,
tool_slug=test_case.tool_slug,
name=test_case.name,
request_fixture=test_case.request_fixture,
created_by=test_case.created_by,
description=test_case.description,
env_key=test_case.env_key,
expected_status=test_case.expected_status,
assertions=test_case.assertions,
timeout=test_case.timeout,
enabled=test_case.enabled,
schedule_cron=test_case.schedule_cron,
alert_on_failure=test_case.alert_on_failure,
alert_threshold=test_case.alert_threshold,
last_run_at=test_case.last_run_at,
last_run_status=test_case.last_run_status,
last_run_duration_ms=test_case.last_run_duration_ms,
last_run_response=test_case.last_run_response,
last_run_error=test_case.last_run_error,
last_run_assertions_result=test_case.last_run_assertions_result,
consecutive_failures=test_case.consecutive_failures,
updated_by=test_case.updated_by,
)
def to_notification_channel_output(
channel: NotificationChannel,
*,
mask: bool = True,
) -> NotificationChannelOutput:
"""NotificationChannel dataclass → NotificationChannelOutput DTO含脱敏
dict 型敏感字段:``config``(可能含 webhook URL、SMTP 密码等)。
"""
config = channel.config
if mask:
config = crypto_service.mask_sensitive_fields(config)
return NotificationChannelOutput(
id=channel.id,
channel_type=channel.channel_type,
name=channel.name,
config=config,
enabled=channel.enabled,
description=channel.description,
created_by=channel.created_by,
updated_by=channel.updated_by,
created_at=channel.created_at,
updated_at=channel.updated_at,
)
def to_secret_refs_governance_output(
system: ExternalSystem,
*,
mask: bool = True,
) -> GovernanceConfigOutput:
"""ExternalSystem.secret_refs → GovernanceConfigOutput DTO含脱敏
dict 型敏感字段:``secret_refs``,调用 ``crypto_service.mask_sensitive_fields``
递归脱敏。整合旧 ``system_service.get_secret_refs`` / ``update_secret_refs``
的脱敏职责,使 service 层不再直接调用 crypto_service。
"""
secret_refs = system.secret_refs
if mask:
secret_refs = crypto_service.mask_sensitive_fields(secret_refs)
return GovernanceConfigOutput(system_id=system.id, config=secret_refs)