ForcePilot/backend/package/yuxi/external_systems/adapters/persistence/mappers.py
Kris 00a5577aac chore: 整理并合并近期多模块迭代变更
本次提交包含了大量跨模块的功能增强、bug修复与代码优化:
1.  清理冗余空行与导入格式,统一代码风格
2.  新增集成超时异常类,替换原生TimeoutError避免穿透核心层
3.  扩展审计日志DTO与用例,新增变更字段追踪能力
4.  完善各类仓储接口,新增批量操作、时间范围过滤支持
5.  重构健康检查返回格式,统一使用status字段替代冗余的reachable/healthy
6.  扩展仪表盘与各类服务端口,新增待办统计、系统dashboard等能力
7.  优化批量操作与事务处理,新增保存点支持隔离失败操作
8.  修复配额管理阈值计算逻辑,支持自定义告警阈值
9.  统一认证类型错误提示,优化插件注册校验逻辑
10. 扩展SOAP适配器预览能力,兼容bytes类型输入
11. 新增批量恢复、批量硬删除回收站资源的接口与DTO
12. 优化测试用例断言逻辑,兼容前端操作符别名与字段名差异
2026-07-11 21:38:08 +08:00

676 lines
24 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""ORM ↔ dataclass 映射与加解密边界。
本模块是 ``adapters/persistence`` 层**唯一**调用 ``yuxi.utils.crypto`` 加解密函数的位置。
所有 ORM 实例 ↔ dataclass 转换在此完成core 层不接触 ORMdataclass 始终持有明文。
设计原则(见设计方案 §4
- **mapper 不调用 ORM 的 ``to_dict`` 方法**:直接读 ORM 字段构造 dataclass。
原因:``ExternalSystemToken.to_dict`` / ``ExternalWebhookSubscription.to_dict``
不返回敏感字段明文(仅返回 ``has_access_token`` / ``has_secret`` 布尔字段),
且 ``to_dict`` 返回的 dict 结构与 dataclass 字段不完全一致。
- **5 个含敏感字段的 dataclass** 在读取时解密、写入前加密:
- dict 型敏感字段(``auth_config`` / ``secret_refs`` / ``adapter_config``)使用
``encrypt_sensitive_fields`` / ``decrypt_sensitive_fields``
- 单值敏感字段(``access_token`` / ``refresh_token`` / ``secret``)使用
``encrypt`` / ``decrypt``。
- **13 个无敏感字段的 dataclass** 直接字段映射。
- **解密失败时向上抛 ``ValueError``**,不掩盖密钥变更问题。
- ``ExternalSystem.env_count`` 是 ORM ``column_property`` 动态计算字段mapper
读取时直接读 ``orm.env_count``dataclass → ORM 转换时跳过该字段(不写入 DB
"""
from __future__ import annotations
from typing import Any
from yuxi.external_systems.core.models import (
AdapterAsset,
Alert,
AuditLog,
Environment,
Execution,
ExecutionTrace,
ExternalSecret,
ExternalSecretRotationRecord,
ExternalSystem,
ExternalTool,
HealthCheck,
Metric,
NotificationChannel,
QuotaUsage,
SecretRotationPolicy,
SystemToken,
ToolAccessRule,
ToolTestCase,
ToolVersion,
WebhookEvent,
WebhookSubscription,
)
from yuxi.storage.postgres.models_external import (
ExternalAdapterAsset,
ExternalNotificationChannel,
ExternalSecretRotationPolicy,
ExternalSystemAlert,
ExternalSystemAuditLog,
ExternalSystemEnvironment,
ExternalSystemHealthCheck,
ExternalSystemMetric,
ExternalSystemQuotaUsage,
ExternalSystemToken,
ExternalToolAccessRule,
ExternalToolConfig,
ExternalToolConfigVersion,
ExternalToolExecution,
ExternalToolExecutionTrace,
ExternalToolTestCase,
ExternalWebhookEvent,
ExternalWebhookSubscription,
)
from yuxi.storage.postgres.models_external import (
ExternalSecret as ExternalSecretORM,
)
from yuxi.storage.postgres.models_external import (
ExternalSecretRotationRecord as ExternalSecretRotationRecordORM,
)
from yuxi.storage.postgres.models_external import (
ExternalSystem as ExternalSystemORM,
)
from yuxi.utils.crypto import (
decrypt,
decrypt_sensitive_fields,
encrypt,
encrypt_sensitive_fields,
)
# ═══════════════════════════════════════════════════════════════════════════
# 含敏感字段的 dataclass读取时解密 / 写入前加密)
# ═══════════════════════════════════════════════════════════════════════════
# ─── ExternalSystemdict 型敏感字段auth_config / secret_refs ──────────
def orm_to_external_system(orm: ExternalSystemORM) -> ExternalSystem:
"""ORM → dataclass解密 ``auth_config`` / ``secret_refs``。
解密失败时抛 ``ValueError``(密钥变更场景),由调用方决定处理策略。
``env_count`` 为 ORM ``column_property`` 动态计算字段,直接读取。
"""
return ExternalSystem(
id=orm.id,
slug=orm.slug,
name=orm.name,
description=orm.description,
category=orm.category,
adapter_type=orm.adapter_type,
source_type=orm.source_type,
enabled=orm.enabled,
connection_config=orm.connection_config or {},
auth_type=orm.auth_type,
auth_config=decrypt_sensitive_fields(orm.auth_config) or {},
secret_refs=decrypt_sensitive_fields(orm.secret_refs) or {},
rate_limit=orm.rate_limit or {},
circuit_breaker=orm.circuit_breaker or {},
pool_config=orm.pool_config or {},
observability=orm.observability or {},
timeout=orm.timeout,
retry_policy=orm.retry_policy or {},
created_by=orm.created_by,
updated_by=orm.updated_by,
version=orm.version,
env_count=orm.env_count,
)
def encrypt_system_for_write(data: dict[str, Any]) -> dict[str, Any]:
"""明文 dict → 密文 dict加密 ``auth_config`` / ``secret_refs``。
注意:``encrypt`` 不幂等,调用方必须保证传入的是明文(未被加密过)。
"""
data = dict(data)
if data.get("auth_config") is not None:
data["auth_config"] = encrypt_sensitive_fields(data["auth_config"])
if data.get("secret_refs") is not None:
data["secret_refs"] = encrypt_sensitive_fields(data["secret_refs"])
return data
# ─── Environmentdict 型敏感字段auth_config ──────────────────────────
def orm_to_environment(orm: ExternalSystemEnvironment) -> Environment:
"""ORM → dataclass解密 ``auth_config``。"""
return Environment(
id=orm.id,
system_id=orm.system_id,
env_key=orm.env_key,
name=orm.name,
connection_config=orm.connection_config or {},
auth_config=decrypt_sensitive_fields(orm.auth_config) or {},
is_default=orm.is_default,
enabled=orm.enabled,
created_by=orm.created_by,
updated_by=orm.updated_by,
created_at=orm.created_at,
updated_at=orm.updated_at,
)
def encrypt_environment_for_write(data: dict[str, Any]) -> dict[str, Any]:
"""明文 dict → 密文 dict加密 ``auth_config``。"""
data = dict(data)
if data.get("auth_config") is not None:
data["auth_config"] = encrypt_sensitive_fields(data["auth_config"])
return data
# ─── ExternalTooldict 型敏感字段auth_config / adapter_config ──────────
def orm_to_external_tool(orm: ExternalToolConfig) -> ExternalTool:
"""ORM → dataclass解密 ``auth_config`` / ``adapter_config``。
注意:``adapter_config`` 在 ORM 中 ``nullable=False``,但可能包含敏感字段
(如 mTLS 证书路径),统一走 ``decrypt_sensitive_fields``。
"""
return ExternalTool(
id=orm.id,
slug=orm.slug,
name=orm.name,
description=orm.description,
category=orm.category,
adapter_type=orm.adapter_type,
auth_type=orm.auth_type,
adapter_config=decrypt_sensitive_fields(orm.adapter_config) or {},
auth_config=decrypt_sensitive_fields(orm.auth_config) or {},
timeout=orm.timeout,
retry_policy=orm.retry_policy or {},
enabled=orm.enabled,
system_id=orm.system_id,
source_type=orm.source_type,
created_by=orm.created_by,
updated_by=orm.updated_by,
created_at=orm.created_at,
updated_at=orm.updated_at,
)
def encrypt_tool_for_write(data: dict[str, Any]) -> dict[str, Any]:
"""明文 dict → 密文 dict加密 ``auth_config`` / ``adapter_config``。"""
data = dict(data)
if data.get("auth_config") is not None:
data["auth_config"] = encrypt_sensitive_fields(data["auth_config"])
if data.get("adapter_config") is not None:
data["adapter_config"] = encrypt_sensitive_fields(data["adapter_config"])
return data
# ─── SystemToken单值加密access_token / refresh_token ─────────────────
def orm_to_system_token(orm: ExternalSystemToken) -> SystemToken:
"""ORM → dataclass单值解密 ``access_token`` / ``refresh_token``。
注意:不调用 ``orm.to_dict()``(它只返回 ``has_access_token`` 布尔字段)。
"""
return SystemToken(
id=orm.id,
system_id=orm.system_id,
env_key=orm.env_key,
token_type=orm.token_type,
access_token=decrypt(orm.access_token) if orm.access_token else "",
refresh_token=decrypt(orm.refresh_token) if orm.refresh_token else None,
expires_at=orm.expires_at,
extra=orm.extra or {},
is_invalidated=orm.is_invalidated,
created_by=orm.created_by,
updated_by=orm.updated_by,
created_at=orm.created_at,
updated_at=orm.updated_at,
)
def encrypt_token_for_write(data: dict[str, Any]) -> dict[str, Any]:
"""明文 dict → 密文 dict单值加密 ``access_token`` / ``refresh_token``。"""
data = dict(data)
if data.get("access_token"):
data["access_token"] = encrypt(data["access_token"])
if data.get("refresh_token"):
data["refresh_token"] = encrypt(data["refresh_token"])
return data
# ─── WebhookSubscription单值加密secret ──────────────────────────────
def orm_to_webhook_subscription(orm: ExternalWebhookSubscription) -> WebhookSubscription:
"""ORM → dataclass单值解密 ``secret``。
注意:不调用 ``orm.to_dict()``(它只返回 ``has_secret`` 布尔字段)。
"""
return WebhookSubscription(
id=orm.id,
slug=orm.slug,
name=orm.name,
system_id=orm.system_id,
source_event_type=orm.source_event_type,
target_handler_type=orm.target_handler_type,
callback_path=orm.callback_path,
description=orm.description or "",
env_key=orm.env_key,
subscription_id=orm.subscription_id,
target_handler_id=orm.target_handler_id,
secret=decrypt(orm.secret) if orm.secret else None,
secret_algorithm=orm.secret_algorithm,
filter_rules=orm.filter_rules or {},
transform_rules=orm.transform_rules or {},
status=orm.status,
enabled=orm.enabled,
expires_at=orm.expires_at,
auto_renew=orm.auto_renew,
last_renewed_at=orm.last_renewed_at,
last_renew_error=orm.last_renew_error,
renewal_attempts=orm.renewal_attempts,
last_received_at=orm.last_received_at,
received_count=orm.received_count,
retention_days=orm.retention_days,
created_by=orm.created_by,
updated_by=orm.updated_by,
)
def encrypt_webhook_subscription_for_write(data: dict[str, Any]) -> dict[str, Any]:
"""明文 dict → 密文 dict单值加密 ``secret``。"""
data = dict(data)
if data.get("secret"):
data["secret"] = encrypt(data["secret"])
return data
# ═══════════════════════════════════════════════════════════════════════════
# 无敏感字段的 dataclass直接字段映射
# ═══════════════════════════════════════════════════════════════════════════
def orm_to_tool_version(orm: ExternalToolConfigVersion) -> ToolVersion:
"""ORM → dataclass无敏感字段直接映射。"""
return ToolVersion(
id=orm.id,
tool_slug=orm.tool_slug,
version_id=orm.version_id,
snapshot=orm.snapshot or {},
created_by=orm.created_by,
created_at=orm.created_at,
updated_by=orm.updated_by,
updated_at=orm.updated_at,
)
def orm_to_adapter_asset(orm: ExternalAdapterAsset) -> AdapterAsset:
"""ORM → dataclass无敏感字段直接映射。``content`` 为二进制原值。"""
return AdapterAsset(
id=orm.id,
adapter_type=orm.adapter_type,
asset_type=orm.asset_type,
name=orm.name,
content=orm.content or b"",
size=orm.size or 0,
checksum=orm.checksum,
status=orm.status,
status_message=orm.status_message,
created_by=orm.created_by,
updated_by=orm.updated_by,
created_at=orm.created_at,
updated_at=orm.updated_at,
)
def orm_to_execution(orm: ExternalToolExecution) -> Execution:
"""ORM → dataclass无敏感字段直接映射。"""
return Execution(
id=orm.id,
execution_id=orm.execution_id,
trace_id=orm.trace_id,
system_id=orm.system_id,
env_key=orm.env_key,
tool_slug=orm.tool_slug,
caller=orm.caller,
caller_id=orm.caller_id,
status=orm.status,
started_at=orm.started_at,
operation=orm.operation,
request_summary=orm.request_summary or {},
response_summary=orm.response_summary,
error=orm.error,
ended_at=orm.ended_at,
duration_ms=orm.duration_ms,
correlation_id=orm.correlation_id,
retry_of=orm.retry_of,
retry_count=orm.retry_count,
tags=orm.tags or {},
)
def orm_to_execution_trace(orm: ExternalToolExecutionTrace) -> ExecutionTrace:
"""ORM → dataclass无敏感字段直接映射。"""
return ExecutionTrace(
id=orm.id,
execution_id=orm.execution_id,
trace_id=orm.trace_id,
spans=orm.spans or {},
total_duration_ms=orm.total_duration_ms,
created_by=orm.created_by,
updated_by=orm.updated_by,
created_at=orm.created_at,
updated_at=orm.updated_at,
)
def orm_to_audit_log(orm: ExternalSystemAuditLog) -> AuditLog:
"""ORM → dataclass无敏感字段直接映射。"""
return AuditLog(
id=orm.id,
system_id=orm.system_id,
env_key=orm.env_key,
event_type=orm.event_type,
action_type=orm.action_type,
resource=orm.resource,
user=orm.user,
detail=orm.detail or {},
snapshot_before=orm.snapshot_before,
snapshot_after=orm.snapshot_after,
description=orm.description,
created_by=orm.created_by,
updated_by=orm.updated_by,
created_at=orm.created_at,
)
def orm_to_metric(orm: ExternalSystemMetric) -> Metric:
"""ORM → dataclass无敏感字段直接映射。"""
return Metric(
id=orm.id,
system_id=orm.system_id,
env_key=orm.env_key,
bucket_at=orm.bucket_at,
total_calls=orm.total_calls,
success_calls=orm.success_calls,
failed_calls=orm.failed_calls,
auth_failed_calls=orm.auth_failed_calls,
ssrf_blocked_calls=orm.ssrf_blocked_calls,
timeout_calls=orm.timeout_calls,
latency_sum_ms=orm.latency_sum_ms,
latency_count=orm.latency_count,
p99_latency_ms=orm.p99_latency_ms,
token_refresh_count=orm.token_refresh_count,
token_refresh_failed=orm.token_refresh_failed,
throttled_count=orm.throttled_count,
circuit_open_count=orm.circuit_open_count,
)
def orm_to_health_check(orm: ExternalSystemHealthCheck) -> HealthCheck:
"""ORM → dataclass无敏感字段直接映射。"""
return HealthCheck(
id=orm.id,
system_id=orm.system_id,
env_key=orm.env_key,
health_status=orm.health_status,
duration_ms=orm.duration_ms,
detail=orm.detail,
error_code=orm.error_code,
error_message=orm.error_message,
checked_at=orm.checked_at,
created_by=orm.created_by,
updated_by=orm.updated_by,
created_at=orm.created_at,
updated_at=orm.updated_at,
)
def orm_to_webhook_event(orm: ExternalWebhookEvent) -> WebhookEvent:
"""ORM → dataclass无敏感字段直接映射。"""
return WebhookEvent(
id=orm.id,
subscription_id=orm.subscription_id,
subscription_slug=orm.subscription_slug,
event_type=orm.event_type,
payload=orm.payload or {},
received_at=orm.received_at,
external_event_id=orm.external_event_id,
payload_hash=orm.payload_hash,
headers=orm.headers or {},
signature=orm.signature,
signature_verified=orm.signature_verified,
source_ip=orm.source_ip,
processing_status=orm.processing_status,
processing_attempts=orm.processing_attempts,
last_processed_at=orm.last_processed_at,
processing_error=orm.processing_error,
correlation_id=orm.correlation_id,
triggered_handler_type=orm.triggered_handler_type,
triggered_handler_id=orm.triggered_handler_id,
triggered_execution_id=orm.triggered_execution_id,
)
def orm_to_quota_usage(orm: ExternalSystemQuotaUsage) -> QuotaUsage:
"""ORM → dataclass无敏感字段直接映射。"""
return QuotaUsage(
id=orm.id,
system_id=orm.system_id,
quota_key=orm.quota_key,
quota_name=orm.quota_name,
quota_window=orm.quota_window,
env_key=orm.env_key,
window_start=orm.window_start,
window_end=orm.window_end,
limit_value=orm.limit_value,
used_value=orm.used_value,
unit=orm.unit,
source=orm.source,
source_header_name=orm.source_header_name,
source_header_strategy=orm.source_header_strategy,
warning_threshold=orm.warning_threshold,
critical_threshold=orm.critical_threshold,
last_updated_at=orm.last_updated_at,
last_response_at=orm.last_response_at,
)
def orm_to_alert(orm: ExternalSystemAlert) -> Alert:
"""ORM → dataclass无敏感字段直接映射。
``system_name`` / ``system_slug`` 非 ORM 列,由 ``SqlAlchemyAlertRepository.list``
在 JOIN 查询后通过 ``dataclasses.replace`` 填充。
"""
return Alert(
id=orm.id,
system_id=orm.system_id,
alert_type=orm.alert_type,
title=orm.title,
triggered_at=orm.triggered_at,
env_key=orm.env_key,
severity=orm.severity,
status=orm.status,
description=orm.description,
detail=orm.detail or {},
resource_type=orm.resource_type,
resource_id=orm.resource_id,
related_execution_id=orm.related_execution_id,
related_trace_id=orm.related_trace_id,
metric_snapshot=orm.metric_snapshot,
dedup_key=orm.dedup_key,
resolved_at=orm.resolved_at,
acknowledged_by=orm.acknowledged_by,
acknowledged_at=orm.acknowledged_at,
resolution_note=orm.resolution_note,
notification_status=orm.notification_status,
notification_sent_at=orm.notification_sent_at,
notification_channels=orm.notification_channels or {},
created_by=orm.created_by,
updated_by=orm.updated_by,
created_at=orm.created_at,
updated_at=orm.updated_at,
)
def orm_to_secret_rotation_policy(orm: ExternalSecretRotationPolicy) -> SecretRotationPolicy:
"""ORM → dataclass无敏感字段直接映射。
注意:``secret_ref`` 仅是引用 key指向 ``ExternalSystem.secret_refs`` 中的具名 secret
非敏感值本身,不加密。
"""
return SecretRotationPolicy(
id=orm.id,
system_id=orm.system_id,
secret_type=orm.secret_type,
secret_name=orm.secret_name,
secret_ref=orm.secret_ref,
env_key=orm.env_key,
rotation_mode=orm.rotation_mode,
rotation_interval_days=orm.rotation_interval_days,
notify_before_days=orm.notify_before_days,
notify_channels=orm.notify_channels or {},
last_rotated_at=orm.last_rotated_at,
next_rotation_at=orm.next_rotation_at,
last_rotation_at=orm.last_rotation_at,
last_rotation_status=orm.last_rotation_status,
last_rotation_error=orm.last_rotation_error,
rotation_count=orm.rotation_count,
max_rotation_retries=orm.max_rotation_retries,
current_retry_count=orm.current_retry_count,
status=orm.status,
version=orm.version,
created_by=orm.created_by,
updated_by=orm.updated_by,
created_at=orm.created_at,
updated_at=orm.updated_at,
)
def orm_to_external_secret(orm: ExternalSecretORM) -> ExternalSecret:
"""ORM → dataclass无敏感字段直接映射。"""
return ExternalSecret(
id=orm.id,
system_id=orm.system_id,
env_key=orm.env_key,
secret_type=orm.secret_type,
secret_name=orm.secret_name,
secret_ref_key=orm.secret_ref_key,
status=orm.status,
version=orm.version,
created_by=orm.created_by,
updated_by=orm.updated_by,
created_at=orm.created_at,
updated_at=orm.updated_at,
)
def orm_to_external_secret_rotation_record(
orm: ExternalSecretRotationRecordORM,
) -> ExternalSecretRotationRecord:
"""ORM → dataclass无敏感字段直接映射。"""
return ExternalSecretRotationRecord(
id=orm.id,
policy_id=orm.policy_id,
secret_id=orm.secret_id,
attempted_at=orm.attempted_at,
status=orm.status,
mode=orm.mode,
error=orm.error,
performed_by=orm.performed_by,
old_secret_version=orm.old_secret_version,
new_secret_version=orm.new_secret_version,
created_by=orm.created_by,
updated_by=orm.updated_by,
created_at=orm.created_at,
updated_at=orm.updated_at,
)
def orm_to_tool_access_rule(orm: ExternalToolAccessRule) -> ToolAccessRule:
"""ORM → dataclass无敏感字段直接映射。"""
return ToolAccessRule(
id=orm.id,
principal_type=orm.principal_type,
principal_id=orm.principal_id,
created_by=orm.created_by,
tool_slug=orm.tool_slug,
system_id=orm.system_id,
principal_name=orm.principal_name,
env_key=orm.env_key,
effect=orm.effect,
priority=orm.priority,
conditions=orm.conditions or {},
expires_at=orm.expires_at,
enabled=orm.enabled,
description=orm.description,
updated_by=orm.updated_by,
created_at=orm.created_at,
updated_at=orm.updated_at,
)
def orm_to_tool_test_case(orm: ExternalToolTestCase) -> ToolTestCase:
"""ORM → dataclass无敏感字段直接映射。"""
return ToolTestCase(
id=orm.id,
tool_slug=orm.tool_slug,
name=orm.name,
request_fixture=orm.request_fixture or {},
created_by=orm.created_by,
description=orm.description,
env_key=orm.env_key,
expected_status=orm.expected_status,
assertions=orm.assertions or [],
timeout=orm.timeout,
enabled=orm.enabled,
schedule_cron=orm.schedule_cron,
alert_on_failure=orm.alert_on_failure,
alert_threshold=orm.alert_threshold,
last_run_at=orm.last_run_at,
last_run_status=orm.last_run_status,
last_run_duration_ms=orm.last_run_duration_ms,
last_run_response=orm.last_run_response,
last_run_error=orm.last_run_error,
last_run_assertions_result=orm.last_run_assertions_result,
consecutive_failures=orm.consecutive_failures,
updated_by=orm.updated_by,
)
# ─── NotificationChanneldict 型敏感字段config ────────────────────────
def orm_to_notification_channel(orm: ExternalNotificationChannel) -> NotificationChannel:
"""ORM → dataclassdict 型解密 ``config``。
解密失败时抛 ``ValueError``(密钥变更场景),由调用方决定处理策略。
注意:不调用 ``orm.to_dict()``(它只返回 ``has_config`` 布尔字段)。
"""
return NotificationChannel(
id=orm.id,
channel_type=orm.channel_type,
name=orm.name,
config=decrypt_sensitive_fields(orm.config) or {},
enabled=orm.enabled,
description=orm.description,
created_by=orm.created_by,
updated_by=orm.updated_by,
created_at=orm.created_at,
updated_at=orm.updated_at,
)
def encrypt_notification_channel_for_write(data: dict[str, Any]) -> dict[str, Any]:
"""明文 dict → 密文 dictdict 型加密 ``config``。
注意:``encrypt_sensitive_fields`` 不幂等,调用方必须保证传入的是明文(未被加密过)。
"""
data = dict(data)
if data.get("config") is not None:
data["config"] = encrypt_sensitive_fields(data["config"])
return data