Kris
a484844ea1
此提交包含了大量跨模块的优化与修复： 1. 配置与依赖调整：调整账号列表分页默认值、重试上限、内容审核长度限制，新增出站文件支持 2. 代码结构优化：重构白名单缓存逻辑、移除冗余日志依赖、清理过时TODO注释 3. 功能增强：添加死信计数接口、会话绑定审计、SSE订阅限流、凭据验证服务 4. 契约更新：修正DTO字段命名、完善类型定义、更新审计操作类型 5. 错误处理：优化限流清理逻辑、添加异常捕获与告警 6. 文档与校验：补充字段校验规则、完善注释与文档说明
454 lines
19 KiB
Python
454 lines
19 KiB
Python
"""集中式 ORM↔dataclass 转换 + 加解密边界。
|
||
|
||
本模块是 ``channels/adapters/`` 下唯一调用 ``yuxi.utils.crypto`` 的文件，
|
||
负责 ORM Model ↔ 领域 dataclass 的字段映射与敏感字段加解密。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import datetime as dt
|
||
from typing import Any
|
||
|
||
from yuxi.channels.contract.dtos.audit import AuditEntry, AuditOperationType
|
||
from yuxi.channels.contract.dtos.channel import (
|
||
AccountStatus,
|
||
ChannelAccount,
|
||
ChannelSession,
|
||
ChannelType,
|
||
OnboardingStatus,
|
||
UserIdentity,
|
||
)
|
||
from yuxi.channels.contract.dtos.outbox import (
|
||
MessageDurabilityPolicy,
|
||
OutboxEntry,
|
||
OutboxStatus,
|
||
)
|
||
from yuxi.channels.contract.dtos.pairing import PairingRecord, PairingStatus
|
||
from yuxi.channels.contract.dtos.route import RouteBindingRule
|
||
from yuxi.channels.contract.errors import DependencyError
|
||
from yuxi.channels.contract.errors.base import Error
|
||
from yuxi.storage.postgres.models_channels import (
|
||
ChannelAccount as ChannelAccountORM,
|
||
)
|
||
from yuxi.storage.postgres.models_channels import (
|
||
ChannelAuditLog as ChannelAuditLogORM,
|
||
)
|
||
from yuxi.storage.postgres.models_channels import (
|
||
ChannelOutboxEntry as ChannelOutboxEntryORM,
|
||
)
|
||
from yuxi.storage.postgres.models_channels import (
|
||
ChannelPairing as ChannelPairingORM,
|
||
)
|
||
from yuxi.storage.postgres.models_channels import (
|
||
ChannelRouteBinding as ChannelRouteBindingORM,
|
||
)
|
||
from yuxi.storage.postgres.models_channels import (
|
||
ChannelSession as ChannelSessionORM,
|
||
)
|
||
from yuxi.storage.postgres.models_channels import (
|
||
ChannelUserIdentity as ChannelUserIdentityORM,
|
||
)
|
||
from yuxi.utils.crypto import decrypt_sensitive_fields, encrypt_sensitive_fields
|
||
|
||
|
||
def _to_aware_utc(value: dt.datetime | None) -> dt.datetime | None:
|
||
"""将 ORM 读出的 naive UTC datetime 转换为 aware UTC datetime。
|
||
|
||
Postgres ``TIMESTAMP WITHOUT TIME ZONE`` 列由 SQLAlchemy 映射为
|
||
offset-naive datetime；领域层统一使用 ``datetime.now(UTC)`` 等
|
||
offset-aware datetime 运算。本 helper 在 adapter→core 边界处转换，
|
||
避免 ``can't subtract offset-naive and offset-aware datetimes`` 等
|
||
时区混用错误。已是 aware 的值原样返回。
|
||
"""
|
||
if value is None:
|
||
return None
|
||
if value.tzinfo is None:
|
||
return value.replace(tzinfo=dt.UTC)
|
||
return value
|
||
|
||
|
||
def _enum[T, V](value: V, cls: type[T], resource: str) -> T:
|
||
"""安全构造枚举，将非法值翻译为 DependencyError。
|
||
|
||
ORM 字段在数据损坏/迁移残留场景下可能持有非法枚举值，原生
|
||
``Enum(...)`` 构造抛出 ``ValueError`` 会穿透至核心层，违反 INV-7。
|
||
本 helper 捕获 ``ValueError`` 并翻译为契约层 ``DependencyError``，
|
||
保留原始异常链，资源标识用于错误追踪。
|
||
|
||
Args:
|
||
value: ORM 字段原始值。
|
||
cls: 目标枚举类型。
|
||
resource: 资源标识（如 ``"channel_account"``），用于错误信息。
|
||
|
||
Returns:
|
||
构造后的枚举实例。
|
||
|
||
Raises:
|
||
DependencyError: ``value`` 不是 ``cls`` 的合法成员。
|
||
"""
|
||
try:
|
||
return cls(value)
|
||
except ValueError as exc:
|
||
raise DependencyError(
|
||
resource,
|
||
Error(f"invalid enum value for {cls.__name__}: {value!r}"),
|
||
) from exc
|
||
|
||
|
||
# ═══════════════════════════════════════════════════════════════════════════
|
||
# 含敏感字段的 dataclass（读取时解密 / 写入前加密）
|
||
# ═══════════════════════════════════════════════════════════════════════════
|
||
|
||
# ─── ChannelAccount（dict 型敏感字段：config） ─────────────────────────────
|
||
|
||
|
||
def orm_to_channel_account(orm: ChannelAccountORM) -> ChannelAccount:
|
||
"""ORM → dataclass，解密 ``config`` 字段。
|
||
|
||
解密失败时翻译为 ``DependencyError`` 抛出，保留原始异常链
|
||
（密钥变更或数据损坏场景），不让原生 ``ValueError`` 穿透至调用方。
|
||
|
||
存量数据兜底：``onboarding_status`` 为空时默认 ``OnboardingStatus.PENDING``。
|
||
"""
|
||
try:
|
||
config = decrypt_sensitive_fields(orm.config) or {}
|
||
except ValueError as exc:
|
||
raise DependencyError(
|
||
"channel_account",
|
||
Error(f"config decryption failed: {exc}"),
|
||
) from exc
|
||
onboarding_status_raw = orm.onboarding_status or OnboardingStatus.PENDING.value
|
||
return ChannelAccount(
|
||
channel_type=ChannelType(orm.channel_type),
|
||
account_id=orm.account_id,
|
||
display_name=orm.display_name,
|
||
config=config,
|
||
enabled=bool(orm.enabled),
|
||
status=_enum(orm.status, AccountStatus, "channel_account"),
|
||
created_at=orm.created_at,
|
||
updated_at=orm.updated_at,
|
||
transport_cursor=orm.transport_cursor or "",
|
||
transport_mode=orm.transport_mode or "both",
|
||
last_rotated_at=orm.last_rotated_at,
|
||
service_user_uid=orm.service_user_uid,
|
||
onboarding_status=_enum(onboarding_status_raw, OnboardingStatus, "channel_account"),
|
||
credential_ref=orm.credential_ref,
|
||
credential_version=orm.credential_version,
|
||
last_error=orm.last_error,
|
||
last_error_at=orm.last_error_at,
|
||
last_health_check_at=orm.last_health_check_at,
|
||
plugin_status=orm.plugin_status,
|
||
version=orm.version,
|
||
created_by=orm.created_by,
|
||
updated_by=orm.updated_by,
|
||
)
|
||
|
||
|
||
def channel_account_for_write(data: dict[str, Any]) -> dict[str, Any]:
|
||
"""明文 dict → 密文 dict，加密 ``config`` 字段。
|
||
|
||
注意：``encrypt_sensitive_fields`` 不幂等，调用方必须保证传入的是明文（未被加密过）。
|
||
``onboarding_status`` 为 ``OnboardingStatus`` 枚举时转为 ``.value`` 字符串以写入 ORM
|
||
（``credential_ref`` / ``credential_version`` 为原生类型，随 ``dict(data)`` 透传）。
|
||
"""
|
||
data = dict(data)
|
||
if data.get("config") is not None:
|
||
data["config"] = encrypt_sensitive_fields(data["config"])
|
||
onboarding_status = data.get("onboarding_status")
|
||
if isinstance(onboarding_status, OnboardingStatus):
|
||
data["onboarding_status"] = onboarding_status.value
|
||
return data
|
||
|
||
|
||
# ═══════════════════════════════════════════════════════════════════════════
|
||
# 无敏感字段的 dataclass（直接字段映射）
|
||
# ═══════════════════════════════════════════════════════════════════════════
|
||
|
||
# ─── ChannelSession（无敏感字段） ───────────────────────────────────────────
|
||
|
||
|
||
def orm_to_channel_session(
|
||
orm: ChannelSessionORM,
|
||
account_orm: ChannelAccountORM,
|
||
) -> ChannelSession:
|
||
"""ORM → dataclass，无敏感字段，直接映射。
|
||
|
||
``account_id`` / ``conversation_id`` 为 ORM 外键（int）。``conversation_id``
|
||
转为 str 业务 ID。``account_id`` 必须由 ``account_orm`` 关联填充为业务标识
|
||
字符串（``channel_accounts.account_id``），而非 ORM 外键 int 的字符串形式，
|
||
保证全栈 ``account_id`` 语义统一为业务标识。
|
||
|
||
参数：
|
||
orm: 渠道会话 ORM 实例。
|
||
account_orm: 关联的渠道账户 ORM 实例（必填，由适配器层查询填充）。
|
||
"""
|
||
return ChannelSession(
|
||
session_id=orm.session_id,
|
||
channel_type=ChannelType(orm.channel_type),
|
||
account_id=account_orm.account_id,
|
||
peer_id=orm.peer_id,
|
||
chat_type=orm.chat_type,
|
||
conversation_id=str(orm.conversation_id) if orm.conversation_id is not None else None,
|
||
unified_identity_id=orm.unified_identity_id,
|
||
owner_peer_id=orm.owner_peer_id,
|
||
is_temporary=bool(orm.is_temporary),
|
||
created_at=orm.created_at,
|
||
updated_at=orm.updated_at,
|
||
deleted_at=orm.deleted_at,
|
||
closed_at=orm.closed_at,
|
||
last_message_at=orm.last_message_at,
|
||
last_route_at=orm.last_route_at,
|
||
route_match_source=orm.route_match_source,
|
||
version=orm.version,
|
||
)
|
||
|
||
|
||
def channel_session_for_write(data: dict[str, Any]) -> dict[str, Any]:
|
||
"""明文 dict → 密文 dict，无敏感字段，直接透传。"""
|
||
return dict(data)
|
||
|
||
|
||
# ─── PairingRecord（无敏感字段） ────────────────────────────────────────────
|
||
|
||
|
||
def orm_to_pairing(
|
||
orm: ChannelPairingORM,
|
||
account_orm: ChannelAccountORM | None = None,
|
||
) -> PairingRecord:
|
||
"""ORM → dataclass，无敏感字段，直接映射。
|
||
|
||
``account_orm`` 非空时从 ``channel_accounts`` 表关联填充业务
|
||
``channel_account_id``（字符串业务标识）与 ``channel_type``；为 ``None``
|
||
时 ``channel_account_id`` 回退为 ORM 外键的字符串形式（向后兼容），
|
||
``channel_type`` 保持 ``None``。
|
||
|
||
参数：
|
||
orm: 配对审批 ORM 实例。
|
||
account_orm: 关联的渠道账户 ORM 实例（可选，由适配器层查询填充）。
|
||
"""
|
||
return PairingRecord(
|
||
pairing_id=orm.pairing_id,
|
||
channel_account_id=(account_orm.account_id if account_orm is not None else str(orm.account_id)),
|
||
peer_id=orm.peer_id,
|
||
status=_enum(orm.status, PairingStatus, "pairing"),
|
||
channel_type=(ChannelType(account_orm.channel_type) if account_orm is not None else None),
|
||
peer_name=orm.peer_name,
|
||
approver_id=orm.approver_id,
|
||
approved_at=orm.approved_at,
|
||
rejected_at=orm.rejected_at,
|
||
expired_at=orm.expired_at,
|
||
revoked_at=orm.revoked_at,
|
||
reason=orm.reason,
|
||
created_at=orm.created_at,
|
||
updated_at=orm.updated_at,
|
||
expires_at=orm.expires_at,
|
||
requested_at=orm.requested_at,
|
||
version=orm.version,
|
||
)
|
||
|
||
|
||
def pairing_for_write(data: dict[str, Any]) -> dict[str, Any]:
|
||
"""明文 dict → 密文 dict，无敏感字段，直接透传。"""
|
||
return dict(data)
|
||
|
||
|
||
# ─── AuditEntry（无敏感字段，脱敏由上层完成） ───────────────────────────────
|
||
|
||
|
||
def orm_to_audit_log(orm: ChannelAuditLogORM) -> AuditEntry:
|
||
"""ORM → dataclass，无敏感字段，直接映射。
|
||
|
||
``params_summary`` 脱敏由上层 ``AuditLog`` 聚合根完成，mapper 不再脱敏。
|
||
``target`` 独立持久化列，直接映射；``target_channel`` 映射到渠道作用域字段。
|
||
``id`` 为 ORM 主键，供 keyset pagination 计算游标使用。
|
||
"""
|
||
return AuditEntry(
|
||
operator=orm.operator,
|
||
operation=_enum(orm.operation, AuditOperationType, "audit_log"),
|
||
target=orm.target if orm.target is not None else orm.target_channel,
|
||
result=orm.result,
|
||
timestamp=orm.timestamp,
|
||
params_summary=orm.params_summary or None,
|
||
trace_id=orm.trace_id,
|
||
source_ip=orm.source_ip,
|
||
request_id=orm.request_id,
|
||
message_id=getattr(orm, "message_id", None),
|
||
content_summary=getattr(orm, "content_summary", None),
|
||
target_channel=orm.target_channel,
|
||
target_account=orm.target_account,
|
||
id=getattr(orm, "id", None),
|
||
)
|
||
|
||
|
||
def _serialize_for_json(value: Any) -> Any:
|
||
"""递归将 datetime/date 对象转为 ISO 字符串，保证 JSONB 可序列化。"""
|
||
if isinstance(value, dt.datetime):
|
||
return value.isoformat()
|
||
if isinstance(value, dt.date):
|
||
return value.isoformat()
|
||
if isinstance(value, dict):
|
||
return {k: _serialize_for_json(v) for k, v in value.items()}
|
||
if isinstance(value, list):
|
||
return [_serialize_for_json(v) for v in value]
|
||
if isinstance(value, tuple):
|
||
return tuple(_serialize_for_json(v) for v in value)
|
||
return value
|
||
|
||
|
||
def audit_log_for_write(data: dict[str, Any]) -> dict[str, Any]:
|
||
"""明文 dict → 密文 dict，无敏感字段。
|
||
|
||
``target`` 字段独立持久化到 ORM ``target`` 列，不再回退填充
|
||
``target_channel``（渠道作用域由上层 cmd 单独提供）。
|
||
|
||
``params_summary`` 中可能混入 datetime/date 对象（如前端传入的时间过滤
|
||
参数或后端生成的字段），在写入 JSONB 前递归转为 ISO 字符串，避免
|
||
``Object of type datetime is not JSON serializable`` 导致事务回滚。
|
||
"""
|
||
data = dict(data)
|
||
if "params_summary" in data:
|
||
data["params_summary"] = _serialize_for_json(data["params_summary"])
|
||
return data
|
||
|
||
|
||
# ─── OutboxEntry（无敏感字段） ───────────────────────────────────────────────
|
||
|
||
|
||
def orm_to_outbox_entry(
|
||
orm: ChannelOutboxEntryORM,
|
||
account_orm: ChannelAccountORM,
|
||
) -> OutboxEntry:
|
||
"""ORM → dataclass，无敏感字段，直接映射。
|
||
|
||
``message_id`` / ``channel_session_id`` 为 ORM 外键（int，
|
||
``channel_session_id`` 可空），转为 str 业务 ID。``account_id`` 必须由
|
||
``account_orm`` 关联填充为业务标识字符串（``channel_accounts.account_id``），
|
||
而非 ORM 外键 int 的字符串形式，保证全栈 ``channel_account_id`` 语义
|
||
统一为业务标识。``channel_type`` 从 ``account_orm.channel_type`` 派生。
|
||
``max_retry`` / ``expires_at`` 从 ORM 列直接映射，消除补偿阶段硬编码
|
||
（P2-7）。
|
||
|
||
参数：
|
||
orm: 发件箱条目 ORM 实例。
|
||
account_orm: 关联的渠道账户 ORM 实例（必填，由适配器层查询填充）。
|
||
"""
|
||
return OutboxEntry(
|
||
outbox_id=orm.outbox_id,
|
||
message_id=str(orm.message_id),
|
||
channel_account_id=account_orm.account_id,
|
||
status=_enum(orm.status, OutboxStatus, "outbox_entry"),
|
||
durability_policy=_enum(orm.durability_policy, MessageDurabilityPolicy, "outbox_entry"),
|
||
retry_count=orm.retry_count,
|
||
max_retry=orm.max_retry,
|
||
next_retry_at=_to_aware_utc(orm.next_retry_at),
|
||
last_error=orm.last_error,
|
||
created_at=_to_aware_utc(orm.created_at),
|
||
updated_at=_to_aware_utc(orm.updated_at),
|
||
expires_at=_to_aware_utc(orm.expires_at),
|
||
channel_msg_id=orm.channel_msg_id,
|
||
version=orm.version,
|
||
channel_session_id=str(orm.channel_session_id) if orm.channel_session_id is not None else None,
|
||
# 聚合视图域 analytics 子域字段（延迟分布 / 漏斗聚合查询）
|
||
latency_ms=orm.latency_ms,
|
||
funnel_node=orm.funnel_node,
|
||
sent_at=_to_aware_utc(orm.sent_at),
|
||
last_retry_at=_to_aware_utc(orm.last_retry_at),
|
||
channel_type=ChannelType(account_orm.channel_type),
|
||
# 投递原子语义与降级追踪字段（O-02/O-09/O-10/O-11/O-16）
|
||
idempotency_key=orm.idempotency_key,
|
||
channel_request_id=orm.channel_request_id,
|
||
partial_failure=orm.partial_failure,
|
||
stream_aborted_at_chunk=orm.stream_aborted_at_chunk,
|
||
degraded_reason=orm.degraded_reason,
|
||
delivered_parts=list(orm.delivered_parts) if orm.delivered_parts else [],
|
||
)
|
||
|
||
|
||
def outbox_entry_for_write(data: dict[str, Any]) -> dict[str, Any]:
|
||
"""明文 dict → 密文 dict，无敏感字段，直接透传。
|
||
|
||
``latency_ms`` / ``funnel_node`` 为聚合视图域 analytics 子域新增字段，
|
||
由调用方按需写入 dict；本函数透传至 ORM 写入层，未提供时由 ORM
|
||
默认 ``None`` 兜底（向后兼容历史数据）。
|
||
"""
|
||
return dict(data)
|
||
|
||
|
||
# ─── UserIdentity（无敏感字段） ─────────────────────────────────────────────
|
||
|
||
|
||
def orm_to_user_identity(orm: ChannelUserIdentityORM) -> UserIdentity:
|
||
"""ORM → dataclass，无敏感字段，直接映射。
|
||
|
||
``user_id`` 为 ORM 外键（int，可空），转为 str 业务 ID。
|
||
``channel_type`` 为可空字符串，转为 ``ChannelType`` 枚举或 None。
|
||
``channel_bindings`` / ``merged_from`` 为 JSON 列，默认空容器兜底。
|
||
"""
|
||
return UserIdentity(
|
||
identity_id=orm.identity_id,
|
||
user_id=str(orm.user_id) if orm.user_id is not None else None,
|
||
identity_type=orm.identity_type,
|
||
identity_value=orm.identity_value,
|
||
channel_type=ChannelType(orm.channel_type) if orm.channel_type is not None else None,
|
||
channel_sender_id=orm.channel_sender_id,
|
||
source=orm.source,
|
||
channel_bindings=orm.channel_bindings or {},
|
||
merged_from=orm.merged_from or [],
|
||
confidence=orm.confidence or "low",
|
||
created_at=orm.created_at,
|
||
updated_at=orm.updated_at,
|
||
version=orm.version,
|
||
pending_review=bool(getattr(orm, "pending_review", False)),
|
||
)
|
||
|
||
|
||
def user_identity_for_write(data: dict[str, Any]) -> dict[str, Any]:
|
||
"""明文 dict → 密文 dict，无敏感字段，直接透传。"""
|
||
return dict(data)
|
||
|
||
|
||
# ─── WhitelistEntry ─────────────────────────────────────────────────────────
|
||
# 注：whitelistEntryToDict / whitelistEntryFromDict 已上移至
|
||
# contract/dtos/whitelist.py（纯 DTO 转换，不依赖 ORM，供应用层共用）
|
||
|
||
|
||
# ─── RouteBindingRule（无敏感字段） ─────────────────────────────────────────
|
||
|
||
|
||
#: 内置匹配层级优先级（与 factory._DEFAULT_MATCH_TIERS 保持一致，仅展示用）
|
||
_TIER_PRIORITY_MAP: dict[str, int] = {
|
||
"session_key": 800,
|
||
"identity_id": 700,
|
||
"peer_id": 600,
|
||
"chat_type": 500,
|
||
"channel_session": 400,
|
||
"channel_type": 300,
|
||
"account": 200,
|
||
"default": 100,
|
||
}
|
||
|
||
|
||
def orm_to_route_binding(orm: ChannelRouteBindingORM) -> RouteBindingRule:
|
||
"""ORM → dataclass，无敏感字段，直接映射。
|
||
|
||
``match_source`` 为 tier 名字符串（非枚举），直接透传；
|
||
``channel_type`` 转为 ``ChannelType`` 枚举；
|
||
``priority`` 根据 ``match_source`` 从内置层级优先级派生，供前端展示。
|
||
"""
|
||
return RouteBindingRule(
|
||
binding_id=orm.binding_id,
|
||
channel_type=ChannelType(orm.channel_type),
|
||
account_id=orm.account_id,
|
||
match_source=orm.match_source,
|
||
match_value=orm.match_value,
|
||
agent_binding=orm.agent_binding,
|
||
enabled=bool(orm.enabled),
|
||
description=orm.description,
|
||
version=orm.version,
|
||
priority=_TIER_PRIORITY_MAP.get(orm.match_source),
|
||
created_by=orm.created_by,
|
||
updated_by=orm.updated_by,
|
||
created_at=orm.created_at,
|
||
updated_at=orm.updated_at,
|
||
)
|