ForcePilot/backend/server/routers/external_systems/import_router.py
Kris 6dd16ad3f8 refactor(external-systems-routers): 统一分页参数格式并完善各路由文档与校验
本次提交对多个外部系统路由进行了多维度优化:
1.  统一分页参数:将所有路由的`page = offset//limit +1`、`page_size=limit`替换为标准的`limit`+`offset`分页格式
2.  完善接口文档:补充多个端点的功能说明、参数含义与返回字段解释
3.  增强参数校验:新增字段长度限制、正则校验、枚举类型约束与业务逻辑校验
4.  优化代码复用:提取重复逻辑为辅助函数,减少样板代码
5.  修复接口问题:修正工具健康检查端点路径参数类型,优化导出接口响应格式
6.  补充异常处理:为批量操作添加异常捕获与日志记录,避免流程中断
2026-07-11 06:57:24 +08:00

564 lines
22 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""Import 子域 Router。
外部系统限界上下文的导入/导出 API覆盖以下能力
系统级导入(两阶段流程):
1. ``POST /preview``:根据适配器类型与源资产生成系统级导入草稿(系统 + 工具)
2. ``POST /confirm``:基于草稿执行实际持久化
工具级导入(两阶段流程):
3. ``POST /tools/preview``:根据适配器类型与源资产生成工具草稿列表(不落库)
4. ``POST /tools/confirm``:基于草稿执行工具持久化
工具导出:
5. ``GET /tools/export``:导出单个工具配置
6. ``POST /tools/bulk-export``:批量导出工具配置
工具包导入/导出:
7. ``POST /tool-package/export``:导出工具包(含工具 + 系统 + 环境 + 资产)
8. ``POST /tool-package/preview``:工具包导入预览(检查冲突,不落库)
9. ``POST /tool-package/confirm``:工具包导入确认(导入系统/环境/资产/工具)
所有端点通过 ``create_use_cases_from_db`` 装配 use_cases经 ``import_service``
端口调用用例。Request Schema 与 Input DTO 不共享类Router 内显式构造 DTO
操作人字段(``created_by``)由 ``current_user.uid`` 填充,角色字段
``requester_role``)由 ``current_user.role`` 填充。
字段约束对齐 ``ExternalSystem`` / ``ExternalTool`` ORM 列定义与其他子域 router
``system_router`` / ``tool_router`` / ``asset_router``),在边界层拦截非法输入。
``include_secrets=True`` 的导出端点在 Router 层显式校验 superadmin 权限,
``AccessDeniedError`` 交由全局异常处理器映射为 403。
"""
from __future__ import annotations
import base64
import binascii
from typing import Any
from fastapi import APIRouter, Depends, Query
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
from sqlalchemy.ext.asyncio import AsyncSession
from yuxi.external_systems.exceptions import AccessDeniedError
from yuxi.external_systems.infrastructure.container import create_use_cases_from_db
from yuxi.external_systems.use_cases.dto.environment import EnvironmentOutput
from yuxi.external_systems.use_cases.dto.import_draft import (
BulkExportToolsInput,
ExportToolInput,
ImportConfirmInput,
ImportPreviewInput,
ToolImportConfirmInput,
ToolImportPreviewInput,
ToolPackageAssetInput,
ToolPackageExportInput,
ToolPackageImportInput,
ToolPackageImportPreviewInput,
ToolPackageInput,
ToolPackageItemInput,
)
from yuxi.external_systems.use_cases.dto.system import (
ExternalSystemCreateInput,
SystemImportDraft,
SystemOutput,
)
from yuxi.external_systems.use_cases.dto.tool import (
ExternalToolCreateInput,
ToolOutput,
)
from yuxi.storage.postgres.models_business import User
from server.utils.auth_middleware import get_admin_user, get_db
import_router = APIRouter(prefix="/imports", tags=["external-systems-import"])
# base64 字符串最大长度≈10MB 二进制,含 33% 编码开销 + padding 余量)
# 对齐 asset_router._MAX_BASE64_LENGTH保证工具包资产与单资产上传同一上限
_MAX_BASE64_LENGTH = 14_000_000
# slug 格式正则,对齐 ExternalSystemCreateInput / ExternalToolCreateInput 的 pattern
_SLUG_PATTERN = r"^[a-zA-Z_][a-zA-Z0-9_-]{0,127}$"
# 批量操作的最大列表长度,对齐 system_router.BatchEnabledRequest.max_length
_MAX_BATCH_LENGTH = 100
def _validate_base64_content(value: str) -> str:
"""校验 base64 编码字符串,非法时抛 ValueError由 Pydantic 映射为 422
对齐 ``asset_router._validate_base64_content``,保证工具包资产与单资产上传
同一校验标准。
"""
try:
base64.b64decode(value, validate=True)
except (binascii.Error, ValueError) as exc:
raise ValueError("content_b64 必须是合法的 base64 编码") from exc
return value
def _check_secrets_export_permission(include_secrets: bool, user: User) -> None:
"""校验导出敏感信息的权限。
``include_secrets=True`` 时要求 superadmin 角色,否则抛 ``AccessDeniedError``。
在 Router 层显式校验,避免 admin 请求穿透到 use_case 层才被拒绝。
"""
if include_secrets and user.role != "superadmin":
raise AccessDeniedError("导出敏感信息需要超级管理员权限")
# ---------------- Request Schemas ----------------
class ImportPreviewRequest(BaseModel):
"""导入预览请求体。字段对齐 ``ImportPreviewInput``。
``payload`` 类型为 ``Any``,透传给适配器 ``generate_from_asset``
不同适配器期望不同结构dict/list/str与 ``ToolImportPreviewRequest``
及 DTO ``ImportPreviewInput`` 保持一致。
字段长度约束对齐 ``ExternalSystem`` ORM 列定义与 ``system_router``
``adapter_type`` 对齐 String(32)、``source_type`` 对齐 String(64)。
"""
model_config = ConfigDict(frozen=True)
adapter_type: str = Field(..., max_length=32)
source_type: str = Field(..., max_length=64)
payload: Any
asset_id: int | None = Field(default=None, ge=1)
class ExternalSystemDraftRequest(BaseModel):
"""系统级导入草稿中的系统请求体。字段对齐 ``ExternalSystemCreateInput``。"""
model_config = ConfigDict(frozen=True)
slug: str = Field(..., min_length=1, max_length=128, pattern=r"^[a-zA-Z_][a-zA-Z0-9_-]{0,127}$")
name: str = Field(..., min_length=1, max_length=128)
description: str = Field(..., min_length=1)
category: str = Field(default="default", max_length=64)
adapter_type: str = Field(default="http", max_length=32)
source_type: str | None = Field(default=None, max_length=64)
enabled: bool = Field(default=True)
connection_config: dict[str, Any] = Field(default_factory=dict)
auth_type: str = Field(default="none", max_length=32)
auth_config: dict[str, Any] | None = Field(default=None)
secret_refs: dict[str, Any] = Field(default_factory=dict)
class ExternalToolDraftRequest(BaseModel):
"""系统级导入草稿中的工具请求体。字段对齐 ``ExternalToolCreateInput``。"""
model_config = ConfigDict(frozen=True)
slug: str = Field(..., min_length=1, max_length=128, pattern=r"^[a-zA-Z_][a-zA-Z0-9_-]{0,127}$")
name: str = Field(..., min_length=1, max_length=128)
description: str = Field(..., min_length=1)
category: str = Field(default="default", max_length=64)
adapter_type: str = Field(default="http", max_length=32)
enabled: bool = Field(default=True)
timeout: int = Field(default=30, ge=1, le=300)
retry_policy: dict[str, Any] = Field(default_factory=dict)
auth_type: str = Field(default="none", max_length=32)
auth_config: dict[str, Any] = Field(default_factory=dict)
adapter_config: dict[str, Any] = Field(default_factory=dict)
class SystemImportDraftRequest(BaseModel):
"""系统导入草稿请求体。字段对齐 ``SystemImportDraft``system + tools"""
model_config = ConfigDict(frozen=True)
system: ExternalSystemDraftRequest
tools: list[ExternalToolDraftRequest] = Field(default_factory=list)
class ImportConfirmRequest(BaseModel):
"""导入确认请求体。``draft`` 对齐 ``SystemImportDraft`` 结构system + tools
``created_by`` 由 Router 用 ``current_user.uid`` 填充,不暴露给客户端。
"""
model_config = ConfigDict(frozen=True)
draft: SystemImportDraftRequest
# ---------------- 工具级导入 Request Schemas ----------------
class ToolImportPreviewRequest(BaseModel):
"""工具级导入预览请求体。字段对齐 ``ToolImportPreviewInput``。
字段长度约束对齐 ``ImportPreviewRequest``。
"""
model_config = ConfigDict(frozen=True)
adapter_type: str = Field(..., max_length=32)
source_type: str = Field(..., max_length=64)
payload: Any
class ToolImportConfirmRequest(BaseModel):
"""工具级导入确认请求体。字段对齐 ``ToolImportConfirmInput``。
``created_by`` 由 Router 用 ``current_user.uid`` 填充,不暴露给客户端。
支持两种输入模式(二选一):
- ``drafts`` 非空:前端回传预览阶段获取的完整草稿列表
- ``adapter_type`` + ``source_type`` + ``payload``:让后端通过适配器重新生成草稿
通过 ``model_validator`` 在 Schema 层强制校验输入完整性,避免空 body
穿透到 use_case 层才报错。
"""
model_config = ConfigDict(frozen=True)
adapter_type: str = Field(default="", max_length=32)
source_type: str = Field(default="", max_length=64)
payload: Any = None
drafts: list[dict[str, Any]] = Field(default_factory=list)
selected_slugs: list[str] = Field(default_factory=list)
override_existing: bool = False
@model_validator(mode="after")
def validate_input_source(self) -> ToolImportConfirmRequest:
"""校验输入源完整性:要么提供 drafts要么提供适配器三件套。"""
if self.drafts:
return self
if self.adapter_type and self.source_type and self.payload is not None:
return self
raise ValueError("必须提供 drafts前端回传草稿或 adapter_type + source_type + payload适配器重新生成")
# ---------------- 工具导出 Request Schemas ----------------
class BulkExportToolsRequest(BaseModel):
"""批量工具导出请求体。字段对齐 ``BulkExportToolsInput``。
``requester_role`` 由 Router 用 ``current_user.role`` 填充,不暴露给客户端。
``slugs`` 限制最多 ``_MAX_BATCH_LENGTH`` 个,防止批量查询过多导致性能问题。
"""
model_config = ConfigDict(frozen=True)
slugs: list[str] = Field(..., min_length=1, max_length=_MAX_BATCH_LENGTH)
include_secrets: bool = False
# ---------------- 工具包 Request Schemas ----------------
class ToolPackageAssetRequest(BaseModel):
"""工具包资产请求体。字段对齐 ``ToolPackageAssetInput``。
字段长度约束对齐 ``asset_router.CreateAssetRequest``
``adapter_type`` / ``asset_type`` 对齐 String(32)、``name`` 对齐 String(128)。
``content_b64`` 限制最大长度并校验 base64 格式,与单资产上传同一标准。
``size`` 与 ``checksum`` 保留字段(导出端返回,导入端前端回传),
但 use_case 层会根据 ``content_b64`` 解码后的实际内容重新计算,
不信任客户端传入的值,防止数据不一致。
"""
model_config = ConfigDict(frozen=True)
id: int | None = Field(default=None, ge=1)
adapter_type: str = Field(..., max_length=32)
asset_type: str = Field(..., max_length=32)
name: str = Field(..., min_length=1, max_length=128)
content_b64: str = Field(default="", max_length=_MAX_BASE64_LENGTH)
size: int = Field(default=0, ge=0)
checksum: str | None = None
@field_validator("content_b64")
@classmethod
def validate_content_b64(cls, v: str) -> str:
if not v:
return v
return _validate_base64_content(v)
class ToolPackageItemRequest(BaseModel):
"""工具包条目请求体。字段对齐 ``ToolPackageItemInput``。
``tool``/``system``/``environments`` 使用 ``dict[str, Any]`` 而非显式 Schema
其结构对齐 Output DTO``ToolOutput``/``SystemOutput``/``EnvironmentOutput``
字段多且包含元数据id 等),工具包导入为"导出-导入"闭环,前端回传的就是
导出端返回的 dict。定义 Request Schema 副本会造成大量字段重复且无业务价值,
结构校验由 ``_build_tool_package`` 中的 ``ToolOutput(**item.tool)`` 等
Pydantic 构造完成。``assets`` 使用显式 Schema 是因为 ``ToolPackageAssetInput``
是专为导入定义的 Input DTO含 ``content_b64``),字段少且独立。
"""
model_config = ConfigDict(frozen=True)
tool: dict[str, Any]
system: dict[str, Any] | None = None
environments: list[dict[str, Any]] = Field(default_factory=list)
assets: list[ToolPackageAssetRequest] = Field(default_factory=list)
class ToolPackageRequest(BaseModel):
"""工具包请求体。字段对齐 ``ToolPackageInput``。"""
model_config = ConfigDict(frozen=True)
items: list[ToolPackageItemRequest] = Field(default_factory=list)
class ToolPackageExportRequest(BaseModel):
"""工具包导出请求体。字段对齐 ``ToolPackageExportInput``。
``requester_role`` 由 Router 用 ``current_user.role`` 填充,不暴露给客户端。
``slugs`` 限制最多 ``_MAX_BATCH_LENGTH`` 个,与 ``BulkExportToolsRequest`` 一致。
"""
model_config = ConfigDict(frozen=True)
slugs: list[str] = Field(..., min_length=1, max_length=_MAX_BATCH_LENGTH)
include_system: bool = True
include_assets: bool = True
include_secrets: bool = False
class ToolPackageImportPreviewRequest(BaseModel):
"""工具包导入预览请求体。字段对齐 ``ToolPackageImportPreviewInput``。"""
model_config = ConfigDict(frozen=True)
package: ToolPackageRequest
class ToolPackageImportConfirmRequest(BaseModel):
"""工具包导入确认请求体。字段对齐 ``ToolPackageImportInput``。
``created_by`` 由 Router 用 ``current_user.uid`` 填充,不暴露给客户端。
``target_system_slug`` 校验 slug 格式,防止非法字符注入。
"""
model_config = ConfigDict(frozen=True)
package: ToolPackageRequest
override_existing: bool = False
target_system_slug: str | None = Field(default=None, pattern=_SLUG_PATTERN)
# ---------------- Endpoints ----------------
def _build_tool_package(package: ToolPackageRequest) -> ToolPackageInput:
"""将 Request Schema ``ToolPackageRequest`` 转换为 Input DTO ``ToolPackageInput``。
显式构造嵌套 DTO``ToolPackageItemInput`` / ``ToolOutput`` / ``SystemOutput`` /
``EnvironmentOutput`` / ``ToolPackageAssetInput``),保持 Router 层 Schema ↔ DTO
不共享类的边界。
"""
return ToolPackageInput(
items=[
ToolPackageItemInput(
tool=ToolOutput(**item.tool),
system=SystemOutput(**item.system) if item.system else None,
environments=[EnvironmentOutput(**env) for env in item.environments],
assets=[
ToolPackageAssetInput(
id=asset.id,
adapter_type=asset.adapter_type,
asset_type=asset.asset_type,
name=asset.name,
content_b64=asset.content_b64,
size=asset.size,
checksum=asset.checksum,
)
for asset in item.assets
],
)
for item in package.items
]
)
@import_router.post("/preview", response_model=dict)
async def preview_import(
payload: ImportPreviewRequest,
db: AsyncSession = Depends(get_db),
current_user: User = Depends(get_admin_user),
) -> dict[str, Any]:
"""生成系统级导入草稿(系统 + 工具),不执行持久化。"""
use_cases = create_use_cases_from_db(db)
input_dto = ImportPreviewInput(
adapter_type=payload.adapter_type,
source_type=payload.source_type,
payload=payload.payload,
asset_id=payload.asset_id,
)
output = await use_cases.import_service.preview_import(input_dto)
return {"success": True, "data": output.model_dump()}
@import_router.post("/confirm", response_model=dict)
async def confirm_import(
payload: ImportConfirmRequest,
db: AsyncSession = Depends(get_db),
current_user: User = Depends(get_admin_user),
) -> dict[str, Any]:
"""基于草稿执行系统级导入持久化。``created_by`` 取自当前管理员。"""
use_cases = create_use_cases_from_db(db)
draft = SystemImportDraft(
system=ExternalSystemCreateInput(**payload.draft.system.model_dump()),
tools=[ExternalToolCreateInput(**tool.model_dump()) for tool in payload.draft.tools],
)
input_dto = ImportConfirmInput(draft=draft, created_by=current_user.uid)
output = await use_cases.import_service.confirm_import(input_dto)
return {"success": True, "data": output.model_dump()}
# ---------------- 工具级导入端点 ----------------
@import_router.post("/tools/preview", response_model=dict)
async def preview_tool_import(
payload: ToolImportPreviewRequest,
db: AsyncSession = Depends(get_db),
current_user: User = Depends(get_admin_user),
) -> dict[str, Any]:
"""工具级导入预览:根据适配器类型与源资产生成工具草稿列表(不落库)。"""
use_cases = create_use_cases_from_db(db)
input_dto = ToolImportPreviewInput(
adapter_type=payload.adapter_type,
source_type=payload.source_type,
payload=payload.payload,
)
output = await use_cases.import_service.preview_tool_import(input_dto)
return {"success": True, "data": output.model_dump()}
@import_router.post("/tools/confirm", response_model=dict)
async def confirm_tool_import(
payload: ToolImportConfirmRequest,
db: AsyncSession = Depends(get_db),
current_user: User = Depends(get_admin_user),
) -> dict[str, Any]:
"""工具级导入确认:基于草稿执行工具持久化。``created_by`` 取自当前管理员。"""
use_cases = create_use_cases_from_db(db)
input_dto = ToolImportConfirmInput(
adapter_type=payload.adapter_type,
source_type=payload.source_type,
payload=payload.payload,
drafts=payload.drafts,
selected_slugs=payload.selected_slugs,
override_existing=payload.override_existing,
created_by=current_user.uid,
)
output = await use_cases.import_service.confirm_tool_import(input_dto)
return {"success": True, "data": output.model_dump()}
# ---------------- 工具导出端点 ----------------
@import_router.get("/tools/export", response_model=dict)
async def export_tool(
slug: str = Query(..., min_length=1, max_length=128, pattern=_SLUG_PATTERN),
include_secrets: bool = Query(False),
db: AsyncSession = Depends(get_db),
current_user: User = Depends(get_admin_user),
) -> dict[str, Any]:
"""导出单个工具配置。``requester_role`` 取自当前用户角色。
``include_secrets=True`` 时在 Router 层校验 superadmin 权限,
避免 admin 请求穿透到 use_case 层才被拒绝。
"""
_check_secrets_export_permission(include_secrets, current_user)
use_cases = create_use_cases_from_db(db)
input_dto = ExportToolInput(
slug=slug,
include_secrets=include_secrets,
requester_role=current_user.role,
)
output = await use_cases.import_service.export_tool(input_dto)
return {"success": True, "data": output.model_dump()}
@import_router.post("/tools/bulk-export", response_model=dict)
async def bulk_export_tools(
payload: BulkExportToolsRequest,
db: AsyncSession = Depends(get_db),
current_user: User = Depends(get_admin_user),
) -> dict[str, Any]:
"""批量导出工具配置。``requester_role`` 取自当前用户角色。
``include_secrets=True`` 时在 Router 层校验 superadmin 权限,
避免 admin 请求穿透到 use_case 层才被拒绝。
"""
_check_secrets_export_permission(payload.include_secrets, current_user)
use_cases = create_use_cases_from_db(db)
input_dto = BulkExportToolsInput(
slugs=payload.slugs,
include_secrets=payload.include_secrets,
requester_role=current_user.role,
)
output = await use_cases.import_service.bulk_export_tools(input_dto)
return {"success": True, "data": output.model_dump()}
# ---------------- 工具包导入导出端点 ----------------
@import_router.post("/tool-package/export", response_model=dict)
async def export_tool_package(
payload: ToolPackageExportRequest,
db: AsyncSession = Depends(get_db),
current_user: User = Depends(get_admin_user),
) -> dict[str, Any]:
"""导出工具包(含工具 + 系统 + 环境 + 资产)。``requester_role`` 取自当前用户角色。
``include_secrets=True`` 时在 Router 层校验 superadmin 权限,
避免 admin 请求穿透到 use_case 层才被拒绝。
"""
_check_secrets_export_permission(payload.include_secrets, current_user)
use_cases = create_use_cases_from_db(db)
input_dto = ToolPackageExportInput(
slugs=payload.slugs,
include_system=payload.include_system,
include_assets=payload.include_assets,
include_secrets=payload.include_secrets,
requester_role=current_user.role,
)
output = await use_cases.import_service.export_tool_package(input_dto)
return {"success": True, "data": output.model_dump()}
@import_router.post("/tool-package/preview", response_model=dict)
async def preview_tool_package_import(
payload: ToolPackageImportPreviewRequest,
db: AsyncSession = Depends(get_db),
current_user: User = Depends(get_admin_user),
) -> dict[str, Any]:
"""工具包导入预览:检查包中每个 tool/system/asset 是否与已有记录冲突(不落库)。"""
use_cases = create_use_cases_from_db(db)
package = _build_tool_package(payload.package)
input_dto = ToolPackageImportPreviewInput(package=package)
output = await use_cases.import_service.preview_tool_package_import(input_dto)
return {"success": True, "data": output.model_dump()}
@import_router.post("/tool-package/confirm", response_model=dict)
async def confirm_tool_package_import(
payload: ToolPackageImportConfirmRequest,
db: AsyncSession = Depends(get_db),
current_user: User = Depends(get_admin_user),
) -> dict[str, Any]:
"""工具包导入确认:导入系统、环境、资产、工具,并重映射资产 ID。``created_by`` 取自当前管理员。"""
use_cases = create_use_cases_from_db(db)
package = _build_tool_package(payload.package)
input_dto = ToolPackageImportInput(
package=package,
override_existing=payload.override_existing,
target_system_slug=payload.target_system_slug,
created_by=current_user.uid,
)
output = await use_cases.import_service.confirm_tool_package_import(input_dto)
return {"success": True, "data": output.model_dump()}