ForcePilot/backend/package/yuxi/external_systems/integrations/jira/operations.py
Kris 55188ff7fa refactor: 整理代码结构并修复多处细节问题
1. 调整导入顺序与格式,修复重复导入
2. 替换通用异常捕获为更精准的ValueError
3. 重构持久层仓库的get_deleted/list_deleted方法,添加ORM转领域模型逻辑
4. 完善配置校验与参数合法性检查,新增缺失字段校验
5. 优化多行字符串与函数调用的换行格式
6. 修复通知服务状态判定逻辑与文档描述
7. 简化无用的TYPE_CHECKING代码块
8. 重构ServiceNow元数据发现逻辑,增加字段合法性校验
9. 修复数据库查询验证器的子查询处理逻辑
10. 完善领域模型的初始化校验与属性语义
2026-07-11 07:34:48 +08:00

713 lines
28 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""Jira / Confluence discover / preview_tools / create_tools handler 实现 +
元数据发现 + cloudId 解析 + 错误提取器。
ADF 序列化与字段类型映射已移至 ``generators.py``(避免循环导入)。
handler 签名遵循 ``OperationHandler`` 协议:
``(source_type: str, system_config: dict[str, Any]) -> Awaitable[Any]``。
handler 内部职责(见设计文档 §4.6 / §6
1. 从 ``system_config`` 提取 ``connection_config`` / ``auth_config`` / ``_resolved_token``
2. 根据 ``deployment_mode`` + ``auth_type`` 解析 base_urlCloud 3LO 需 cloudId 拼接 OAuth 代理地址)
3. 构建 ``auth_headers``token 由 use_cases 层注入到 ``system_config["_resolved_token"]``
非 token 类认证从 ``auth_config`` 构建)
4. 构造带 429 退避的 ``_request_with_retry``,发起元数据发现请求
5. 调用 ``JiraConfigGenerator`` / ``ConfluenceConfigGenerator`` 生成工具配置草稿
两层错误处理(见设计文档 §8
- **L1**: ``extract_jira_error`` / ``extract_confluence_error`` — 注册到
``http_error_extractor_registry``,由框架 executor 在运行时工具执行失败时调用。
- **L2**: ``_map_atlassian_error`` — 不注册到任何注册表,仅在 handler 内部
discover/preview/create 阶段失败后调用,将 HTTP 状态码映射为
``ExternalSystemError`` 子类。
429 退避分层(见设计文档 §4.6
- discover/preview/create 阶段:本模块 ``_request_with_retry`` 实现
- 运行时工具执行:通过工具 ``retry_policy.retry_status_codes: [429]`` 配置
"""
from __future__ import annotations
import asyncio
import base64
from typing import TYPE_CHECKING, Any
from yuxi.external_systems.exceptions import (
AccessDeniedError,
AuthError,
DomainValidationError,
EntityNotFoundError,
ExecutionError,
ExternalSystemError,
RateLimitExceededError,
)
from yuxi.external_systems.integrations.jira.generators import (
ConfluenceConfigGenerator,
JiraConfigGenerator,
)
from yuxi.external_systems.integrations.jira.metadata import (
CLOUD_OAUTH_PROXY_BASE,
CONFLUENCE_CLOUD_SPACES_PATH,
CONFLUENCE_DC_SPACES_PATH,
JIRA_FIELDS_PATH,
JIRA_ISSUE_TYPES_PATH,
JIRA_PROJECTS_PATH,
JIRA_STATUSES_PATH,
)
from yuxi.external_systems.integrations.schemas import GeneratedToolsDraft
if TYPE_CHECKING:
import httpx
# discover/preview/create 阶段 429 退避参数
_RETRY_MAX_ATTEMPTS = 5
_RETRY_BASE_DELAY = 1.0
# Atlassian 通用请求头(除 Authorization 外)
_ATLASSIAN_HEADERS: dict[str, str] = {
"Accept": "application/json",
"Content-Type": "application/json; charset=utf-8",
}
# ═══════════════════════════════════════════════════════════════════════════
# 错误提取器L1— 注册到 http_error_extractor_registry
# ═══════════════════════════════════════════════════════════════════════════
def extract_jira_error(response: httpx.Response) -> str | None:
"""从 Jira 错误响应中提取友好错误信息L1
Jira 错误响应含 ``errorMessages``(字符串数组)与 ``errors``(字段级错误字典)::
{
"errorMessages": ["Issue does not exist or you do not have permission to see it."],
"errors": {"summary": "Field 'summary' cannot be empty."}
}
Returns:
拼接后的错误消息字符串,非 Jira 错误格式时返回 None。
"""
try:
body = response.json()
except ValueError:
return None
if not isinstance(body, dict):
return None
parts: list[str] = []
error_messages = body.get("errorMessages") or []
for msg in error_messages:
if isinstance(msg, str):
parts.append(msg)
errors = body.get("errors") or {}
if isinstance(errors, dict):
for field, message in errors.items():
parts.append(f"[{field}] {message}")
return "; ".join(parts) if parts else None
def extract_confluence_error(response: httpx.Response) -> str | None:
"""从 Confluence 错误响应中提取友好错误信息L1
Confluence 错误响应格式与 Jira 类似,但字段级错误可能嵌套在 ``data.errors`` 中::
{
"message": "Content not found",
"data": {"errors": {"title": "Title is required."}}
}
"""
try:
body = response.json()
except ValueError:
return None
if not isinstance(body, dict):
return None
parts: list[str] = []
message = body.get("message")
if isinstance(message, str) and message:
parts.append(message)
data = body.get("data") or {}
if isinstance(data, dict):
errors = data.get("errors") or {}
if isinstance(errors, dict):
for field, err_msg in errors.items():
parts.append(f"[{field}] {err_msg}")
# Confluence 也可能用 errorMessages 格式
error_messages = body.get("errorMessages") or []
for msg in error_messages:
if isinstance(msg, str):
parts.append(msg)
return "; ".join(parts) if parts else None
def _map_atlassian_error(
status_code: int,
error_body: dict[str, Any],
*,
vendor: str = "Atlassian",
) -> Exception:
"""将 Atlassian 错误响应转换为 ExternalSystemError 子类L2
仅在 handler 内部discover/preview/create 阶段)使用,不注册到框架。
"""
# 提取错误消息(兼容 Jira / Confluence 两种格式)
parts: list[str] = []
if isinstance(error_body, dict):
for msg in error_body.get("errorMessages") or []:
if isinstance(msg, str):
parts.append(msg)
message = error_body.get("message")
if isinstance(message, str) and message:
parts.append(message)
errors = error_body.get("errors") or {}
if isinstance(errors, dict):
for field, err_msg in errors.items():
parts.append(f"[{field}] {err_msg}")
data = error_body.get("data") or {}
if isinstance(data, dict):
data_errors = data.get("errors") or {}
if isinstance(data_errors, dict):
for field, err_msg in data_errors.items():
parts.append(f"[{field}] {err_msg}")
full_message = "; ".join(parts) if parts else f"{vendor} 未知错误 (status={status_code})"
if status_code == 401:
return AuthError(f"{vendor} 认证失败: {full_message}")
if status_code == 403:
return AccessDeniedError(f"{vendor} 访问被拒: {full_message}")
if status_code == 404:
return EntityNotFoundError(f"{vendor} 资源不存在: {full_message}")
if status_code == 429:
return RateLimitExceededError(
f"{vendor} 限流: {full_message}",
retry_after=60,
limit_type="qps",
)
if 500 <= status_code < 600:
return ExecutionError(f"{vendor} 服务端错误: {full_message} (status={status_code})")
return ExecutionError(f"{vendor} 错误: {full_message} (status={status_code})")
# ═══════════════════════════════════════════════════════════════════════════
# system_config 解析辅助
# ═══════════════════════════════════════════════════════════════════════════
def _resolve_deployment_mode(system_config: dict[str, Any]) -> str:
"""从 system_config 提取 deployment_mode默认 cloud。"""
connection_config = system_config.get("connection_config") or {}
return str(connection_config.get("deployment_mode", "cloud"))
def _resolve_api_version(system_config: dict[str, Any], deployment_mode: str) -> str:
"""从 system_config 提取 api_version未配置时按部署模式推断cloud=v3, data_center=v2"""
connection_config = system_config.get("connection_config") or {}
api_version = connection_config.get("api_version")
if api_version:
return str(api_version)
return "v3" if deployment_mode == "cloud" else "v2"
def _resolve_auth_type(system_config: dict[str, Any]) -> str:
"""从 system_config 提取 auth_type。"""
return str(system_config.get("auth_type", "jira_cloud_3lo"))
def _build_auth_headers(system_config: dict[str, Any]) -> dict[str, str]:
"""从 system_config 提取认证信息,构建 Atlassian 请求头。
支持的认证方式:
- token 类jira_cloud_3lo / oauth2_authorization_code从 ``_resolved_token.access_token`` 取 Bearer token
- basicCloud API token从 ``auth_config`` 取 username/password 构建 Basic Auth
- jira_dc_patDC PAT从 ``auth_config`` 取 token 构建 Bearer Auth
"""
auth_type = _resolve_auth_type(system_config)
headers = dict(_ATLASSIAN_HEADERS)
if auth_type in ("jira_cloud_3lo", "oauth2_authorization_code"):
token_info = system_config.get("_resolved_token")
if not isinstance(token_info, dict) or not token_info.get("access_token"):
raise AuthError(
f"jira: system_config 缺少 _resolved_tokenuse_cases 层未注入 tokenauth_type={auth_type}"
)
access_token = token_info["access_token"]
token_type = token_info.get("token_type", "Bearer")
headers["Authorization"] = f"{token_type} {access_token}"
return headers
auth_config = system_config.get("auth_config") or {}
if auth_type == "basic":
username = auth_config.get("username")
password = auth_config.get("password")
if not username or not password:
raise AuthError("jira basic 认证缺少 username / passwordemail + API token")
credentials = f"{username}:{password}"
encoded = base64.b64encode(credentials.encode("utf-8")).decode("ascii")
headers["Authorization"] = f"Basic {encoded}"
return headers
if auth_type == "jira_dc_pat":
token = auth_config.get("token")
if not token:
raise AuthError("jira_dc_pat 认证缺少 token")
headers["Authorization"] = f"Bearer {token}"
return headers
raise AuthError(f"jira 不支持的认证类型: {auth_type}")
def _resolve_base_url(system_config: dict[str, Any], deployment_mode: str) -> str:
"""根据部署模式与认证方式解析 API base_url见设计文档 §6.3)。
- Cloud 3LO: ``https://api.atlassian.com/ex/{product}/{cloudId}``
cloudId 从 ``_resolved_token.extra.cloud_id`` 读取)
- Cloud API token: ``https://{site}.atlassian.net``(从 ``connection_config.site_url`` 读取)
- DC OAuth2 / PAT: ``https://{host}:{port}/[context]``(从 ``connection_config.base_url`` 读取)
多站点场景:若 ``connection_config.cloud_id`` 显式配置,校验与 ``extra.cloud_id`` 是否一致。
"""
connection_config = system_config.get("connection_config") or {}
if deployment_mode != "cloud":
base_url = connection_config.get("base_url")
if not base_url:
raise DomainValidationError("Jira DC 缺少 connection_config.base_url")
return str(base_url).rstrip("/")
# Cloud: 优先使用 _resolved_token.extra.cloud_id 拼接 OAuth 代理地址
token_info = system_config.get("_resolved_token") or {}
extra = token_info.get("extra") or {}
cloud_id_from_token = extra.get("cloud_id")
# connection_config.cloud_id 为用户显式配置的期望 cloudId多站点场景
expected_cloud_id = connection_config.get("cloud_id")
if expected_cloud_id and cloud_id_from_token and expected_cloud_id != cloud_id_from_token:
raise AuthError(
f"Jira Cloud 3LO cloudId 不匹配: token 解析为 {cloud_id_from_token}"
f"但 connection_config.cloud_id 配置为 {expected_cloud_id},请重新授权"
)
if cloud_id_from_token:
product = connection_config.get("product", "jira")
return f"{CLOUD_OAUTH_PROXY_BASE}/{product}/{cloud_id_from_token}"
# Cloud API token: 使用 site URL
site_url = connection_config.get("site_url") or connection_config.get("base_url")
if not site_url:
raise DomainValidationError(
"Jira Cloud 缺少 _resolved_token.extra.cloud_id 且 connection_config.site_url 未配置"
)
return str(site_url).rstrip("/")
def _resolve_selected_projects(system_config: dict[str, Any]) -> list[str] | None:
"""从 system_config 提取用户选择的项目 key 列表(可选)。
优先从顶层 ``selected_projects`` 读取(见设计文档 §7.4
回退到 ``discovery_options.selected_projects``(兼容 dynamics365 模式)。
"""
selected = system_config.get("selected_projects")
if not selected:
discovery_options = system_config.get("discovery_options") or {}
selected = discovery_options.get("selected_projects")
if selected:
return list(selected)
return None
# ═══════════════════════════════════════════════════════════════════════════
# 429 退避 HTTP 请求
# ═══════════════════════════════════════════════════════════════════════════
async def _request_with_retry(
method: str,
url: str,
headers: dict[str, str],
*,
params: dict[str, Any] | None = None,
json_body: dict[str, Any] | None = None,
max_retries: int = _RETRY_MAX_ATTEMPTS,
base_delay: float = _RETRY_BASE_DELAY,
vendor: str = "Atlassian",
) -> httpx.Response:
"""handler 内部 HTTP 调用的 429 退避(仅用于 discover/preview/create 阶段)。
优先使用响应头 ``Retry-After``,否则指数退避(``base_delay * 2 ** attempt``)。
运行时工具执行的 429 退避由框架 executor + 工具 retry_policy 处理。
非 2xx 响应(非 429通过 ``_map_atlassian_error`` 转换为 ExternalSystemError 子类。
网络层异常(超时 / 连接错误)转换为 ``ExecutionError``。
"""
import httpx
try:
async with httpx.AsyncClient() as client:
last_response: httpx.Response | None = None
for attempt in range(max_retries + 1):
response = await client.request(method, url, headers=headers, params=params, json=json_body)
if response.status_code != 429:
last_response = response
break
if attempt == max_retries:
retry_after_header = response.headers.get("Retry-After", "60")
try:
retry_after = int(retry_after_header)
except ValueError:
retry_after = 60
raise RateLimitExceededError(
f"{vendor} 限流,已重试 {max_retries} 次仍失败: {url}",
retry_after=retry_after,
limit_type="qps",
)
retry_after_header = response.headers.get("Retry-After")
if retry_after_header:
try:
delay = float(retry_after_header)
except ValueError:
delay = base_delay * (2**attempt)
else:
delay = base_delay * (2**attempt)
await asyncio.sleep(delay)
except httpx.RequestError as exc:
raise ExecutionError(f"{vendor} 网络请求失败: {exc}") from exc
if last_response is None:
raise ExecutionError(f"{vendor} 请求未返回有效响应")
if last_response.is_success:
return last_response
try:
error_body = last_response.json()
except ValueError:
error_body = {"errorMessages": [last_response.text]}
raise _map_atlassian_error(last_response.status_code, error_body, vendor=vendor)
# ═══════════════════════════════════════════════════════════════════════════
# Jira 元数据发现函数
# ═══════════════════════════════════════════════════════════════════════════
async def _list_projects(
base_url: str,
auth_headers: dict[str, str],
api_version: str,
) -> list[dict[str, Any]]:
"""调用 ``GET /rest/api/{2|3}/project`` 获取项目列表。"""
path = JIRA_PROJECTS_PATH.format(api_version=api_version)
response = await _request_with_retry(
method="GET",
url=f"{base_url}{path}",
headers=auth_headers,
)
data = response.json()
return data if isinstance(data, list) else []
async def _list_issue_types_for_project(
base_url: str,
auth_headers: dict[str, str],
project_id_or_key: str,
api_version: str,
) -> list[dict[str, Any]]:
"""调用 ``GET /rest/api/{2|3}/issue/createmeta/{projectIdOrKey}/issuetypes`` 获取项目下可用 issue types。
替代已弃用的 ``/issue/createmeta`` 旧端点(见设计文档 §4.5)。
"""
path = JIRA_ISSUE_TYPES_PATH.format(
api_version=api_version,
project_id_or_key=project_id_or_key,
)
response = await _request_with_retry(
method="GET",
url=f"{base_url}{path}",
headers=auth_headers,
)
data = response.json()
return data.get("issueTypes", []) if isinstance(data, dict) else []
async def _list_fields(
base_url: str,
auth_headers: dict[str, str],
api_version: str,
) -> list[dict[str, Any]]:
"""调用 ``GET /rest/api/{2|3}/field`` 获取所有字段(含 ``customfield_*``)。"""
path = JIRA_FIELDS_PATH.format(api_version=api_version)
response = await _request_with_retry(
method="GET",
url=f"{base_url}{path}",
headers=auth_headers,
)
data = response.json()
return data if isinstance(data, list) else []
async def _list_statuses(
base_url: str,
auth_headers: dict[str, str],
api_version: str,
) -> list[dict[str, Any]]:
"""调用 ``GET /rest/api/{2|3}/status`` 获取状态列表。"""
path = JIRA_STATUSES_PATH.format(api_version=api_version)
response = await _request_with_retry(
method="GET",
url=f"{base_url}{path}",
headers=auth_headers,
)
data = response.json()
return data if isinstance(data, list) else []
# ═══════════════════════════════════════════════════════════════════════════
# Confluence 元数据发现函数
# ═══════════════════════════════════════════════════════════════════════════
async def _list_confluence_spaces(
base_url: str,
auth_headers: dict[str, str],
deployment_mode: str,
) -> list[dict[str, Any]]:
"""获取 Confluence 空间列表。
- Cloud v2: ``GET /wiki/rest/api/v2/spaces``cursor 分页,首次取第一页)
- DC: ``GET /rest/api/space``offset 分页,首次取第一页)
"""
if deployment_mode == "cloud":
path = CONFLUENCE_CLOUD_SPACES_PATH
else:
path = CONFLUENCE_DC_SPACES_PATH
response = await _request_with_retry(
method="GET",
url=f"{base_url}{path}",
headers=auth_headers,
params={"limit": 50},
vendor="Confluence",
)
data = response.json()
return data.get("results", []) if isinstance(data, dict) else []
# ═══════════════════════════════════════════════════════════════════════════
# Jira OperationHandler 实现
# ═══════════════════════════════════════════════════════════════════════════
async def discover_jira(
source_type: str,
system_config: dict[str, Any],
) -> list[dict[str, Any]]:
"""discover handler发现 Jira 可用项目列表(含 issue types 摘要)。
Returns:
项目摘要列表,每项含 ``key`` / ``id`` / ``name`` / ``issue_type_count``。
"""
deployment_mode = _resolve_deployment_mode(system_config)
api_version = _resolve_api_version(system_config, deployment_mode)
base_url = _resolve_base_url(system_config, deployment_mode)
auth_headers = _build_auth_headers(system_config)
projects = await _list_projects(base_url, auth_headers, api_version)
results: list[dict[str, Any]] = []
for project in projects:
project_key = project.get("key", "")
if not project_key:
continue
# 获取项目下 issue types 摘要
try:
issue_types = await _list_issue_types_for_project(base_url, auth_headers, project_key, api_version)
issue_type_count = len(issue_types)
except ExternalSystemError:
# 单个项目失败不影响整体发现
issue_type_count = 0
results.append(
{
"key": project_key,
"id": project.get("id", ""),
"name": project.get("name", project_key),
"issue_type_count": issue_type_count,
}
)
return results
async def preview_jira_tools(
source_type: str,
system_config: dict[str, Any],
) -> list[dict[str, Any]]:
"""preview_tools handler预览将生成的工具列表。
Returns:
工具预览列表,每项为 ExternalToolCreateInput 兼容的 dict。
"""
payload = await _build_jira_generator_payload(system_config)
generator = JiraConfigGenerator()
return await generator.generate(payload)
async def create_jira_tools(
source_type: str,
system_config: dict[str, Any],
) -> GeneratedToolsDraft:
"""create_tools handler生成工具草稿不直接持久化
Returns:
GeneratedToolsDraft由 use_cases 层统一持久化。
"""
payload = await _build_jira_generator_payload(system_config)
generator = JiraConfigGenerator()
tool_configs = await generator.generate(payload)
return GeneratedToolsDraft(tool_configs=tool_configs, override_existing=False)
async def _build_jira_generator_payload(system_config: dict[str, Any]) -> dict[str, Any]:
"""从 system_config 构造 Jira 生成器 payload。
流程:
1. 解析 deployment_mode / api_version / base_url / auth_headers / auth_type / auth_config
2. 发现项目列表
3. 为每个项目发现 issue types
4. 发现全局字段列表(含 customfield_*
5. 发现状态列表
6. 汇总为生成器 payload
"""
deployment_mode = _resolve_deployment_mode(system_config)
api_version = _resolve_api_version(system_config, deployment_mode)
base_url = _resolve_base_url(system_config, deployment_mode)
auth_headers = _build_auth_headers(system_config)
auth_type = _resolve_auth_type(system_config)
auth_config = system_config.get("auth_config") or {}
selected_projects = _resolve_selected_projects(system_config)
# 发现项目列表
projects = await _list_projects(base_url, auth_headers, api_version)
# 按选择过滤项目
if selected_projects:
projects = [p for p in projects if p.get("key") in selected_projects]
# 为每个项目发现 issue types
issue_type_map: dict[str, list[dict[str, Any]]] = {}
field_map: dict[tuple[str, str], list[dict[str, Any]]] = {}
for project in projects:
project_key = project.get("key", "")
if not project_key:
continue
issue_types = await _list_issue_types_for_project(base_url, auth_headers, project_key, api_version)
issue_type_map[project_key] = issue_types
# 发现全局字段列表(含 customfield_*
fields = await _list_fields(base_url, auth_headers, api_version)
# 发现状态列表
statuses = await _list_statuses(base_url, auth_headers, api_version)
status_map: dict[str, list[dict[str, Any]]] = {}
for project in projects:
project_key = project.get("key", "")
if project_key:
status_map[project_key] = statuses
return {
"projects": projects,
"issue_type_map": issue_type_map,
"field_map": field_map,
"fields": fields,
"status_map": status_map,
"base_url": base_url,
"api_version": api_version,
"deployment_mode": deployment_mode,
"selected_projects": selected_projects,
"auth_type": auth_type,
"auth_config": auth_config,
}
# ═══════════════════════════════════════════════════════════════════════════
# Confluence OperationHandler 实现
# ═══════════════════════════════════════════════════════════════════════════
async def discover_confluence(
source_type: str,
system_config: dict[str, Any],
) -> list[dict[str, Any]]:
"""discover handler发现 Confluence 可用空间列表。
Returns:
空间摘要列表,每项含 ``id`` / ``key`` / ``name`` / ``type``。
"""
deployment_mode = _resolve_deployment_mode(system_config)
base_url = _resolve_base_url(system_config, deployment_mode)
auth_headers = _build_auth_headers(system_config)
spaces = await _list_confluence_spaces(base_url, auth_headers, deployment_mode)
return [
{
"id": s.get("id", ""),
"key": s.get("key", ""),
"name": s.get("name", ""),
"type": s.get("type", ""),
}
for s in spaces
]
async def preview_confluence_tools(
source_type: str,
system_config: dict[str, Any],
) -> list[dict[str, Any]]:
"""preview_tools handler预览将生成的 Confluence 工具列表。"""
payload = await _build_confluence_generator_payload(system_config)
generator = ConfluenceConfigGenerator()
return await generator.generate(payload)
async def create_confluence_tools(
source_type: str,
system_config: dict[str, Any],
) -> GeneratedToolsDraft:
"""create_tools handler生成 Confluence 工具草稿(不直接持久化)。"""
payload = await _build_confluence_generator_payload(system_config)
generator = ConfluenceConfigGenerator()
tool_configs = await generator.generate(payload)
return GeneratedToolsDraft(tool_configs=tool_configs, override_existing=False)
async def _build_confluence_generator_payload(system_config: dict[str, Any]) -> dict[str, Any]:
"""从 system_config 构造 Confluence 生成器 payload。"""
deployment_mode = _resolve_deployment_mode(system_config)
api_version = _resolve_api_version(system_config, deployment_mode)
base_url = _resolve_base_url(system_config, deployment_mode)
auth_headers = _build_auth_headers(system_config)
auth_type = _resolve_auth_type(system_config)
auth_config = system_config.get("auth_config") or {}
spaces = await _list_confluence_spaces(base_url, auth_headers, deployment_mode)
return {
"spaces": spaces,
"base_url": base_url,
"api_version": api_version,
"deployment_mode": deployment_mode,
"auth_type": auth_type,
"auth_config": auth_config,
}