diff --git a/.env.template b/.env.template index 71b296f3..d2f85abe 100644 --- a/.env.template +++ b/.env.template @@ -29,4 +29,8 @@ TAVILY_API_KEY= # 获取搜索服务的 api key 请访问 https://app.tavily.co # YUXI_URL_WHITELIST=github.com,docs.example.com,gitlab.example.com,127.0.0.1 # # MinerU -# MINERU_API_KEY= \ No newline at end of file +# MINERU_API_KEY= + +# LightRag llm 并发限制 +# MAX_ASYNC=5 +# EMBEDDING_FUNC_MAX_ASYNC=8 \ No newline at end of file diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 9d869783..e1d4b675 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -44,6 +44,7 @@ export default defineConfig({ { text: '配置系统详解', link: '/latest/advanced/configuration' }, { text: '文档解析', link: '/latest/advanced/document-processing' }, { text: '智能体', link: '/latest/advanced/agents-config' }, + { text: 'Skills 管理', link: '/latest/advanced/skills-management' }, { text: '品牌自定义', link: '/latest/advanced/branding' }, { text: '其他配置', link: '/latest/advanced/misc' }, { text: '生产部署', link: '/latest/advanced/deployment' } diff --git a/docs/latest/advanced/agents-config.md b/docs/latest/advanced/agents-config.md index 90ea3198..37637e6e 100644 --- a/docs/latest/advanced/agents-config.md +++ b/docs/latest/advanced/agents-config.md @@ -61,6 +61,7 @@ async def get_graph(self, **kwargs): | `tools` | list[str] | 启用的内置工具列表 | | `knowledges` | list[str] | 关联的知识库列表 | | `mcps` | list[str] | 启用的 MCP 服务器名称 | +| `skills` | list[str] | 关联的 Skills（运行时只读挂载到 `/skills`） | ```python from src.agents.common import BaseContext @@ -98,6 +99,14 @@ class ReporterContext(BaseContext): 更多动态工具选择与 MCP 注册的例子，见 `src/agents/chatbot/graph.py` 中的中间件组合。 +### Skills 只读挂载 + +`BaseContext.skills` 用于声明当前智能体可访问的技能目录（slug 列表）。运行时会将这些目录只读挂载到 `/skills//...`： + +1. 仅显示配置中选中的 skills，未选中的 slug 在运行时不可见。 +2. `/skills` 仅支持读取能力（`ls/read/glob/grep`），写入和编辑会被拒绝。 +3. skills 元数据来自数据库索引，内容目录来自共享存储 `/app/saves/skills`。 + ### 拓展现有智能体 智能体保持为 LangGraph 的标准节点组合，因此可以在原有 `graph.py` 中添加节点、条件与消息转换器。复用现成上下文时，只需扩展当前 `context_schema` 的字段；若功能差异较大，可以创建新的上下文类并替换 `context_schema`。 diff --git a/docs/latest/advanced/skills-management.md b/docs/latest/advanced/skills-management.md new file mode 100644 index 00000000..71228eb6 --- /dev/null +++ b/docs/latest/advanced/skills-management.md @@ -0,0 +1,42 @@ +# Skills 管理 + +Skills 管理模块用于集中维护可供 Agent 只读引用的技能包。 +本期采用“文件系统存内容，数据库存索引”模式： + +1. 技能目录存储在 `/app/saves/skills`（本地 `save_dir/skills`）。 +2. 技能元数据（slug/name/description/dir_path）存储在 `skills` 表。 +3. Agent 配置通过 `context.skills` 选择技能，运行时挂载到 `/skills` 且只读。 + +## 权限与入口 + +1. 系统设置中新增 `Skills 管理` 页签（仅 `superadmin` 可见）。 +2. `admin` 仅可调用列表接口（用于 Agent 配置选择 skills）。 +3. `user` 无 skills 管理权限。 + +## 导入规范（ZIP） + +1. 单包单技能，且必须包含一个 `SKILL.md`。 +2. `SKILL.md` 必须包含 frontmatter，且 `name`、`description` 必填。 +3. `name` 需满足 slug 规则：小写字母/数字/短横线。 +4. 导入时执行路径安全校验，拒绝绝对路径与 `..` 路径穿越。 +5. slug 冲突时自动追加 `-v2/-v3...`，并自动改写 `SKILL.md` 中 `name` 为最终 slug。 +6. 导入采用临时目录 + 原子替换，避免半成品落盘。 + +## 在线管理能力 + +1. Skills 列表：来自数据库，避免全量目录扫描。 +2. 目录树：按原生目录结构展示。 +3. 文件级 CRUD：支持新建文件/目录、编辑文本文件、删除文件/目录。 +4. 文件编辑仅允许文本类型（如 md/py/js/ts/json/yaml/toml/txt 等）。 +5. `SKILL.md` 保存后会重新解析，并同步更新数据库中的 `name/description`。 +6. 支持导出单个 skill 为 ZIP。 +7. 删除 skill 时会同时删除目录与数据库记录（硬删除）。 + +## Agent 运行时行为 + +1. `context.skills` 用于配置技能 slug 列表。 +2. 运行时按会话构建 `SkillResolver` 快照（同一会话首次构建，后续复用）。 +3. 运行时仅暴露快照中的可见 skills 到 `/skills//...`。 +4. `/skills` 路径只读，不允许写入、编辑、上传。 +5. 同会话内若 `context.skills` 变化会触发快照重建。 +6. 后台修改 skills 内容后，已有会话不会自动刷新，需新会话或调整 `context.skills` 才生效。 diff --git a/scripts/migrate_all.py b/scripts/migrate_all.py index 1dca7acb..d77f0356 100644 --- a/scripts/migrate_all.py +++ b/scripts/migrate_all.py @@ -658,6 +658,7 @@ class MigrationRunner: url=sqlite_server.url, command=sqlite_server.command, args=sqlite_server.args, + env=getattr(sqlite_server, "env", None), headers=sqlite_server.headers, timeout=sqlite_server.timeout, sse_read_timeout=sqlite_server.sse_read_timeout, diff --git a/scripts/migrate_business_from_sqlite.py b/scripts/migrate_business_from_sqlite.py index 252ee2d0..3d8447b8 100644 --- a/scripts/migrate_business_from_sqlite.py +++ b/scripts/migrate_business_from_sqlite.py @@ -532,6 +532,7 @@ async def migrate_mcp_servers(sqlite_reader: SQLiteReader, dry_run: bool, execut url=sqlite_server.url, command=sqlite_server.command, args=sqlite_server.args, + env=getattr(sqlite_server, "env", None), headers=sqlite_server.headers, timeout=sqlite_server.timeout, sse_read_timeout=sqlite_server.sse_read_timeout, diff --git a/server/routers/__init__.py b/server/routers/__init__.py index fa25fe84..a66420ec 100644 --- a/server/routers/__init__.py +++ b/server/routers/__init__.py @@ -9,6 +9,7 @@ from server.routers.knowledge_router import knowledge from server.routers.evaluation_router import evaluation from server.routers.mcp_router import mcp from server.routers.mindmap_router import mindmap +from server.routers.skill_router import skills from server.routers.system_router import system from server.routers.task_router import tasks @@ -26,3 +27,4 @@ router.include_router(mindmap) # /api/mindmap/* router.include_router(graph) # /api/graph/* router.include_router(tasks) # /api/tasks/* router.include_router(mcp) # /api/system/mcp-servers/* +router.include_router(skills) # /api/system/skills/* diff --git a/server/routers/chat_router.py b/server/routers/chat_router.py index 95b41a01..fe2f44ca 100644 --- a/server/routers/chat_router.py +++ b/server/routers/chat_router.py @@ -75,7 +75,7 @@ async def get_default_agent(current_user: User = Depends(get_required_user)): default_agent_id = conf.default_agent_id # 如果没有设置默认智能体，尝试获取第一个可用的智能体 if not default_agent_id: - agents = await agent_manager.get_agents_info() + agents = await agent_manager.get_agents_info(include_configurable_items=False) if agents: default_agent_id = agents[0].get("id", "") @@ -94,7 +94,7 @@ async def set_default_agent(request_data: dict = Body(...), current_user=Depends raise HTTPException(status_code=422, detail="缺少必需的 agent_id 字段") # 验证智能体是否存在 - agents = await agent_manager.get_agents_info() + agents = await agent_manager.get_agents_info(include_configurable_items=False) agent_ids = [agent.get("id", "") for agent in agents] if agent_id not in agent_ids: @@ -137,7 +137,7 @@ async def call(query: str = Body(...), meta: dict = Body(None), current_user: Us @chat.get("/agent") async def get_agent(current_user: User = Depends(get_required_user)): """获取所有可用智能体的基本信息（需要登录）""" - agents_info = await agent_manager.get_agents_info() + agents_info = await agent_manager.get_agents_info(include_configurable_items=False) # Return agents with basic information (without configurable_items for performance) agents = [ diff --git a/server/routers/evaluation_router.py b/server/routers/evaluation_router.py index 5fee1ff2..ee635ce2 100644 --- a/server/routers/evaluation_router.py +++ b/server/routers/evaluation_router.py @@ -1,6 +1,7 @@ import traceback from fastapi import APIRouter, HTTPException, Depends, File, Form, Body, UploadFile +from fastapi.responses import FileResponse from src.storage.postgres.models_business import User from server.utils.auth_middleware import get_admin_user from src.utils import logger @@ -53,6 +54,29 @@ async def delete_evaluation_benchmark(benchmark_id: str, current_user: User = De raise HTTPException(status_code=500, detail=f"删除评估基准失败: {str(e)}") +@evaluation.get("/benchmarks/{benchmark_id}/download") +async def download_evaluation_benchmark(benchmark_id: str, current_user: User = Depends(get_admin_user)): + """下载评估基准文件""" + from src.services.evaluation_service import EvaluationService + + try: + service = EvaluationService() + download_info = await service.get_benchmark_download_info(benchmark_id) + return FileResponse( + path=download_info["file_path"], + filename=download_info["filename"], + media_type="application/x-ndjson", + ) + except ValueError as e: + if "not found" in str(e).lower(): + raise HTTPException(status_code=404, detail=str(e)) + logger.error(f"下载评估基准失败: {e}, {traceback.format_exc()}") + raise HTTPException(status_code=500, detail=f"下载评估基准失败: {str(e)}") + except Exception as e: + logger.error(f"下载评估基准失败: {e}, {traceback.format_exc()}") + raise HTTPException(status_code=500, detail=f"下载评估基准失败: {str(e)}") + + @evaluation.get("/databases/{db_id}/results/{task_id}") async def get_evaluation_results_by_db( db_id: str, diff --git a/server/routers/knowledge_router.py b/server/routers/knowledge_router.py index 1c748356..f79a6cb1 100644 --- a/server/routers/knowledge_router.py +++ b/server/routers/knowledge_router.py @@ -13,6 +13,7 @@ from starlette.responses import StreamingResponse from src.services.task_service import TaskContext, tasker from server.utils.auth_middleware import get_admin_user, get_required_user from src import config, knowledge_base +from src.knowledge.chunking.ragflow_like.presets import ensure_chunk_defaults_in_additional_params from src.knowledge.indexing import SUPPORTED_FILE_EXTENSIONS, is_supported_file_extension, process_file_to_markdown from src.knowledge.utils import calculate_content_hash from src.models.embed import test_all_embedding_models_status, test_embedding_model_status @@ -116,6 +117,7 @@ async def create_database( params.pop("reranker_config", None) remove_reranker_config(kb_type, additional_params) + additional_params = ensure_chunk_defaults_in_additional_params(additional_params) embed_info = config.embed_model_names[embed_model_name] # 将Pydantic模型转换为字典以便JSON序列化 @@ -180,7 +182,7 @@ async def update_database_info( name: str = Body(...), description: str = Body(...), llm_info: dict = Body(None), - additional_params: dict = Body({}), + additional_params: dict | None = Body(None), share_config: dict = Body(None), current_user: User = Depends(get_admin_user), ): @@ -190,6 +192,9 @@ async def update_database_info( f"additional_params={additional_params}, share_config={share_config}" ) try: + if additional_params is not None: + additional_params = ensure_chunk_defaults_in_additional_params(additional_params) + database = await knowledge_base.update_database( db_id, name, @@ -267,7 +272,13 @@ async def add_documents( "chunk_size": params.get("chunk_size", 1000), "chunk_overlap": params.get("chunk_overlap", 200), "qa_separator": params.get("qa_separator", ""), + "chunk_preset_id": params.get("chunk_preset_id"), + "chunk_parser_config": params.get("chunk_parser_config"), } + if not indexing_params.get("chunk_preset_id"): + indexing_params.pop("chunk_preset_id", None) + if not isinstance(indexing_params.get("chunk_parser_config"), dict): + indexing_params.pop("chunk_parser_config", None) # URL 解析与入库（需白名单验证） if content_type == "url": diff --git a/server/routers/mcp_router.py b/server/routers/mcp_router.py index 66b31f0d..f1c9828c 100644 --- a/server/routers/mcp_router.py +++ b/server/routers/mcp_router.py @@ -33,6 +33,7 @@ class CreateMcpServerRequest(BaseModel): url: str | None = Field(None, description="服务器 URL（sse/streamable_http）") command: str | None = Field(None, description="命令（stdio）") args: list | None = Field(None, description="命令参数数组（stdio）") + env: dict | None = Field(None, description="环境变量（stdio）") description: str | None = Field(None, description="描述") headers: dict | None = Field(None, description="HTTP 请求头") timeout: int | None = Field(None, description="HTTP 超时时间（秒）") @@ -46,6 +47,7 @@ class UpdateMcpServerRequest(BaseModel): url: str | None = Field(None, description="服务器 URL") command: str | None = Field(None, description="命令（stdio）") args: list | None = Field(None, description="命令参数数组（stdio）") + env: dict | None = Field(None, description="环境变量（stdio）") description: str | None = Field(None, description="描述") headers: dict | None = Field(None, description="HTTP 请求头") timeout: int | None = Field(None, description="HTTP 超时时间（秒）") @@ -112,6 +114,7 @@ async def create_mcp_server_route( url=request.url, command=request.command, args=request.args, + env=request.env, description=request.description, headers=request.headers, timeout=request.timeout, @@ -159,6 +162,11 @@ async def update_mcp_server_route( raise HTTPException(status_code=400, detail=f"传输类型必须是 {', '.join(valid_transports)} 之一") try: + fields_set = getattr(request, "model_fields_set", getattr(request, "__fields_set__", set())) + update_kwargs = {} + if "env" in fields_set: + update_kwargs["env"] = request.env + server = await update_mcp_server( db, name=name, @@ -173,6 +181,7 @@ async def update_mcp_server_route( tags=request.tags, icon=request.icon, updated_by=current_user.username, + **update_kwargs, ) return {"success": True, "data": server.to_dict()} except ValueError as ve: diff --git a/server/routers/skill_router.py b/server/routers/skill_router.py new file mode 100644 index 00000000..ce77b993 --- /dev/null +++ b/server/routers/skill_router.py @@ -0,0 +1,294 @@ +"""Skills 管理路由""" + +from __future__ import annotations + +from pathlib import Path + +from fastapi import APIRouter, BackgroundTasks, Depends, File, HTTPException, Query, UploadFile +from fastapi.responses import FileResponse +from pydantic import BaseModel, Field +from sqlalchemy.ext.asyncio import AsyncSession + +from server.utils.auth_middleware import get_admin_user, get_db, get_superadmin_user +from src.services.skill_service import ( + create_skill_node, + delete_skill, + delete_skill_node, + export_skill_zip, + get_skill_dependency_options, + get_skill_tree, + import_skill_zip, + list_skills, + read_skill_file, + update_skill_dependencies, + update_skill_file, +) +from src.storage.postgres.models_business import User +from src.utils.logging_config import logger + +skills = APIRouter(prefix="/system/skills", tags=["skills"]) + + +class SkillNodeCreateRequest(BaseModel): + path: str = Field(..., description="相对 skill 根目录的路径") + is_dir: bool = Field(False, description="是否创建目录") + content: str | None = Field("", description="文件内容（仅文件创建时生效）") + + +class SkillFileUpdateRequest(BaseModel): + path: str = Field(..., description="相对 skill 根目录的路径") + content: str = Field(..., description="文件内容") + + +class SkillDependenciesUpdateRequest(BaseModel): + tool_dependencies: list[str] = Field(default_factory=list, description="依赖的内置工具列表") + mcp_dependencies: list[str] = Field(default_factory=list, description="依赖的 MCP 服务列表") + skill_dependencies: list[str] = Field(default_factory=list, description="依赖的其他 skill slug 列表") + + +def _raise_from_value_error(e: ValueError) -> None: + message = str(e) + status_code = 404 if "不存在" in message else 400 + raise HTTPException(status_code=status_code, detail=message) + + +def _cleanup_export_file(path: str) -> None: + try: + Path(path).unlink(missing_ok=True) + except Exception as e: + logger.warning(f"Failed to cleanup exported skill archive '{path}': {e}") + + +@skills.get("") +async def list_skills_route( + _current_user: User = Depends(get_admin_user), + db: AsyncSession = Depends(get_db), +): + """获取技能列表（管理员可读）。""" + try: + items = await list_skills(db) + return {"success": True, "data": [item.to_dict() for item in items]} + except Exception as e: + logger.error(f"Failed to list skills: {e}") + raise HTTPException(status_code=500, detail="获取技能列表失败") + + +@skills.get("/dependency-options") +async def get_skill_dependency_options_route( + _current_user: User = Depends(get_superadmin_user), + db: AsyncSession = Depends(get_db), +): + """获取 skill 依赖项可选列表（仅超级管理员）。""" + try: + return {"success": True, "data": await get_skill_dependency_options(db)} + except Exception as e: + logger.error(f"Failed to get skill dependency options: {e}") + raise HTTPException(status_code=500, detail="获取 skill 依赖选项失败") + + +@skills.post("/import") +async def import_skill_route( + file: UploadFile = File(...), + current_user: User = Depends(get_superadmin_user), + db: AsyncSession = Depends(get_db), +): + """导入技能压缩包（仅超级管理员）。""" + try: + file_bytes = await file.read() + item = await import_skill_zip( + db, + filename=file.filename or "", + file_bytes=file_bytes, + created_by=current_user.username, + ) + return {"success": True, "data": item.to_dict()} + except ValueError as e: + _raise_from_value_error(e) + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to import skill zip: {e}") + raise HTTPException(status_code=500, detail="导入技能失败") + + +@skills.get("/{slug}/tree") +async def get_skill_tree_route( + slug: str, + _current_user: User = Depends(get_superadmin_user), + db: AsyncSession = Depends(get_db), +): + """获取技能目录树（仅超级管理员）。""" + try: + tree = await get_skill_tree(db, slug) + return {"success": True, "data": tree} + except ValueError as e: + _raise_from_value_error(e) + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to get skill tree '{slug}': {e}") + raise HTTPException(status_code=500, detail="获取技能目录树失败") + + +@skills.get("/{slug}/file") +async def get_skill_file_route( + slug: str, + path: str = Query(..., description="相对 skill 根目录路径"), + _current_user: User = Depends(get_superadmin_user), + db: AsyncSession = Depends(get_db), +): + """读取技能文本文件（仅超级管理员）。""" + try: + data = await read_skill_file(db, slug, path) + return {"success": True, "data": data} + except ValueError as e: + _raise_from_value_error(e) + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to read skill file '{slug}/{path}': {e}") + raise HTTPException(status_code=500, detail="读取技能文件失败") + + +@skills.post("/{slug}/file") +async def create_skill_file_route( + slug: str, + payload: SkillNodeCreateRequest, + current_user: User = Depends(get_superadmin_user), + db: AsyncSession = Depends(get_db), +): + """创建技能文件或目录（仅超级管理员）。""" + try: + await create_skill_node( + db, + slug=slug, + relative_path=payload.path, + is_dir=payload.is_dir, + content=payload.content, + updated_by=current_user.username, + ) + return {"success": True} + except ValueError as e: + _raise_from_value_error(e) + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to create skill node '{slug}/{payload.path}': {e}") + raise HTTPException(status_code=500, detail="创建技能文件失败") + + +@skills.put("/{slug}/file") +async def update_skill_file_route( + slug: str, + payload: SkillFileUpdateRequest, + current_user: User = Depends(get_superadmin_user), + db: AsyncSession = Depends(get_db), +): + """更新技能文本文件（仅超级管理员）。""" + try: + await update_skill_file( + db, + slug=slug, + relative_path=payload.path, + content=payload.content, + updated_by=current_user.username, + ) + return {"success": True} + except ValueError as e: + _raise_from_value_error(e) + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to update skill file '{slug}/{payload.path}': {e}") + raise HTTPException(status_code=500, detail="更新技能文件失败") + + +@skills.put("/{slug}/dependencies") +async def update_skill_dependencies_route( + slug: str, + payload: SkillDependenciesUpdateRequest, + current_user: User = Depends(get_superadmin_user), + db: AsyncSession = Depends(get_db), +): + """更新 skill 依赖（仅超级管理员）。""" + try: + item = await update_skill_dependencies( + db, + slug=slug, + tool_dependencies=payload.tool_dependencies, + mcp_dependencies=payload.mcp_dependencies, + skill_dependencies=payload.skill_dependencies, + updated_by=current_user.username, + ) + return {"success": True, "data": item.to_dict()} + except ValueError as e: + _raise_from_value_error(e) + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to update skill dependencies '{slug}': {e}") + raise HTTPException(status_code=500, detail="更新 skill 依赖失败") + + +@skills.delete("/{slug}/file") +async def delete_skill_file_route( + slug: str, + path: str = Query(..., description="相对 skill 根目录路径"), + _current_user: User = Depends(get_superadmin_user), + db: AsyncSession = Depends(get_db), +): + """删除技能文件或目录（仅超级管理员）。""" + try: + await delete_skill_node(db, slug=slug, relative_path=path) + return {"success": True} + except ValueError as e: + _raise_from_value_error(e) + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to delete skill file '{slug}/{path}': {e}") + raise HTTPException(status_code=500, detail="删除技能文件失败") + + +@skills.get("/{slug}/export") +async def export_skill_route( + slug: str, + background_tasks: BackgroundTasks, + _current_user: User = Depends(get_superadmin_user), + db: AsyncSession = Depends(get_db), +): + """导出技能压缩包（仅超级管理员）。""" + try: + export_path, download_name = await export_skill_zip(db, slug) + background_tasks.add_task(_cleanup_export_file, export_path) + return FileResponse( + path=export_path, + media_type="application/zip", + filename=download_name, + ) + except ValueError as e: + _raise_from_value_error(e) + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to export skill '{slug}': {e}") + raise HTTPException(status_code=500, detail="导出技能失败") + + +@skills.delete("/{slug}") +async def delete_skill_route( + slug: str, + _current_user: User = Depends(get_superadmin_user), + db: AsyncSession = Depends(get_db), +): + """删除技能（目录 + 数据库记录，仅超级管理员）。""" + try: + await delete_skill(db, slug=slug) + return {"success": True} + except ValueError as e: + _raise_from_value_error(e) + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to delete skill '{slug}': {e}") + raise HTTPException(status_code=500, detail="删除技能失败") diff --git a/server/utils/lifespan.py b/server/utils/lifespan.py index 222feb11..e9c2a9bc 100644 --- a/server/utils/lifespan.py +++ b/server/utils/lifespan.py @@ -4,6 +4,7 @@ from fastapi import FastAPI from src.services.task_service import tasker from src.services.mcp_service import init_mcp_servers +from src.services.run_queue_service import close_queue_clients, get_redis_client from src.storage.postgres.manager import pg_manager from src.knowledge import knowledge_base from src.utils import logger @@ -16,6 +17,7 @@ async def lifespan(app: FastAPI): try: pg_manager.initialize() await pg_manager.create_business_tables() + await pg_manager.ensure_business_schema() await pg_manager.ensure_knowledge_schema() except Exception as e: logger.error(f"Failed to initialize database during startup: {e}") diff --git a/src/agents/__init__.py b/src/agents/__init__.py index abbe0bda..23c83d81 100644 --- a/src/agents/__init__.py +++ b/src/agents/__init__.py @@ -39,9 +39,9 @@ class AgentManager(metaclass=SingletonMeta): for agent_id in self._classes.keys(): self.get_agent(agent_id, reload=True) - async def get_agents_info(self): + async def get_agents_info(self, include_configurable_items: bool = True): agents = self.get_agents() - return await asyncio.gather(*[a.get_info() for a in agents]) + return await asyncio.gather(*[a.get_info(include_configurable_items=include_configurable_items) for a in agents]) def auto_discover_agents(self): """自动发现并注册 src/agents/ 下的所有智能体。 diff --git a/src/agents/chatbot/graph.py b/src/agents/chatbot/graph.py index 99740bc1..32223288 100644 --- a/src/agents/chatbot/graph.py +++ b/src/agents/chatbot/graph.py @@ -1,9 +1,9 @@ -from deepagents.backends import StateBackend from deepagents.middleware.filesystem import FilesystemMiddleware from langchain.agents import create_agent from langchain.agents.middleware import ModelRetryMiddleware from src.agents.common import BaseAgent, load_chat_model +from src.agents.common.backends import create_agent_composite_backend from src.agents.common.middlewares import ( RuntimeConfigMiddleware, save_attachments_to_fs, @@ -13,7 +13,7 @@ from src.services.mcp_service import get_tools_from_all_servers def _create_fs_backend(rt): """创建文件存储后端""" - return StateBackend(rt) + return create_agent_composite_backend(rt) class ChatbotAgent(BaseAgent): diff --git a/src/agents/common/backends/__init__.py b/src/agents/common/backends/__init__.py new file mode 100644 index 00000000..6cb67b79 --- /dev/null +++ b/src/agents/common/backends/__init__.py @@ -0,0 +1,6 @@ +from .skills_backend import SelectedSkillsReadonlyBackend, create_agent_composite_backend + +__all__ = [ + "SelectedSkillsReadonlyBackend", + "create_agent_composite_backend", +] diff --git a/src/agents/common/backends/skills_backend.py b/src/agents/common/backends/skills_backend.py new file mode 100644 index 00000000..fb9c6617 --- /dev/null +++ b/src/agents/common/backends/skills_backend.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +from pathlib import PurePosixPath +from typing import Any + +from deepagents.backends import CompositeBackend, FilesystemBackend, StateBackend +from deepagents.backends.protocol import EditResult, FileDownloadResponse, FileUploadResponse, WriteResult + +from src.services.skill_resolver import normalize_selected_skills +from src.services.skill_service import get_skills_root_dir, is_valid_skill_slug + + +class SelectedSkillsReadonlyBackend(FilesystemBackend): + """只读 skills backend，仅暴露选中的技能目录。""" + + def __init__(self, *, selected_slugs: list[str] | None): + super().__init__(root_dir=get_skills_root_dir(), virtual_mode=True) + self._selected_slugs = { + str(slug).strip() + for slug in (selected_slugs or []) + if isinstance(slug, str) and is_valid_skill_slug(str(slug)) + } + + def _extract_slug(self, path: str | None) -> str | None: + if not path: + return None + normalized = (path or "").strip() + if not normalized or normalized == "/": + return None + pure = PurePosixPath(normalized if normalized.startswith("/") else f"/{normalized}") + parts = [p for p in pure.parts if p not in ("/", "")] + return parts[0] if parts else None + + def _is_allowed_path(self, path: str | None) -> bool: + slug = self._extract_slug(path) + if slug is None: + return True + return slug in self._selected_slugs + + def _is_allowed_file(self, file_path: str) -> bool: + slug = self._extract_slug(file_path) + return slug is not None and slug in self._selected_slugs + + def ls_info(self, path: str) -> list[dict[str, Any]]: + if not self._selected_slugs: + return [] + + normalized = (path or "/").strip() or "/" + if not self._is_allowed_path(normalized): + return [] + + infos = super().ls_info(normalized) + if normalized == "/": + result = [] + for item in infos: + slug = self._extract_slug(item.get("path", "")) + if slug in self._selected_slugs: + result.append(item) + return result + return infos + + def read(self, file_path: str, offset: int = 0, limit: int = 2000) -> str: + if not self._is_allowed_file(file_path): + return "Access denied: file is outside selected skills." + return super().read(file_path, offset=offset, limit=limit) + + def grep_raw( + self, + pattern: str, + path: str | None = None, + glob: str | None = None, + ) -> list[dict[str, Any]] | str: + if not self._selected_slugs: + return [] + + if path is not None: + if not self._is_allowed_path(path): + return "Access denied: path is outside selected skills." + return super().grep_raw(pattern=pattern, path=path, glob=glob) + + matches: list[dict[str, Any]] = [] + for slug in sorted(self._selected_slugs): + result = super().grep_raw(pattern=pattern, path=f"/{slug}", glob=glob) + if isinstance(result, str): + continue + matches.extend(result) + return matches + + def glob_info(self, pattern: str, path: str = "/") -> list[dict[str, Any]]: + if not self._selected_slugs: + return [] + if not self._is_allowed_path(path): + return [] + infos = super().glob_info(pattern=pattern, path=path) + return [item for item in infos if self._extract_slug(item.get("path", "")) in self._selected_slugs] + + def write(self, file_path: str, content: str) -> WriteResult: + return WriteResult(error="Skills path is read-only.") + + def edit(self, file_path: str, old_string: str, new_string: str, replace_all: bool = False) -> EditResult: + return EditResult(error="Skills path is read-only.") + + def upload_files(self, files: list[tuple[str, bytes]]) -> list[FileUploadResponse]: + return [FileUploadResponse(path=p, error="permission_denied") for p, _ in files] + + def download_files(self, paths: list[str]) -> list[FileDownloadResponse]: + responses: list[FileDownloadResponse] = [] + for path in paths: + if not self._is_allowed_file(path): + responses.append(FileDownloadResponse(path=path, content=None, error="invalid_path")) + continue + target = self._resolve_path(path) + if not target.exists(): + responses.append(FileDownloadResponse(path=path, content=None, error="file_not_found")) + continue + if target.is_dir(): + responses.append(FileDownloadResponse(path=path, content=None, error="is_directory")) + continue + responses.append(FileDownloadResponse(path=path, content=target.read_bytes(), error=None)) + return responses + + +def create_agent_composite_backend(runtime) -> CompositeBackend: + """为 agent 构建 backend：默认 StateBackend + /skills 路由只读 backend。""" + visible_skills = _get_visible_skills_from_runtime(runtime) + return CompositeBackend( + default=StateBackend(runtime), + routes={ + "/skills/": SelectedSkillsReadonlyBackend(selected_slugs=visible_skills), + }, + ) + + +def _get_visible_skills_from_runtime(runtime) -> list[str]: + state = getattr(runtime, "state", None) + if isinstance(state, dict): + snapshot = state.get("skill_session_snapshot") + if isinstance(snapshot, dict): + visible = snapshot.get("visible_skills") + if isinstance(visible, list): + return [slug for slug in visible if isinstance(slug, str) and is_valid_skill_slug(slug)] + + selected = getattr(runtime.context, "skills", None) or [] + return normalize_selected_skills(selected) diff --git a/src/agents/common/base.py b/src/agents/common/base.py index fe341801..d052b90a 100644 --- a/src/agents/common/base.py +++ b/src/agents/common/base.py @@ -12,6 +12,7 @@ from langgraph.graph.state import CompiledStateGraph from src import config as sys_config from src.agents.common.context import BaseContext +from src.services.skill_resolver import get_skill_options_from_db from src.utils import logger @@ -43,9 +44,15 @@ class BaseAgent: """Get the agent's class name.""" return self.__class__.__name__ - async def get_info(self): + async def get_info(self, include_configurable_items: bool = True): # Load metadata from file metadata = self.load_metadata() + configurable_items = {} + if include_configurable_items: + configurable_items = self.context_schema.get_configurable_items() + if "skills" in configurable_items: + configurable_items["skills"] = dict(configurable_items["skills"]) + configurable_items["skills"]["options"] = await get_skill_options_from_db() # Merge metadata with class attributes, metadata takes precedence return { @@ -53,7 +60,7 @@ class BaseAgent: "name": metadata.get("name", getattr(self, "name", "Unknown")), "description": metadata.get("description", getattr(self, "description", "Unknown")), "examples": metadata.get("examples", []), - "configurable_items": self.context_schema.get_configurable_items(), + "configurable_items": configurable_items, "has_checkpointer": await self.check_checkpointer(), "capabilities": getattr(self, "capabilities", []), # 智能体能力列表 } diff --git a/src/agents/common/context.py b/src/agents/common/context.py index 60440f86..6e5ba3aa 100644 --- a/src/agents/common/context.py +++ b/src/agents/common/context.py @@ -91,6 +91,16 @@ class BaseContext: }, ) + skills: Annotated[list[str], {"__template_metadata__": {"kind": "skills"}}] = field( + default_factory=list, + metadata={ + "name": "Skills", + "options": [], + "description": "可选技能列表（由超级管理员维护）。运行时仅挂载并只读暴露选中的 skills。", + "type": "list", + }, + ) + @classmethod def from_file(cls, module_name: str, input_context: dict = None) -> "BaseContext": """Load configuration from a YAML file. 用于持久化配置""" diff --git a/src/agents/common/middlewares/attachment_middleware.py b/src/agents/common/middlewares/attachment_middleware.py index 5c6136e8..0e423508 100644 --- a/src/agents/common/middlewares/attachment_middleware.py +++ b/src/agents/common/middlewares/attachment_middleware.py @@ -10,15 +10,17 @@ from typing import NotRequired from langchain.agents import AgentState from langchain.agents.middleware import AgentMiddleware, ModelRequest, ModelResponse +from langchain_core.messages import SystemMessage from src.utils import logger +ATTACHMENT_PROMPT_MARKER = "" + class AttachmentState(AgentState): """扩展 AgentState 以支持附件""" attachments: NotRequired[list[dict]] - files: NotRequired[dict[str, str]] # {"/attachments/xxx/file.md": content} def _build_attachment_prompt(attachments: Sequence[dict]) -> str | None: @@ -61,7 +63,7 @@ class AttachmentMiddleware(AgentMiddleware[AttachmentState]): LangChain 标准中间件：从 State 中读取附件并注入提示词。 LangGraph 会自动从 checkpointer 恢复 state，包括 attachments。 - 从 request.state 中读取附件，将其转换为 SystemMessage 并注入到消息列表开头。 + 从 request.state 中读取附件，将其转换为上下文块 并注入到系统提示词中。 """ state_schema = AttachmentState @@ -79,23 +81,21 @@ class AttachmentMiddleware(AgentMiddleware[AttachmentState]): if attachment_prompt: logger.info("AttachmentMiddleware: injecting attachment prompt") + existing_blocks = list(request.system_message.content_blocks) if request.system_message else [] + existing_text = "\n".join( + block.get("text", "") + for block in existing_blocks + if isinstance(block, dict) and block.get("type") == "text" + ) - messages = list(request.messages) - insert_idx = 0 - for idx, msg in enumerate(messages): - if isinstance(msg, dict): - role = msg.get("role") or msg.get("type") - is_system = role == "system" - else: - msg_type = getattr(msg, "type", None) or getattr(msg, "role", None) - is_system = msg_type == "system" + if ATTACHMENT_PROMPT_MARKER in existing_text: + logger.info("AttachmentMiddleware: attachment prompt already injected, skip") + return await handler(request) - if not is_system: - break - insert_idx = idx + 1 - - messages.insert(insert_idx, {"role": "system", "content": attachment_prompt}) - request = request.override(messages=messages) + merged_blocks = existing_blocks + [ + {"type": "text", "text": f"{ATTACHMENT_PROMPT_MARKER}\n{attachment_prompt}"} + ] + request = request.override(system_message=SystemMessage(content=merged_blocks)) return await handler(request) diff --git a/src/agents/common/middlewares/runtime_config_middleware.py b/src/agents/common/middlewares/runtime_config_middleware.py index c3808cd5..a93ecf49 100644 --- a/src/agents/common/middlewares/runtime_config_middleware.py +++ b/src/agents/common/middlewares/runtime_config_middleware.py @@ -1,18 +1,52 @@ from __future__ import annotations from collections.abc import Callable -from typing import Any +from pathlib import PurePosixPath +from typing import Annotated, Any, NotRequired +from deepagents.middleware.skills import SKILLS_SYSTEM_PROMPT +from langchain.agents import AgentState from langchain.agents.middleware import AgentMiddleware, ModelRequest, ModelResponse -from langchain_core.messages import SystemMessage +from langchain.tools.tool_node import ToolCallRequest +from langchain_core.messages import SystemMessage, ToolMessage +from langgraph.types import Command from src.agents.common import load_chat_model from src.agents.common.tools import get_buildin_tools, get_kb_based_tools from src.services.mcp_service import get_enabled_mcp_tools +from src.services.skill_resolver import ( + SkillSessionSnapshot, + build_dependency_bundle, + collect_prompt_metadata, + is_snapshot_match_selected_skills, + normalize_selected_skills, + resolve_session_snapshot, +) +from src.services.skill_service import is_valid_skill_slug from src.utils.datetime_utils import shanghai_now from src.utils.logging_config import logger +def _activated_skills_reducer(left: list[str] | None, right: list[str] | None) -> list[str]: + merged: list[str] = [] + seen: set[str] = set() + for group in (left or [], right or []): + for value in group: + if not isinstance(value, str): + continue + slug = value.strip() + if not slug or slug in seen: + continue + seen.add(slug) + merged.append(slug) + return merged + + +class RuntimeConfigState(AgentState): + activated_skills: NotRequired[Annotated[list[str], _activated_skills_reducer]] + skill_session_snapshot: NotRequired[SkillSessionSnapshot] + + class RuntimeConfigMiddleware(AgentMiddleware): """运行时配置中间件 - 应用模型/工具/知识库/MCP/提示词配置 @@ -21,6 +55,7 @@ class RuntimeConfigMiddleware(AgentMiddleware): 支持自定义上下文字段名称，以便在不同场景（如主智能体/子智能体）使用不同的配置字段 """ + state_schema = RuntimeConfigState def __init__( self, @@ -31,9 +66,12 @@ class RuntimeConfigMiddleware(AgentMiddleware): tools_context_name: str = "tools", knowledges_context_name: str = "knowledges", mcps_context_name: str = "mcps", + skills_context_name: str = "skills", enable_model_override: bool = True, enable_system_prompt_override: bool = True, enable_tools_override: bool = True, + enable_skills_prompt_override: bool = True, + skills_sources_for_prompt: list[str] | None = None, ): """初始化中间件 @@ -44,9 +82,12 @@ class RuntimeConfigMiddleware(AgentMiddleware): tools_context_name: 上下文中的工具列表字段名称（默认 "tools"） knowledges_context_name: 上下文中的知识库列表字段名称（默认 "knowledges"） mcps_context_name: 上下文中的 MCP 服务器列表字段名称（默认 "mcps"） + skills_context_name: 上下文中的 skills 列表字段名称（默认 "skills"） enable_model_override: 是否允许覆盖模型配置（默认 True） enable_system_prompt_override: 是否允许覆盖系统提示词（默认 True） enable_tools_override: 是否允许覆盖工具列表（默认 True） + enable_skills_prompt_override: 是否启用 skills 提示段注入（默认 True） + skills_sources_for_prompt: skills 来源路径（用于提示词展示，默认 ["/skills/"]） """ super().__init__() # 存储自定义字段名称 @@ -55,10 +96,13 @@ class RuntimeConfigMiddleware(AgentMiddleware): self.tools_context_name = tools_context_name self.knowledges_context_name = knowledges_context_name self.mcps_context_name = mcps_context_name + self.skills_context_name = skills_context_name # 存储覆盖配置 self.enable_model_override = enable_model_override self.enable_system_prompt_override = enable_system_prompt_override self.enable_tools_override = enable_tools_override + self.enable_skills_prompt_override = enable_skills_prompt_override + self.skills_sources_for_prompt = skills_sources_for_prompt or ["/skills/"] self.tools: list[Any] = [] # 预加载工具列表（仅当启用工具覆盖时） @@ -75,13 +119,15 @@ class RuntimeConfigMiddleware(AgentMiddleware): logger.debug( f"Initialized RuntimeConfigMiddleware with custom field names: model={model_context_name}, " f"system_prompt={system_prompt_context_name}, tools={tools_context_name}, " - f"knowledges={knowledges_context_name}, mcps={mcps_context_name}" + f"knowledges={knowledges_context_name}, mcps={mcps_context_name}, " + f"skills={skills_context_name}" ) async def awrap_model_call( self, request: ModelRequest, handler: Callable[[ModelRequest], ModelResponse] ) -> ModelResponse: runtime_context = request.runtime.context + snapshot, request = await self._ensure_skill_snapshot(request) overrides: dict[str, Any] = {} # 1. 模型覆盖（可选） @@ -91,13 +137,24 @@ class RuntimeConfigMiddleware(AgentMiddleware): # 2. 工具覆盖（可选） if self.enable_tools_override: - enabled_tools = await self.get_tools_from_context(runtime_context) + state = request.state if isinstance(request.state, dict) else {} + activated_skills = state.get("activated_skills", []) + if not isinstance(activated_skills, list): + activated_skills = [] + deps_bundle = build_dependency_bundle(snapshot, activated_skills) + enabled_tools = await self.get_tools_from_context( + runtime_context, + extra_tool_names=deps_bundle["tools"], + extra_mcps=deps_bundle["mcps"], + ) existing_tools = list(request.tools or []) + enabled_tool_names = {t.name for t in enabled_tools} + managed_tool_names = {t.name for t in self.tools} merged_tools = [] for t_bind in existing_tools: # (1) 已启用的工具保留 # (2) 非本中间件管理的工具保留 - if t_bind.name in [t.name for t in enabled_tools] or t_bind.name not in [t.name for t in self.tools]: + if t_bind.name in enabled_tool_names or t_bind.name not in managed_tool_names: merged_tools.append(t_bind) overrides["tools"] = merged_tools logger.debug(f"RuntimeConfigMiddleware selected tools: {[t.name for t in merged_tools]}") @@ -106,9 +163,24 @@ class RuntimeConfigMiddleware(AgentMiddleware): if self.enable_system_prompt_override: cur_datetime = f"当前时间：{shanghai_now().strftime('%Y-%m-%d %H:%M:%S')} UTC" system_prompt = getattr(runtime_context, self.system_prompt_context_name, "") or "" - new_content = list(request.system_message.content_blocks) + [ - {"type": "text", "text": f"{cur_datetime}\n\n{system_prompt}"} - ] + merged_system_prompt = f"{cur_datetime}\n\n{system_prompt}" + + configured_skills = getattr(runtime_context, self.skills_context_name, None) or [] + if self.enable_skills_prompt_override and configured_skills: + if self._supports_skill_prompt(request): + skills_for_prompt = configured_skills + if snapshot and isinstance(snapshot.get("visible_skills"), list): + skills_for_prompt = snapshot.get("visible_skills") or [] + skills_meta = collect_prompt_metadata(snapshot, skills_for_prompt) + skills_section = self._build_skills_section(skills_meta) + merged_system_prompt = f"{merged_system_prompt}\n\n{skills_section}" + else: + logger.warning( + "RuntimeConfigMiddleware: skills configured but read_file unavailable, skip skills prompt" + ) + + content_blocks = list(request.system_message.content_blocks) if request.system_message else [] + new_content = content_blocks + [{"type": "text", "text": merged_system_prompt}] new_system_message = SystemMessage(content=new_content) overrides["system_message"] = new_system_message @@ -117,17 +189,36 @@ class RuntimeConfigMiddleware(AgentMiddleware): return await handler(request) - async def get_tools_from_context(self, context) -> list: + async def get_tools_from_context( + self, + context, + *, + extra_tool_names: list[str] | None = None, + extra_mcps: list[str] | None = None, + ) -> list: """从上下文配置中获取工具列表""" selected_tools = [] + selected_tool_names: set[str] = set() # 1. 基础工具 (从 context.tools 中筛选) - tools = getattr(context, self.tools_context_name, None) - if tools: - tools_map = {t.name: t for t in self.tools} - for tool_name in tools: - if tool_name in tools_map: - selected_tools.append(tools_map[tool_name]) + tools = getattr(context, self.tools_context_name, None) or [] + all_tool_names = [] + for tool_name in tools: + if isinstance(tool_name, str): + all_tool_names.append(tool_name) + for tool_name in extra_tool_names or []: + if isinstance(tool_name, str): + all_tool_names.append(tool_name) + + tools_map = {t.name: t for t in self.tools} + for tool_name in all_tool_names: + if tool_name in selected_tool_names: + continue + if tool_name in tools_map: + selected_tools.append(tools_map[tool_name]) + selected_tool_names.add(tool_name) + continue + logger.warning(f"RuntimeConfigMiddleware: tool dependency not found, skip: {tool_name}") # 2. 知识库工具 knowledges = getattr(context, self.knowledges_context_name, None) @@ -136,10 +227,175 @@ class RuntimeConfigMiddleware(AgentMiddleware): selected_tools.extend(kb_tools) # 3. MCP 工具（使用统一入口，自动过滤 disabled_tools） - mcps = getattr(context, self.mcps_context_name, None) - if mcps: - for server_name in mcps: + mcps = getattr(context, self.mcps_context_name, None) or [] + all_mcp_names: list[str] = [] + for server_name in mcps: + if isinstance(server_name, str): + all_mcp_names.append(server_name) + for server_name in extra_mcps or []: + if isinstance(server_name, str): + all_mcp_names.append(server_name) + + selected_mcp_servers: set[str] = set() + for server_name in all_mcp_names: + if server_name in selected_mcp_servers: + continue + selected_mcp_servers.add(server_name) + try: mcp_tools = await get_enabled_mcp_tools(server_name) + if not mcp_tools: + logger.warning(f"RuntimeConfigMiddleware: mcp dependency unavailable, skip: {server_name}") selected_tools.extend(mcp_tools) + except Exception as e: + logger.warning(f"RuntimeConfigMiddleware: failed to load mcp dependency '{server_name}': {e}") return selected_tools + + async def awrap_tool_call( + self, + request: ToolCallRequest, + handler: Callable[[ToolCallRequest], Any], + ): + result = await handler(request) + if request.tool_call.get("name") != "read_file": + return result + + args = request.tool_call.get("args") or {} + file_path = args.get("file_path") if isinstance(args, dict) else None + slug = self._extract_skill_slug_from_skill_md_path(file_path) + if not slug: + return result + if not self._is_visible_skill_slug(request, slug): + logger.warning(f"RuntimeConfigMiddleware: deny skill activation for invisible slug: {slug}") + return result + logger.debug(f"RuntimeConfigMiddleware: activated skill by read_file: {slug}") + return self._merge_activated_skill_update(result, slug) + + def wrap_tool_call( + self, + request: ToolCallRequest, + handler: Callable[[ToolCallRequest], Any], + ): + result = handler(request) + if request.tool_call.get("name") != "read_file": + return result + + args = request.tool_call.get("args") or {} + file_path = args.get("file_path") if isinstance(args, dict) else None + slug = self._extract_skill_slug_from_skill_md_path(file_path) + if not slug: + return result + if not self._is_visible_skill_slug(request, slug): + logger.warning(f"RuntimeConfigMiddleware: deny skill activation for invisible slug: {slug}") + return result + logger.debug(f"RuntimeConfigMiddleware: activated skill by read_file: {slug}") + return self._merge_activated_skill_update(result, slug) + + async def _ensure_skill_snapshot(self, request: ModelRequest) -> tuple[SkillSessionSnapshot | None, ModelRequest]: + runtime_context = request.runtime.context + configured_skills = getattr(runtime_context, self.skills_context_name, None) or [] + normalized_skills = normalize_selected_skills(configured_skills) + state = request.state if isinstance(request.state, dict) else {} + + snapshot = self._get_skill_snapshot_from_state(state) + if is_snapshot_match_selected_skills(snapshot, normalized_skills): + return snapshot, request + + try: + snapshot = await resolve_session_snapshot(normalized_skills) + except Exception as e: + logger.warning(f"RuntimeConfigMiddleware: failed to resolve skill snapshot, fallback empty: {e}") + snapshot = None + + if isinstance(request.state, dict): + if snapshot: + request.state["skill_session_snapshot"] = snapshot + else: + request.state.pop("skill_session_snapshot", None) + + return snapshot, request + + def _extract_skill_slug_from_skill_md_path(self, file_path: Any) -> str | None: + if not isinstance(file_path, str): + return None + raw = file_path.strip() + if not raw: + return None + pure = PurePosixPath(raw if raw.startswith("/") else f"/{raw}") + parts = [p for p in pure.parts if p not in ("/", "")] + if len(parts) != 3: + return None + if parts[0] != "skills" or parts[2] != "SKILL.md": + return None + slug = parts[1] + if not is_valid_skill_slug(slug): + return None + return slug + + def _merge_activated_skill_update(self, result: Any, slug: str): + if isinstance(result, Command): + update = dict(result.update or {}) + current = update.get("activated_skills") or [] + update["activated_skills"] = _activated_skills_reducer(current, [slug]) + return Command(graph=result.graph, update=update, resume=result.resume, goto=result.goto) + + if isinstance(result, ToolMessage): + return Command(update={"messages": [result], "activated_skills": [slug]}) + + return result + + def _is_visible_skill_slug(self, request: ToolCallRequest, slug: str) -> bool: + snapshot = self._get_skill_snapshot_from_state(request.state) + if snapshot: + visible_skills = snapshot.get("visible_skills") + if isinstance(visible_skills, list): + return slug in visible_skills + + configured_skills = getattr(request.runtime.context, self.skills_context_name, None) or [] + normalized = normalize_selected_skills(configured_skills) + return slug in normalized + + def _get_skill_snapshot_from_state(self, state: Any) -> SkillSessionSnapshot | None: + if not isinstance(state, dict): + return None + snapshot = state.get("skill_session_snapshot") + if not isinstance(snapshot, dict): + return None + visible_skills = snapshot.get("visible_skills") + selected_skills = snapshot.get("selected_skills") + if not isinstance(visible_skills, list) or not isinstance(selected_skills, list): + return None + return snapshot + + def _supports_skill_prompt(self, request: ModelRequest) -> bool: + """仅当请求工具中包含 read_file 时，才注入 skills 指引。""" + for tool in request.tools or []: + if getattr(tool, "name", None) == "read_file": + return True + return False + + def _format_skills_locations(self, sources: list[str]) -> str: + locations = [] + for i, source_path in enumerate(sources): + name = PurePosixPath(source_path.rstrip("/")).name.capitalize() + suffix = " (higher priority)" if i == len(sources) - 1 else "" + locations.append(f"**{name} Skills**: `{source_path}`{suffix}") + return "\n".join(locations) + + def _format_skills_list(self, skills_meta: list[dict[str, str]]) -> str: + if not skills_meta: + return f"(No skills available yet. You can create skills in {' or '.join(self.skills_sources_for_prompt)})" + + lines = [] + for skill in skills_meta: + lines.append(f"- **{skill['name']}**: {skill['description']}") + lines.append(f" -> Read `{skill['path']}` for full instructions") + return "\n".join(lines) + + def _build_skills_section(self, skills_meta: list[dict[str, str]]) -> str: + skills_locations = self._format_skills_locations(self.skills_sources_for_prompt) + skills_list = self._format_skills_list(skills_meta) + return SKILLS_SYSTEM_PROMPT.format( + skills_locations=skills_locations, + skills_list=skills_list, + ) diff --git a/src/agents/deep_agent/graph.py b/src/agents/deep_agent/graph.py index b3b6af01..68e99500 100644 --- a/src/agents/deep_agent/graph.py +++ b/src/agents/deep_agent/graph.py @@ -1,6 +1,5 @@ """Deep Agent - 基于create_deep_agent的深度分析智能体""" -from deepagents.backends import StateBackend from deepagents.middleware.filesystem import FilesystemMiddleware from deepagents.middleware.patch_tool_calls import PatchToolCallsMiddleware from deepagents.middleware.subagents import SubAgentMiddleware @@ -10,16 +9,18 @@ from langchain.agents.middleware import ( ) from src.agents.common import BaseAgent, load_chat_model +from src.agents.common.backends import create_agent_composite_backend from src.agents.common.middlewares import RuntimeConfigMiddleware, SummaryOffloadMiddleware, save_attachments_to_fs from src.agents.common.tools import get_tavily_search from src.services.mcp_service import get_tools_from_all_servers + from .context import DeepContext def _create_fs_backend(rt): """创建文件存储后端""" - return StateBackend(rt) + return create_agent_composite_backend(rt) def _get_research_sub_agent(search_tools: list) -> dict: diff --git a/src/knowledge/base.py b/src/knowledge/base.py index d66fbe57..ebfc09c6 100644 --- a/src/knowledge/base.py +++ b/src/knowledge/base.py @@ -3,6 +3,10 @@ import os from abc import ABC, abstractmethod from typing import Any +from src.knowledge.chunking.ragflow_like.presets import ( + ensure_chunk_defaults_in_additional_params, + resolve_chunk_processing_params, +) from src.utils import logger from src.utils.datetime_utils import coerce_any_to_utc_datetime, utc_isoformat @@ -76,6 +80,7 @@ class KnowledgeBase(ABC): self.databases_meta = {} for db_id, meta in global_databases_meta.items(): if meta.get("kb_type") == self.kb_type: + normalized_additional_params = ensure_chunk_defaults_in_additional_params(meta.get("additional_params")) self.databases_meta[db_id] = { "name": meta.get("name"), "description": meta.get("description"), @@ -83,7 +88,7 @@ class KnowledgeBase(ABC): "embed_info": meta.get("embed_info"), "llm_info": meta.get("llm_info"), "query_params": meta.get("query_params"), - "metadata": meta.get("additional_params", {}), + "metadata": normalized_additional_params, "created_at": meta.get("created_at"), } @@ -91,7 +96,14 @@ class KnowledgeBase(ABC): self.files_meta = {} for file_id, meta in files_meta.items(): if meta.get("database_id") in self.databases_meta: - self.files_meta[file_id] = meta + db_id = meta.get("database_id") + kb_additional_params = self.databases_meta.get(db_id, {}).get("metadata") or {} + normalized_meta = dict(meta) + normalized_meta["processing_params"] = resolve_chunk_processing_params( + kb_additional_params=kb_additional_params, + file_processing_params=meta.get("processing_params"), + ) + self.files_meta[file_id] = normalized_meta # 过滤评估基准 self.benchmarks_meta = {} @@ -199,6 +211,11 @@ class KnowledgeBase(ABC): # Prepare metadata metadata = await prepare_item_metadata(item, content_type, db_id, params=params) file_id = metadata["file_id"] + kb_additional_params = self.databases_meta.get(db_id, {}).get("metadata") or {} + metadata["processing_params"] = resolve_chunk_processing_params( + kb_additional_params=kb_additional_params, + file_processing_params=metadata.get("processing_params"), + ) # Initial status metadata["status"] = FileStatus.UPLOADED @@ -323,13 +340,16 @@ class KnowledgeBase(ABC): if not params: return - # Merge or overwrite? Usually merge is safer, or replace. - # User might want to change chunk size. current_params = self.files_meta[file_id].get("processing_params", {}) or {} + kb_additional_params = self.databases_meta.get(db_id, {}).get("metadata") or {} logger.debug(f"[update_file_params] file_id={file_id}, current_params={current_params}, new_params={params}") - current_params.update(params) + current_params = resolve_chunk_processing_params( + kb_additional_params=kb_additional_params, + file_processing_params=current_params, + request_params=params, + ) self.files_meta[file_id]["processing_params"] = current_params self.files_meta[file_id]["updated_at"] = utc_isoformat() @@ -414,6 +434,8 @@ class KnowledgeBase(ABC): """ from src.utils import hashstr + kwargs = ensure_chunk_defaults_in_additional_params(kwargs) + # 从 kwargs 中获取 is_private 配置 is_private = kwargs.get("is_private", False) prefix = "kb_private_" if is_private else "kb_" @@ -982,7 +1004,7 @@ class KnowledgeBase(ABC): "embed_info": kb.embed_info, "llm_info": kb.llm_info, "query_params": kb.query_params, - "metadata": kb.additional_params or {}, + "metadata": ensure_chunk_defaults_in_additional_params(kb.additional_params), "created_at": utc_isoformat(kb.created_at) if kb.created_at else utc_isoformat(), } for kb in databases @@ -990,6 +1012,7 @@ class KnowledgeBase(ABC): self.files_meta = {} for kb in databases: + kb_additional_params = self.databases_meta.get(kb.db_id, {}).get("metadata") or {} for record in await file_repo.list_by_db_id(kb.db_id): self.files_meta[record.file_id] = { "file_id": record.file_id, @@ -1003,7 +1026,10 @@ class KnowledgeBase(ABC): "content_hash": record.content_hash, "size": record.file_size, "content_type": record.content_type, - "processing_params": record.processing_params, + "processing_params": resolve_chunk_processing_params( + kb_additional_params=kb_additional_params, + file_processing_params=record.processing_params, + ), "is_folder": record.is_folder, "error": record.error_message, "created_by": record.created_by, diff --git a/src/knowledge/chunking/__init__.py b/src/knowledge/chunking/__init__.py new file mode 100644 index 00000000..a9a2c5b3 --- /dev/null +++ b/src/knowledge/chunking/__init__.py @@ -0,0 +1 @@ +__all__ = [] diff --git a/src/knowledge/chunking/ragflow_like/__init__.py b/src/knowledge/chunking/ragflow_like/__init__.py new file mode 100644 index 00000000..8200a554 --- /dev/null +++ b/src/knowledge/chunking/ragflow_like/__init__.py @@ -0,0 +1,3 @@ +from src.knowledge.chunking.ragflow_like.dispatcher import chunk_file, chunk_markdown + +__all__ = ["chunk_file", "chunk_markdown"] diff --git a/src/knowledge/chunking/ragflow_like/dispatcher.py b/src/knowledge/chunking/ragflow_like/dispatcher.py new file mode 100644 index 00000000..2c845077 --- /dev/null +++ b/src/knowledge/chunking/ragflow_like/dispatcher.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +from typing import Any + +from src.knowledge.chunking.ragflow_like.parsers import book, general, laws, qa +from src.knowledge.chunking.ragflow_like.presets import map_to_internal_parser_id, normalize_chunk_preset_id + + +def _build_chunk_records(text_chunks: list[str], file_id: str, filename: str) -> list[dict[str, Any]]: + records: list[dict[str, Any]] = [] + + for idx, chunk_content in enumerate(text_chunks): + text = (chunk_content or "").strip() + if not text: + continue + + records.append( + { + "id": f"{file_id}_chunk_{idx}", + "content": text, + "file_id": file_id, + "filename": filename, + "chunk_index": idx, + "source": filename, + "chunk_id": f"{file_id}_chunk_{idx}", + } + ) + + return records + + +def _dispatch_markdown_parser(preset_id: str, filename: str, markdown_content: str, parser_config: dict[str, Any]) -> list[str]: + parser_id = map_to_internal_parser_id(preset_id) + + if parser_id == "naive": + return general.chunk_markdown(markdown_content, parser_config) + if parser_id == "qa": + return qa.chunk_markdown(filename, markdown_content, parser_config) + if parser_id == "book": + return book.chunk_markdown(markdown_content, parser_config) + if parser_id == "laws": + return laws.chunk_markdown(filename, markdown_content, parser_config) + + return general.chunk_markdown(markdown_content, parser_config) + + +def chunk_markdown(markdown_content: str, file_id: str, filename: str, processing_params: dict[str, Any]) -> list[dict[str, Any]]: + params = dict(processing_params or {}) + preset_id = normalize_chunk_preset_id(params.get("chunk_preset_id")) + parser_config = params.get("chunk_parser_config") if isinstance(params.get("chunk_parser_config"), dict) else {} + + text_chunks = _dispatch_markdown_parser(preset_id, filename, markdown_content, parser_config) + return _build_chunk_records(text_chunks, file_id, filename) + + +def chunk_file(file_content: str, file_id: str, filename: str, processing_params: dict[str, Any]) -> list[dict[str, Any]]: + # 当前链路中入库前均已转换为 markdown，因此与 chunk_markdown 保持同实现。 + return chunk_markdown(file_content, file_id, filename, processing_params) diff --git a/src/knowledge/chunking/ragflow_like/nlp.py b/src/knowledge/chunking/ragflow_like/nlp.py new file mode 100644 index 00000000..dd3196e2 --- /dev/null +++ b/src/knowledge/chunking/ragflow_like/nlp.py @@ -0,0 +1,536 @@ +from __future__ import annotations + +import random +import re +from collections import Counter +from dataclasses import dataclass, field + + +BULLET_PATTERN = [ + [ + r"第[零一二三四五六七八九十百0-9]+(分?编|部分)", + r"第[零一二三四五六七八九十百0-9]+章", + r"第[零一二三四五六七八九十百0-9]+节", + r"第[零一二三四五六七八九十百0-9]+条", + r"[\(（][零一二三四五六七八九十百]+[\)）]", + ], + [ + r"第[0-9]+章", + r"第[0-9]+节", + r"[0-9]{,2}[\. 、]", + r"[0-9]{,2}\.[0-9]{,2}[^a-zA-Z/%~-]", + r"[0-9]{,2}\.[0-9]{,2}\.[0-9]{,2}", + r"[0-9]{,2}\.[0-9]{,2}\.[0-9]{,2}\.[0-9]{,2}", + ], + [ + r"第[零一二三四五六七八九十百0-9]+章", + r"第[零一二三四五六七八九十百0-9]+节", + r"[零一二三四五六七八九十百]+[ 、]", + r"[\(（][零一二三四五六七八九十百]+[\)）]", + r"[\(（][0-9]{,2}[\)）]", + ], + [ + r"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)", + r"Chapter (I+V?|VI*|XI|IX|X)", + r"Section [0-9]+", + r"Article [0-9]+", + ], + [ + r"^#[^#]", + r"^##[^#]", + r"^###.*", + r"^####.*", + r"^#####.*", + r"^######.*", + ], +] + +MARKDOWN_BULLET_GROUP_INDEX = 4 + + +def count_tokens(text: str) -> int: + """近似 token 计数，避免引入额外依赖。""" + if not text: + return 0 + # 英文单词 + 数字 + CJK 单字 + parts = re.findall(r"[A-Za-z0-9_]+|[\u4e00-\u9fff]", text) + return max(1, len(parts)) if text.strip() else 0 + + +def random_choices(arr: list[str], k: int) -> list[str]: + if not arr: + return [] + return random.choices(arr, k=min(len(arr), k)) + + +def is_english(texts: str | list[str]) -> bool: + if not texts: + return False + + patt = re.compile(r"[`a-zA-Z0-9\s.,':;/\"?<>!\(\)\-]+") + if isinstance(texts, str): + seq = [texts] + else: + seq = [t for t in texts if isinstance(t, str) and t.strip()] + + if not seq: + return False + + hits = sum(1 for t in seq if patt.fullmatch(t.strip())) + return (hits / len(seq)) > 0.8 + + +def not_bullet(line: str) -> bool: + patt = [ + r"0", + r"[0-9]+ +[0-9~个只-]", + r"[0-9]+\.{2,}", + ] + return any(re.match(p, line) for p in patt) + + +def is_probable_heading_line(line: str) -> bool: + text = (line or "").strip() + if not text: + return False + + if re.match(r"^#{1,6}\s+\S", text): + return True + + # 表格/HTML 残留通常不是标题。 + if re.search(r"]*>", text, flags=re.IGNORECASE): + return False + + # 超长行基本是正文或条款，不是章节标题。 + if len(text) > 96: + return False + if count_tokens(text) > 72: + return False + + # 标题前段通常不会出现明显句号/逗号；出现则大概率是正文。 + if re.search(r"[，。；！？!?:：]", text[:24]): + return False + + if text.endswith(("。", "；", "！", "!", "？", "?")) and len(text) > 20: + return False + + return True + + +def _is_mid_sentence_bullet(line: str) -> bool: + text = (line or "").strip() + if not text: + return False + + if re.match(r"^#{1,6}\s+\S", text): + return False + + marker = re.search( + r"([一二三四五六七八九十百]+、|[\(（][一二三四五六七八九十百]+[\)）]|[0-9]{1,2}[\.、])", + text, + ) + if not marker: + return False + + if marker.start() == 0: + return False + + prev = text[marker.start() - 1] + return prev not in {"#", "\n"} + + +def bullets_category(sections: list[str]) -> int: + hits: list[float] = [0.0] * len(BULLET_PATTERN) + + def bullet_weight(group_idx: int, line: str) -> float: + # 对 markdown 标题候选增加权重，避免“正文里的 一、/（一）”压过真正的 # 标题层级。 + if group_idx != MARKDOWN_BULLET_GROUP_INDEX: + return 1.0 + + heading = line.strip() + if not re.match(r"^#{1,6}\s+\S", heading): + return 1.0 + + level = len(heading) - len(heading.lstrip("#")) + if level <= 2: + return 4.0 + if level <= 4: + return 3.0 + return 2.0 + + for i, pro in enumerate(BULLET_PATTERN): + for sec in sections: + sec = sec.strip() + for p in pro: + if re.match(p, sec) and not not_bullet(sec): + w = bullet_weight(i, sec) + if _is_mid_sentence_bullet(sec): + w *= 0.1 + if i != MARKDOWN_BULLET_GROUP_INDEX and not is_probable_heading_line(sec): + w *= 0.2 + hits[i] += w + break + maximum = 0 + res = -1 + for i, hit in enumerate(hits): + if hit <= maximum: + continue + res = i + maximum = hit + return res + + +def _get_text(section: str | tuple[str, str]) -> str: + if isinstance(section, str): + return section.strip() + return (section[0] or "").strip() + + +def remove_contents_table(sections: list[str] | list[tuple[str, str]], eng: bool = False) -> None: + i = 0 + while i < len(sections): + line = re.sub(r"( |　|\u3000)+", "", _get_text(sections[i]).split("@@")[0], flags=re.IGNORECASE) + if not re.match(r"(contents|目录|目次|tableofcontents|致谢|acknowledge)$", line, flags=re.IGNORECASE): + i += 1 + continue + + sections.pop(i) + if i >= len(sections): + break + + prefix = _get_text(sections[i])[:3] if not eng else " ".join(_get_text(sections[i]).split()[:2]) + while not prefix and i < len(sections): + sections.pop(i) + if i >= len(sections): + break + prefix = _get_text(sections[i])[:3] if not eng else " ".join(_get_text(sections[i]).split()[:2]) + + if i >= len(sections) or not prefix: + break + + sections.pop(i) + if i >= len(sections): + break + + for j in range(i, min(i + 128, len(sections))): + if not re.match(re.escape(prefix), _get_text(sections[j])): + continue + for _ in range(i, j): + sections.pop(i) + break + + +def make_colon_as_title(sections: list[str] | list[tuple[str, str]]) -> list[str] | list[tuple[str, str]]: + if not sections: + return sections + if isinstance(sections[0], str): + return sections + + i = 0 + while i < len(sections): + text, layout = sections[i] + i += 1 + text = text.split("@")[0].strip() + if not text or text[-1] not in ":：": + continue + + rev = text[::-1] + arr = re.split(r"([。？！!?;；]| \.)", rev) + if len(arr) < 2 or len(arr[1]) < 32: + continue + + sections.insert(i - 1, (arr[0][::-1], "title")) + i += 1 + + return sections + + +def not_title(text: str) -> bool: + if re.match(r"第[零一二三四五六七八九十百0-9]+条", text): + return False + if len(text.split()) > 12 or (" " not in text and len(text) >= 32): + return True + return bool(re.search(r"[,;，。；！!]", text)) + + +def tree_merge(bull: int, sections: list[str] | list[tuple[str, str]], depth: int) -> list[str]: + if not sections or bull < 0: + return [s if isinstance(s, str) else s[0] for s in sections] + + if isinstance(sections[0], str): + typed_sections: list[tuple[str, str]] = [(s, "") for s in sections] + else: + typed_sections = sections # type: ignore[assignment] + + typed_sections = [ + (t, o) + for t, o in typed_sections + if t and len(t.split("@")[0].strip()) > 1 and not re.match(r"[0-9]+$", t.split("@")[0].strip()) + ] + + def get_level(section: tuple[str, str]) -> tuple[int, str]: + text, layout = section + text = re.sub(r"\u3000", " ", text).strip() + + for i, patt in enumerate(BULLET_PATTERN[bull]): + if re.match(patt, text) and is_probable_heading_line(text): + return i + 1, text + + if re.search(r"(title|head)", layout) and not not_title(text): + return len(BULLET_PATTERN[bull]) + 1, text + + return len(BULLET_PATTERN[bull]) + 2, text + + lines: list[tuple[int, str]] = [] + level_set: set[int] = set() + for section in typed_sections: + level, text = get_level(section) + if not text.strip("\n"): + continue + lines.append((level, text)) + level_set.add(level) + + if not lines: + return [] + + sorted_levels = sorted(level_set) + target_level = sorted_levels[depth - 1] if depth <= len(sorted_levels) else sorted_levels[-1] + + max_body_level = len(BULLET_PATTERN[bull]) + 2 + if target_level == max_body_level: + target_level = sorted_levels[-2] if len(sorted_levels) > 1 else sorted_levels[0] + + root = Node(level=0, depth=target_level, texts=[]) + root.build_tree(lines) + return [item for item in root.get_tree() if item] + + +def hierarchical_merge(bull: int, sections: list[str] | list[tuple[str, str]], depth: int) -> list[list[str]]: + if not sections or bull < 0: + return [] + + if isinstance(sections[0], str): + typed_sections: list[tuple[str, str]] = [(s, "") for s in sections] + else: + typed_sections = sections # type: ignore[assignment] + + typed_sections = [ + (t, o) + for t, o in typed_sections + if t and len(t.split("@")[0].strip()) > 1 and not re.match(r"[0-9]+$", t.split("@")[0].strip()) + ] + + bullets_size = len(BULLET_PATTERN[bull]) + levels: list[list[int]] = [[] for _ in range(bullets_size + 2)] + + for i, (text, layout) in enumerate(typed_sections): + for j, patt in enumerate(BULLET_PATTERN[bull]): + if re.match(patt, text.strip()) and is_probable_heading_line(text): + levels[j].append(i) + break + else: + if re.search(r"(title|head)", layout) and not not_title(text): + levels[bullets_size].append(i) + else: + levels[bullets_size + 1].append(i) + + pure_sections = [t for t, _ in typed_sections] + + def binary_search(arr: list[int], target: int) -> int: + if not arr: + return -1 + if target > arr[-1]: + return len(arr) - 1 + if target < arr[0]: + return -1 + + s, e = 0, len(arr) + while e - s > 1: + mid = (e + s) // 2 + if target > arr[mid]: + s = mid + elif target < arr[mid]: + e = mid + else: + return mid + return s + + cks: list[list[int]] = [] + readed = [False] * len(pure_sections) + levels = list(reversed(levels)) + for i, arr in enumerate(levels[:depth]): + for j in arr: + if readed[j]: + continue + readed[j] = True + cks.append([j]) + if i + 1 == len(levels) - 1: + continue + + for ii in range(i + 1, len(levels)): + jj = binary_search(levels[ii], j) + if jj < 0: + continue + if levels[ii][jj] > cks[-1][-1]: + cks[-1].pop(-1) + cks[-1].append(levels[ii][jj]) + + for ii in cks[-1]: + readed[ii] = True + + if not cks: + return [] + + for i in range(len(cks)): + cks[i] = [pure_sections[j] for j in reversed(cks[i])] + + res: list[list[str]] = [[]] + num = [0] + for ck in cks: + if len(ck) == 1: + n = count_tokens(re.sub(r"@@[0-9]+.*", "", ck[0])) + if n + num[-1] < 218: + res[-1].append(ck[0]) + num[-1] += n + continue + res.append(ck) + num.append(n) + continue + res.append(ck) + num.append(218) + + return [chunk for chunk in res if chunk] + + +def _remove_pdf_tags(text: str) -> str: + return re.sub(r"@@[0-9-]+\t[0-9.\t]+##", "", text or "") + + +def _extract_custom_delimiters(delimiter: str) -> list[str]: + return [m.group(1) for m in re.finditer(r"`([^`]+)`", delimiter or "")] + + +def naive_merge( + sections: str | list[str] | list[tuple[str, str]], + chunk_token_num: int = 128, + delimiter: str = "\n。；！？", + overlapped_percent: int = 0, +) -> list[str]: + if not sections: + return [] + + if isinstance(sections, str): + typed_sections: list[tuple[str, str]] = [(sections, "")] + elif isinstance(sections[0], str): + typed_sections = [(s, "") for s in sections] # type: ignore[index] + else: + typed_sections = sections # type: ignore[assignment] + + chunk_token_num = max(int(chunk_token_num or 0), 0) + overlap = max(0, min(int(overlapped_percent or 0), 99)) + + custom_delimiters = _extract_custom_delimiters(delimiter) + if custom_delimiters: + pattern = "|".join(re.escape(t) for t in sorted(set(custom_delimiters), key=len, reverse=True)) + chunks: list[str] = [] + for sec, pos in typed_sections: + split_sec = re.split(rf"({pattern})", sec, flags=re.DOTALL) + for sub in split_sec: + if re.fullmatch(pattern, sub or ""): + continue + text = "\n" + sub + local_pos = pos if count_tokens(text) >= 8 else "" + if local_pos and local_pos not in text: + text += local_pos + if text.strip(): + chunks.append(text) + return chunks + + if chunk_token_num <= 0: + merged = "\n".join(sec for sec, _ in typed_sections if sec and sec.strip()) + return [merged] if merged.strip() else [] + + chunks = [""] + token_nums = [0] + + def add_chunk(text: str, pos: str) -> None: + tnum = count_tokens(text) + local_pos = pos or "" + if tnum < 8: + local_pos = "" + + threshold = chunk_token_num * (100 - overlap) / 100.0 + if chunks[-1] == "" or token_nums[-1] > threshold: + if chunks: + prev = _remove_pdf_tags(chunks[-1]) + start = int(len(prev) * (100 - overlap) / 100.0) + text = prev[start:] + text + if local_pos and local_pos not in text: + text += local_pos + chunks.append(text) + token_nums.append(tnum) + else: + if local_pos and local_pos not in chunks[-1]: + text += local_pos + chunks[-1] += text + token_nums[-1] += tnum + + for sec, pos in typed_sections: + if not sec: + continue + add_chunk("\n" + sec, pos) + + return [chunk for chunk in chunks if chunk.strip()] + + +@dataclass +class Node: + level: int + depth: int = -1 + texts: list[str] = field(default_factory=list) + children: list["Node"] = field(default_factory=list) + + def add_child(self, child_node: "Node") -> None: + self.children.append(child_node) + + def add_text(self, text: str) -> None: + self.texts.append(text) + + def build_tree(self, lines: list[tuple[int, str]]) -> "Node": + stack: list[Node] = [self] + for level, text in lines: + if self.depth != -1 and level > self.depth: + stack[-1].add_text(text) + continue + + while len(stack) > 1 and level <= stack[-1].level: + stack.pop() + + node = Node(level=level, texts=[text]) + stack[-1].add_child(node) + stack.append(node) + + return self + + def get_tree(self) -> list[str]: + tree_list: list[str] = [] + self._dfs(self, tree_list, []) + return tree_list + + def _dfs(self, node: "Node", tree_list: list[str], titles: list[str]) -> None: + level = node.level + texts = node.texts + child = node.children + + if level == 0 and texts: + tree_list.append("\n".join(titles + texts)) + + path_titles = titles + texts if 1 <= level <= self.depth else titles + + if level > self.depth and texts: + tree_list.append("\n".join(path_titles + texts)) + elif not child and (1 <= level <= self.depth): + tree_list.append("\n".join(path_titles)) + + for c in child: + self._dfs(c, tree_list, path_titles) diff --git a/src/knowledge/chunking/ragflow_like/parsers/__init__.py b/src/knowledge/chunking/ragflow_like/parsers/__init__.py new file mode 100644 index 00000000..9f250470 --- /dev/null +++ b/src/knowledge/chunking/ragflow_like/parsers/__init__.py @@ -0,0 +1,3 @@ +from src.knowledge.chunking.ragflow_like.parsers import book, general, laws, qa + +__all__ = ["general", "qa", "book", "laws"] diff --git a/src/knowledge/chunking/ragflow_like/parsers/book.py b/src/knowledge/chunking/ragflow_like/parsers/book.py new file mode 100644 index 00000000..389e8ca5 --- /dev/null +++ b/src/knowledge/chunking/ragflow_like/parsers/book.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +from typing import Any + +from src.knowledge.chunking.ragflow_like import nlp + + +def _unescape_delimiter(delimiter: str) -> str: + return delimiter.replace("\\n", "\n").replace("\\r", "\r").replace("\\t", "\t").replace("\\\\", "\\") + + +def _iter_sections(markdown_content: str) -> list[tuple[str, str]]: + sections: list[tuple[str, str]] = [] + for line in (markdown_content or "").splitlines(): + text = line.strip() + if not text: + continue + sections.append((text, "")) + + if not sections and markdown_content and markdown_content.strip(): + sections.append((markdown_content.strip(), "")) + + return sections + + +def chunk_markdown(markdown_content: str, parser_config: dict[str, Any] | None = None) -> list[str]: + parser_config = parser_config or {} + + delimiter = _unescape_delimiter(str(parser_config.get("delimiter", "\n") or "\n")) + chunk_token_num = int(parser_config.get("chunk_token_num", 512) or 512) + overlapped_percent = int(parser_config.get("overlapped_percent", 0) or 0) + + sections = _iter_sections(markdown_content) + if not sections: + return [] + + section_texts = [text for text, _ in sections] + nlp.remove_contents_table(sections, eng=nlp.is_english(nlp.random_choices(section_texts, k=200))) + nlp.make_colon_as_title(sections) + + bull = nlp.bullets_category([t for t in nlp.random_choices([t for t, _ in sections], k=100)]) + + if bull >= 0: + chunks = ["\n".join(ck) for ck in nlp.hierarchical_merge(bull, sections, depth=5)] + else: + chunks = nlp.naive_merge( + sections, + chunk_token_num=chunk_token_num, + delimiter=delimiter, + overlapped_percent=overlapped_percent, + ) + + if chunks: + return chunks + + return nlp.naive_merge( + sections, + chunk_token_num=chunk_token_num, + delimiter=delimiter, + overlapped_percent=overlapped_percent, + ) diff --git a/src/knowledge/chunking/ragflow_like/parsers/general.py b/src/knowledge/chunking/ragflow_like/parsers/general.py new file mode 100644 index 00000000..0ed01d63 --- /dev/null +++ b/src/knowledge/chunking/ragflow_like/parsers/general.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +from typing import Any + +from src.knowledge.chunking.ragflow_like import nlp + + +def _unescape_delimiter(delimiter: str) -> str: + return delimiter.replace("\\n", "\n").replace("\\r", "\r").replace("\\t", "\t").replace("\\\\", "\\") + + +def _iter_sections(markdown_content: str, delimiter: str) -> list[tuple[str, str]]: + sections: list[tuple[str, str]] = [] + text = markdown_content or "" + if delimiter and delimiter not in {"\n", "\r\n"} and "`" not in delimiter: + for part in text.split(delimiter): + block = part.strip() + if block: + sections.append((block, "")) + else: + for line in text.splitlines(): + block = line.strip() + if not block: + continue + sections.append((block, "")) + + if not sections and text.strip(): + sections.append((text.strip(), "")) + + return sections + + +def chunk_markdown(markdown_content: str, parser_config: dict[str, Any] | None = None) -> list[str]: + parser_config = parser_config or {} + + delimiter = _unescape_delimiter(str(parser_config.get("delimiter", "\n") or "\n")) + chunk_token_num = int(parser_config.get("chunk_token_num", 512) or 512) + overlapped_percent = int(parser_config.get("overlapped_percent", 0) or 0) + + sections = _iter_sections(markdown_content, delimiter) + return nlp.naive_merge( + sections, + chunk_token_num=chunk_token_num, + delimiter=delimiter, + overlapped_percent=overlapped_percent, + ) diff --git a/src/knowledge/chunking/ragflow_like/parsers/laws.py b/src/knowledge/chunking/ragflow_like/parsers/laws.py new file mode 100644 index 00000000..513ff7ae --- /dev/null +++ b/src/knowledge/chunking/ragflow_like/parsers/laws.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +import re +from typing import Any + +from src.knowledge.chunking.ragflow_like import nlp + + +def _unescape_delimiter(delimiter: str) -> str: + return delimiter.replace("\\n", "\n").replace("\\r", "\r").replace("\\t", "\t").replace("\\\\", "\\") + + +def _iter_lines(markdown_content: str) -> list[str]: + return [line.strip() for line in (markdown_content or "").splitlines() if line.strip()] + + +def _docx_heading_tree(markdown_content: str) -> list[str]: + lines: list[tuple[int, str]] = [] + level_set: set[int] = set() + + for raw in (markdown_content or "").splitlines(): + text = raw.strip() + if not text: + continue + + heading_match = re.match(r"^(#{1,6})\s+(.*)$", text) + if heading_match: + level = len(heading_match.group(1)) + value = heading_match.group(2).strip() + else: + level = 99 + value = text + + if not value: + continue + + lines.append((level, value)) + level_set.add(level) + + if not lines: + return [] + + sorted_levels = sorted(level_set) + h2_level = sorted_levels[1] if len(sorted_levels) > 1 else 1 + h2_level = sorted_levels[-2] if h2_level == sorted_levels[-1] and len(sorted_levels) > 2 else h2_level + + root = nlp.Node(level=0, depth=h2_level, texts=[]) + root.build_tree(lines) + return [element for element in root.get_tree() if element] + + +def chunk_markdown(filename: str, markdown_content: str, parser_config: dict[str, Any] | None = None) -> list[str]: + parser_config = parser_config or {} + + delimiter = _unescape_delimiter(str(parser_config.get("delimiter", "\n") or "\n")) + chunk_token_num = int(parser_config.get("chunk_token_num", 512) or 512) + overlapped_percent = int(parser_config.get("overlapped_percent", 0) or 0) + + if re.search(r"\.docx$", filename or "", re.IGNORECASE): + chunks = _docx_heading_tree(markdown_content) + if chunks: + return chunks + + sections = _iter_lines(markdown_content) + if not sections: + return [] + + eng = nlp.is_english(sections) + nlp.remove_contents_table(sections, eng=eng) + + typed_sections = [(s, "") for s in sections] + nlp.make_colon_as_title(typed_sections) + + bull = nlp.bullets_category([s for s, _ in typed_sections]) + merged = nlp.tree_merge(bull, typed_sections, depth=2) + + if merged: + return merged + + return nlp.naive_merge( + typed_sections, + chunk_token_num=chunk_token_num, + delimiter=delimiter, + overlapped_percent=overlapped_percent, + ) diff --git a/src/knowledge/chunking/ragflow_like/parsers/qa.py b/src/knowledge/chunking/ragflow_like/parsers/qa.py new file mode 100644 index 00000000..cdfb4272 --- /dev/null +++ b/src/knowledge/chunking/ragflow_like/parsers/qa.py @@ -0,0 +1,261 @@ +from __future__ import annotations + +import csv +import re +from io import StringIO +from typing import Any + + +def _rm_prefix(text: str) -> str: + return re.sub( + r"^(问题|答案|回答|user|assistant|Q|A|Question|Answer|问|答)[\t:： ]+", + "", + (text or "").strip(), + flags=re.IGNORECASE, + ) + + +def _to_qa_chunk(question: str, answer: str, eng: bool = False) -> str: + qprefix = "Question: " if eng else "问题：" + aprefix = "Answer: " if eng else "回答：" + return "\t".join([qprefix + _rm_prefix(question), aprefix + _rm_prefix(answer)]) + + +def _guess_delimiter(lines: list[str]) -> str: + comma = 0 + tab = 0 + for line in lines: + if len(line.split(",")) == 2: + comma += 1 + if len(line.split("\t")) == 2: + tab += 1 + return "\t" if tab >= comma else "," + + +def _extract_pairs_with_delimiter(lines: list[str], delimiter: str) -> list[tuple[str, str]]: + pairs: list[tuple[str, str]] = [] + question = "" + answer = "" + + for line in lines: + arr = line.split(delimiter) + if len(arr) != 2: + if question: + answer += "\n" + line + continue + + if question and answer: + pairs.append((question, answer)) + question, answer = arr + + if question: + pairs.append((question, answer)) + + return [(q.strip(), a.strip()) for q, a in pairs if q.strip()] + + +def _extract_pairs_from_csv(lines: list[str], delimiter: str) -> list[tuple[str, str]]: + pairs: list[tuple[str, str]] = [] + question = "" + answer = "" + + reader = csv.reader(lines, delimiter=delimiter) + for row, raw_line in zip(reader, lines, strict=False): + if len(row) != 2: + if question: + answer += "\n" + raw_line + continue + + if question and answer: + pairs.append((question, answer)) + question, answer = row + + if question: + pairs.append((question, answer)) + + return [(q.strip(), a.strip()) for q, a in pairs if q.strip()] + + +def _parse_markdown_table_row(line: str) -> list[str] | None: + if "|" not in line: + return None + + text = line.strip() + if not text: + return None + + if text.startswith("|"): + text = text[1:] + if text.endswith("|"): + text = text[:-1] + + cells = [cell.strip() for cell in text.split("|")] + if not cells: + return None + + if all(re.fullmatch(r":?-{3,}:?", c.replace(" ", "")) for c in cells if c): + return None + + return cells + + +def _extract_pairs_from_markdown_tables(markdown_content: str) -> list[tuple[str, str]]: + pairs: list[tuple[str, str]] = [] + + for line in (markdown_content or "").splitlines(): + cells = _parse_markdown_table_row(line) + if not cells or len(cells) < 2: + continue + + question = cells[0] + answer = cells[1] + if question and answer: + pairs.append((question, answer)) + + return pairs + + +def _md_question_level(line: str) -> tuple[int, str]: + match = re.match(r"^#*", line) + if not match: + return 0, line + return len(match.group(0)), line.lstrip("#").lstrip() + + +def _extract_pairs_from_markdown_headings(markdown_content: str) -> list[tuple[str, str]]: + lines = (markdown_content or "").splitlines() + if not lines: + return [] + + pairs: list[tuple[str, str]] = [] + last_answer = "" + question_stack: list[str] = [] + level_stack: list[int] = [] + code_block = False + + for line in lines: + if line.strip().startswith("```"): + code_block = not code_block + + question_level = 0 + question = "" + if not code_block: + question_level, question = _md_question_level(line) + + if not question_level or question_level > 6: + last_answer = f"{last_answer}\n{line}" + continue + + if last_answer.strip(): + sum_question = "\n".join(question_stack) + if sum_question: + pairs.append((sum_question, last_answer.strip())) + last_answer = "" + + while question_stack and question_level <= level_stack[-1]: + question_stack.pop() + level_stack.pop() + + question_stack.append(question) + level_stack.append(question_level) + + if last_answer.strip(): + sum_question = "\n".join(question_stack) + if sum_question: + pairs.append((sum_question, last_answer.strip())) + + return pairs + + +def _extract_pairs_by_prefix(lines: list[str]) -> list[tuple[str, str]]: + pairs: list[tuple[str, str]] = [] + question = "" + answer_lines: list[str] = [] + + for line in lines: + if re.match(r"^(Q|Question|问|问题)\s*[:：]", line, flags=re.IGNORECASE): + if question: + pairs.append((question, "\n".join(answer_lines))) + question = re.sub(r"^(Q|Question|问|问题)\s*[:：]", "", line, flags=re.IGNORECASE).strip() + answer_lines = [] + continue + + if re.match(r"^(A|Answer|答|回答)\s*[:：]", line, flags=re.IGNORECASE): + answer_lines.append(re.sub(r"^(A|Answer|答|回答)\s*[:：]", "", line, flags=re.IGNORECASE).strip()) + continue + + if question: + answer_lines.append(line) + + if question: + pairs.append((question, "\n".join(answer_lines))) + + return [(q.strip(), a.strip()) for q, a in pairs if q.strip() and a.strip()] + + +def _dedupe_pairs(pairs: list[tuple[str, str]]) -> list[tuple[str, str]]: + res: list[tuple[str, str]] = [] + seen: set[tuple[str, str]] = set() + + for question, answer in pairs: + q = question.strip() + a = answer.strip() + if not q or not a: + continue + key = (q, a) + if key in seen: + continue + seen.add(key) + res.append((q, a)) + + return res + + +def chunk_markdown(filename: str, markdown_content: str, parser_config: dict[str, Any] | None = None) -> list[str]: + parser_config = parser_config or {} + eng = str(parser_config.get("language", "Chinese")).lower() == "english" + + suffix = "" + if filename and "." in filename: + suffix = "." + filename.lower().split(".")[-1] + + lines = [line for line in (markdown_content or "").splitlines() if line.strip()] + pairs: list[tuple[str, str]] = [] + + if suffix in {".xlsx", ".xls"}: + pairs.extend(_extract_pairs_from_markdown_tables(markdown_content)) + if not pairs: + delimiter = _guess_delimiter(lines) + pairs.extend(_extract_pairs_with_delimiter(lines, delimiter)) + elif suffix == ".csv": + pairs.extend(_extract_pairs_from_markdown_tables(markdown_content)) + delimiter = "\t" if any("\t" in line for line in lines) else "," + pairs.extend(_extract_pairs_from_csv(lines, delimiter)) + elif suffix == ".txt": + delimiter = _guess_delimiter(lines) + pairs.extend(_extract_pairs_with_delimiter(lines, delimiter)) + elif suffix in {".md", ".markdown", ".mdx"}: + pairs.extend(_extract_pairs_from_markdown_headings(markdown_content)) + pairs.extend(_extract_pairs_from_markdown_tables(markdown_content)) + elif suffix == ".docx": + pairs.extend(_extract_pairs_from_markdown_headings(markdown_content)) + pairs.extend(_extract_pairs_from_markdown_tables(markdown_content)) + else: + pairs.extend(_extract_pairs_from_markdown_headings(markdown_content)) + pairs.extend(_extract_pairs_from_markdown_tables(markdown_content)) + pairs.extend(_extract_pairs_by_prefix(lines)) + if not pairs: + delimiter = _guess_delimiter(lines) + pairs.extend(_extract_pairs_with_delimiter(lines, delimiter)) + + pairs = _dedupe_pairs(pairs) + + if not pairs and lines: + # 最后兜底：把内容按 2 行一组构成问答 + for i in range(0, len(lines), 2): + q = lines[i] + a = lines[i + 1] if i + 1 < len(lines) else "" + if q.strip() and a.strip(): + pairs.append((q, a)) + + return [_to_qa_chunk(q, a, eng=eng) for q, a in pairs] diff --git a/src/knowledge/chunking/ragflow_like/presets.py b/src/knowledge/chunking/ragflow_like/presets.py new file mode 100644 index 00000000..53dfe917 --- /dev/null +++ b/src/knowledge/chunking/ragflow_like/presets.py @@ -0,0 +1,235 @@ +from __future__ import annotations + +from copy import deepcopy +from typing import Any + +from src.utils import logger + +CHUNK_PRESET_GENERAL = "general" +CHUNK_PRESET_QA = "qa" +CHUNK_PRESET_BOOK = "book" +CHUNK_PRESET_LAWS = "laws" + +CHUNK_PRESET_IDS = { + CHUNK_PRESET_GENERAL, + CHUNK_PRESET_QA, + CHUNK_PRESET_BOOK, + CHUNK_PRESET_LAWS, +} + +CHUNK_PRESET_DESCRIPTIONS: dict[str, str] = { + CHUNK_PRESET_GENERAL: "通用分块：按分隔符和长度切分，适合大多数普通文档。", + CHUNK_PRESET_QA: "问答分块：优先抽取问题-回答结构，适合 FAQ、题库、问答手册。", + CHUNK_PRESET_BOOK: "书籍分块：强化章节标题识别并做层级合并，适合教材、手册、长章节文档。", + CHUNK_PRESET_LAWS: "法规分块：按法条层级组织与合并，适合法律法规、制度规范类文本。", +} + +CHUNK_ENGINE_VERSION = "ragflow_like_v1" +GENERAL_INTERNAL_PARSER_ID = "naive" + +_BASE_DEFAULTS: dict[str, Any] = { + "table_context_size": 0, + "image_context_size": 0, +} + +_PRESET_DEFAULTS: dict[str, dict[str, Any] | None] = { + CHUNK_PRESET_GENERAL: { + "layout_recognize": "DeepDOC", + "chunk_token_num": 512, + "delimiter": "\n", + "auto_keywords": 0, + "auto_questions": 0, + "html4excel": False, + "topn_tags": 3, + "raptor": { + "use_raptor": True, + "prompt": "Please summarize the following paragraphs. Be careful with the numbers, do not make things up. Paragraphs as following:\n {cluster_content}\nThe above is the content you need to summarize.", + "max_token": 256, + "threshold": 0.1, + "max_cluster": 64, + "random_seed": 0, + }, + "graphrag": { + "use_graphrag": True, + "entity_types": ["organization", "person", "geo", "event", "category"], + "method": "light", + }, + }, + CHUNK_PRESET_QA: {"raptor": {"use_raptor": False}, "graphrag": {"use_graphrag": False}}, + CHUNK_PRESET_BOOK: {"raptor": {"use_raptor": False}, "graphrag": {"use_graphrag": False}}, + CHUNK_PRESET_LAWS: {"raptor": {"use_raptor": False}, "graphrag": {"use_graphrag": False}}, +} + + +def deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]: + result = deepcopy(base) + for key, value in (override or {}).items(): + if isinstance(value, dict) and isinstance(result.get(key), dict): + result[key] = deep_merge(result[key], value) + else: + result[key] = value + return result + + +def normalize_chunk_preset_id(value: str | None) -> str: + if not value: + return CHUNK_PRESET_GENERAL + + normalized = str(value).strip().lower() + if normalized == GENERAL_INTERNAL_PARSER_ID: + return CHUNK_PRESET_GENERAL + + if normalized in CHUNK_PRESET_IDS: + return normalized + + logger.warning(f"Unknown chunk preset id '{value}', fallback to general") + return CHUNK_PRESET_GENERAL + + +def map_to_internal_parser_id(preset_id: str) -> str: + normalized = normalize_chunk_preset_id(preset_id) + if normalized == CHUNK_PRESET_GENERAL: + return GENERAL_INTERNAL_PARSER_ID + return normalized + + +def get_default_chunk_parser_config(preset_id: str) -> dict[str, Any]: + normalized = normalize_chunk_preset_id(preset_id) + default_config = deepcopy(_PRESET_DEFAULTS.get(normalized) or {}) + return deep_merge(_BASE_DEFAULTS, default_config) + + +def _safe_int(value: Any) -> int | None: + if value is None: + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _legacy_params_to_parser_config(params: dict[str, Any] | None) -> dict[str, Any]: + if not isinstance(params, dict): + return {} + + parser_config: dict[str, Any] = {} + + chunk_size = _safe_int(params.get("chunk_size")) + chunk_overlap = _safe_int(params.get("chunk_overlap")) + + if chunk_size and chunk_size > 0: + parser_config["chunk_token_num"] = chunk_size + if chunk_size and chunk_size > 0 and chunk_overlap is not None: + overlap_percent = round(max(0, min(chunk_overlap, chunk_size - 1)) * 100 / chunk_size) + parser_config["overlapped_percent"] = max(0, min(overlap_percent, 99)) + + if isinstance(params.get("qa_separator"), str) and params.get("qa_separator"): + parser_config["delimiter"] = params["qa_separator"] + + if isinstance(params.get("delimiter"), str) and params.get("delimiter"): + parser_config["delimiter"] = params["delimiter"] + + if "chunk_token_num" in params: + normalized_chunk_token_num = _safe_int(params.get("chunk_token_num")) + if normalized_chunk_token_num is not None: + parser_config["chunk_token_num"] = normalized_chunk_token_num + + if "overlapped_percent" in params: + normalized_overlapped_percent = _safe_int(params.get("overlapped_percent")) + if normalized_overlapped_percent is not None: + parser_config["overlapped_percent"] = max(0, min(normalized_overlapped_percent, 99)) + + return parser_config + + +def ensure_chunk_defaults_in_additional_params(additional_params: dict[str, Any] | None) -> dict[str, Any]: + params = dict(additional_params or {}) + params["chunk_preset_id"] = normalize_chunk_preset_id(params.get("chunk_preset_id")) + + if "chunk_parser_config" in params and not isinstance(params.get("chunk_parser_config"), dict): + logger.warning("Invalid chunk_parser_config in additional_params, fallback to empty dict") + params["chunk_parser_config"] = {} + + return params + + +def resolve_chunk_processing_params( + kb_additional_params: dict[str, Any] | None, + file_processing_params: dict[str, Any] | None, + request_params: dict[str, Any] | None = None, +) -> dict[str, Any]: + kb_additional = ensure_chunk_defaults_in_additional_params(kb_additional_params) + file_params = dict(file_processing_params or {}) + request = dict(request_params or {}) + + preset_id = normalize_chunk_preset_id( + request.get("chunk_preset_id") + or file_params.get("chunk_preset_id") + or kb_additional.get("chunk_preset_id") + ) + + parser_config = get_default_chunk_parser_config(preset_id) + + kb_parser_config = kb_additional.get("chunk_parser_config") + if isinstance(kb_parser_config, dict): + parser_config = deep_merge(parser_config, kb_parser_config) + + file_parser_config = file_params.get("chunk_parser_config") + if isinstance(file_parser_config, dict): + parser_config = deep_merge(parser_config, file_parser_config) + + req_parser_config = request.get("chunk_parser_config") + if isinstance(req_parser_config, dict): + parser_config = deep_merge(parser_config, req_parser_config) + + merged_legacy = {} + merged_legacy.update(file_params) + merged_legacy.update(request) + parser_config = deep_merge(parser_config, _legacy_params_to_parser_config(merged_legacy)) + + # Build processing params snapshot (keep existing + request overrides for non-chunk fields) + snapshot: dict[str, Any] = {} + snapshot.update(file_params) + snapshot.update(request) + snapshot["chunk_preset_id"] = preset_id + snapshot["chunk_parser_config"] = parser_config + snapshot["chunk_engine_version"] = CHUNK_ENGINE_VERSION + + # Keep backward-compatible fields for current UI + if "chunk_size" not in snapshot and isinstance(parser_config.get("chunk_token_num"), int): + snapshot["chunk_size"] = parser_config["chunk_token_num"] + + if "chunk_overlap" not in snapshot and isinstance(parser_config.get("overlapped_percent"), int): + token_num = parser_config.get("chunk_token_num") + if isinstance(token_num, int) and token_num > 0: + snapshot["chunk_overlap"] = int(token_num * parser_config["overlapped_percent"] / 100) + + if "qa_separator" not in snapshot and isinstance(parser_config.get("delimiter"), str): + snapshot["qa_separator"] = parser_config["delimiter"] + + return snapshot + + +def get_chunk_preset_options() -> list[dict[str, str]]: + return [ + { + "value": CHUNK_PRESET_GENERAL, + "label": "General", + "description": CHUNK_PRESET_DESCRIPTIONS[CHUNK_PRESET_GENERAL], + }, + { + "value": CHUNK_PRESET_QA, + "label": "QA", + "description": CHUNK_PRESET_DESCRIPTIONS[CHUNK_PRESET_QA], + }, + { + "value": CHUNK_PRESET_BOOK, + "label": "Book", + "description": CHUNK_PRESET_DESCRIPTIONS[CHUNK_PRESET_BOOK], + }, + { + "value": CHUNK_PRESET_LAWS, + "label": "Laws", + "description": CHUNK_PRESET_DESCRIPTIONS[CHUNK_PRESET_LAWS], + }, + ] diff --git a/src/knowledge/implementations/lightrag.py b/src/knowledge/implementations/lightrag.py index 940ebec4..2744bcf2 100644 --- a/src/knowledge/implementations/lightrag.py +++ b/src/knowledge/implementations/lightrag.py @@ -11,6 +11,8 @@ from pymilvus import connections, utility from src import config from src.knowledge.base import FileStatus, KnowledgeBase +from src.knowledge.chunking.ragflow_like.dispatcher import chunk_markdown +from src.knowledge.chunking.ragflow_like.presets import resolve_chunk_processing_params from src.knowledge.indexing import process_file_to_markdown from src.knowledge.utils.kb_utils import get_embedding_config from src.utils import hashstr, logger @@ -40,6 +42,18 @@ class LightRagKB(KnowledgeBase): """知识库类型标识""" return "lightrag" + @staticmethod + def _prepare_lightrag_insert_payload(chunks: list[dict]) -> tuple[str, str | None, bool]: + if not chunks: + return "", None, False + + if len(chunks) == 1: + return chunks[0]["content"], None, False + + delimiter = "\n<|YUXI_CHUNK_DELIM|>\n" + payload = delimiter.join(chunk["content"] for chunk in chunks if chunk.get("content")) + return payload, delimiter, True + def delete_database(self, db_id: str) -> dict: """删除数据库，同时清除Milvus和Neo4j中的数据""" # Drop Milvus collection @@ -137,6 +151,21 @@ class LightRagKB(KnowledgeBase): await instance.initialize_storages() await initialize_pipeline_status() + @staticmethod + async def _ensure_doc_processed(rag: LightRAG, file_id: str) -> None: + """确保 LightRAG 文档处理成功，否则抛出异常。""" + status_doc = await rag.doc_status.get_by_id(file_id) + if not status_doc: + raise ValueError(f"LightRAG 文档状态缺失: {file_id}") + + status = status_doc.get("status") + status_value = status.value if hasattr(status, "value") else status + if status_value not in {"processed", "preprocessed"}: + error_msg = status_doc.get("error_msg") or "unknown error" + raise ValueError( + f"LightRAG 实体关系抽取失败: file_id={file_id}, status={status_value}, error={error_msg}" + ) + async def _get_lightrag_instance(self, db_id: str) -> LightRAG | None: """获取或创建 LightRAG 实例""" if db_id in self.instances: @@ -293,14 +322,36 @@ class LightRagKB(KnowledgeBase): # Read markdown markdown_content = await self._read_markdown_from_minio(file_meta["markdown_file"]) file_path = file_meta.get("path") + filename = file_meta.get("filename") or file_id + processing_params = resolve_chunk_processing_params( + kb_additional_params=self.databases_meta.get(db_id, {}).get("metadata"), + file_processing_params=file_meta.get("processing_params"), + ) + self.files_meta[file_id]["processing_params"] = processing_params + await self._save_metadata() + + chunks = chunk_markdown(markdown_content, file_id, filename, processing_params) + chunk_input, split_by_character, split_by_character_only = self._prepare_lightrag_insert_payload(chunks) + if not chunk_input: + chunk_input = markdown_content # Clean up existing chunks if any (for re-indexing) await self.delete_file_chunks_only(db_id, file_id) # Insert - await rag.ainsert(input=markdown_content, ids=file_id, file_paths=file_path) + await rag.ainsert( + input=chunk_input, + ids=file_id, + file_paths=file_path, + split_by_character=split_by_character, + split_by_character_only=split_by_character_only, + ) + await self._ensure_doc_processed(rag, file_id) - logger.info(f"Indexed file {file_id} into LightRAG") + logger.info( + f"Indexed file {file_id} into LightRAG with {len(chunks)} chunks, " + f"chunk_preset_id={processing_params.get('chunk_preset_id')}" + ) # Update status self.files_meta[file_id]["status"] = FileStatus.INDEXED @@ -358,7 +409,12 @@ class LightRagKB(KnowledgeBase): try: # 更新状态为处理中 - self.files_meta[file_id]["processing_params"] = params.copy() + resolved_params = resolve_chunk_processing_params( + kb_additional_params=self.databases_meta.get(db_id, {}).get("metadata"), + file_processing_params=self.files_meta[file_id].get("processing_params"), + request_params=params, + ) + self.files_meta[file_id]["processing_params"] = resolved_params self.files_meta[file_id]["status"] = "processing" await self._persist_file(file_id) @@ -368,12 +424,24 @@ class LightRagKB(KnowledgeBase): markdown_content = await process_file_to_markdown(file_path, params=params) markdown_content_lines = markdown_content[:100].replace("\n", " ") logger.info(f"Markdown content: {markdown_content_lines}...") + filename = file_meta.get("filename") or file_id + chunks = chunk_markdown(markdown_content, file_id, filename, resolved_params) + chunk_input, split_by_character, split_by_character_only = self._prepare_lightrag_insert_payload(chunks) + if not chunk_input: + chunk_input = markdown_content # 先删除现有的 LightRAG 数据（仅删除chunks，保留元数据） await self.delete_file_chunks_only(db_id, file_id) # 使用 LightRAG 重新插入内容 - await rag.ainsert(input=markdown_content, ids=file_id, file_paths=file_path) + await rag.ainsert( + input=chunk_input, + ids=file_id, + file_paths=file_path, + split_by_character=split_by_character, + split_by_character_only=split_by_character_only, + ) + await self._ensure_doc_processed(rag, file_id) logger.info(f"Updated {content_type} {file_path} in LightRAG. Done.") diff --git a/src/knowledge/implementations/milvus.py b/src/knowledge/implementations/milvus.py index 9896e4c2..8077cb4a 100644 --- a/src/knowledge/implementations/milvus.py +++ b/src/knowledge/implementations/milvus.py @@ -10,11 +10,10 @@ from pymilvus import Collection, CollectionSchema, DataType, FieldSchema, connec from src import config from src.knowledge.base import FileStatus, KnowledgeBase +from src.knowledge.chunking.ragflow_like.dispatcher import chunk_markdown +from src.knowledge.chunking.ragflow_like.presets import resolve_chunk_processing_params from src.knowledge.indexing import process_file_to_markdown -from src.knowledge.utils.kb_utils import ( - get_embedding_config, - split_text_into_chunks, -) +from src.knowledge.utils.kb_utils import get_embedding_config from src.models.embed import OtherEmbedding from src.utils import hashstr, logger from src.utils.datetime_utils import utc_isoformat @@ -222,7 +221,7 @@ class MilvusKB(KnowledgeBase): def _split_text_into_chunks(self, text: str, file_id: str, filename: str, params: dict) -> list[dict]: """将文本分割成块""" - return split_text_into_chunks(text, file_id, filename, params) + return chunk_markdown(text, file_id, filename, params) async def index_file(self, db_id: str, file_id: str, operator_id: str | None = None) -> dict: """ @@ -284,7 +283,12 @@ class MilvusKB(KnowledgeBase): await self._persist_file(file_id) # Read processing params inside lock to ensure we get the latest values - params = file_meta.get("processing_params", {}) or {} + params = resolve_chunk_processing_params( + kb_additional_params=self.databases_meta.get(db_id, {}).get("metadata"), + file_processing_params=file_meta.get("processing_params"), + ) + self.files_meta[file_id]["processing_params"] = params + await self._save_metadata() logger.debug(f"[index_file] file_id={file_id}, processing_params={params}") # Add to processing queue @@ -299,6 +303,7 @@ class MilvusKB(KnowledgeBase): chunks = self._split_text_into_chunks(markdown_content, file_id, filename, params) logger.info( f"Split {filename} into {len(chunks)} chunks with params: " + f"chunk_preset_id={params.get('chunk_preset_id')}, " f"chunk_size={params.get('chunk_size')}, " f"chunk_overlap={params.get('chunk_overlap')}, " f"qa_separator={params.get('qa_separator')}" @@ -391,7 +396,12 @@ class MilvusKB(KnowledgeBase): try: # 更新状态为处理中 async with self._metadata_lock: - self.files_meta[file_id]["processing_params"] = params.copy() + resolved_params = resolve_chunk_processing_params( + kb_additional_params=self.databases_meta.get(db_id, {}).get("metadata"), + file_processing_params=self.files_meta[file_id].get("processing_params"), + request_params=params, + ) + self.files_meta[file_id]["processing_params"] = resolved_params self.files_meta[file_id]["status"] = "processing" await self._persist_file(file_id) @@ -404,7 +414,7 @@ class MilvusKB(KnowledgeBase): await self.delete_file_chunks_only(db_id, file_id) # 重新生成 chunks - chunks = self._split_text_into_chunks(markdown_content, file_id, filename, params) + chunks = self._split_text_into_chunks(markdown_content, file_id, filename, resolved_params) logger.info(f"Split {filename} into {len(chunks)} chunks") if chunks: diff --git a/src/knowledge/manager.py b/src/knowledge/manager.py index 1f262bc9..a74192c4 100644 --- a/src/knowledge/manager.py +++ b/src/knowledge/manager.py @@ -2,6 +2,10 @@ import asyncio import os from src.knowledge.base import KBNotFoundError, KnowledgeBase +from src.knowledge.chunking.ragflow_like.presets import ( + deep_merge, + ensure_chunk_defaults_in_additional_params, +) from src.knowledge.factory import KnowledgeBaseFactory from src.utils import logger from src.utils.datetime_utils import utc_isoformat @@ -179,7 +183,7 @@ class KnowledgeBaseManager: if db_info: # 补充 share_config 和 additional_params db_info["share_config"] = row.share_config or {"is_shared": True, "accessible_departments": []} - db_info["additional_params"] = row.additional_params or {} + db_info["additional_params"] = ensure_chunk_defaults_in_additional_params(row.additional_params) all_databases.append(db_info) return {"databases": all_databases} @@ -310,6 +314,8 @@ class KnowledgeBaseManager: if share_config is None: share_config = {"is_shared": True, "accessible_departments": []} + kwargs = ensure_chunk_defaults_in_additional_params(kwargs) + kb_instance = self._get_or_create_kb_instance(kb_type) db_info = await kb_instance.create_database(database_name, description, embed_info, **kwargs) db_id = db_info["db_id"] @@ -414,7 +420,7 @@ class KnowledgeBaseManager: } # 添加数据库中的附加字段 - db_info["additional_params"] = kb.additional_params or {} + db_info["additional_params"] = ensure_chunk_defaults_in_additional_params(kb.additional_params) db_info["share_config"] = kb.share_config or {"is_shared": True, "accessible_departments": []} db_info["mindmap"] = kb.mindmap db_info["sample_questions"] = kb.sample_questions or [] @@ -566,6 +572,11 @@ class KnowledgeBaseManager: """更新数据库""" from src.repositories.knowledge_base_repository import KnowledgeBaseRepository + kb_repo = KnowledgeBaseRepository() + kb = await kb_repo.get_by_id(db_id) + if kb is None: + raise ValueError(f"数据库 {db_id} 不存在") + kb_instance = await self._get_kb_for_database(db_id) kb_instance.update_database(db_id, name, description, llm_info) @@ -576,13 +587,19 @@ class KnowledgeBaseManager: } if llm_info is not None: update_data["llm_info"] = llm_info + if additional_params is not None: - update_data["additional_params"] = additional_params + merged_additional_params = ensure_chunk_defaults_in_additional_params( + deep_merge(kb.additional_params or {}, additional_params) + ) + update_data["additional_params"] = merged_additional_params + if db_id in kb_instance.databases_meta: + kb_instance.databases_meta[db_id]["metadata"] = merged_additional_params + if share_config is not None: update_data["share_config"] = share_config # 保存到数据库 - kb_repo = KnowledgeBaseRepository() await kb_repo.update(db_id, update_data) return await self.get_database_info(db_id) diff --git a/src/repositories/skill_repository.py b/src/repositories/skill_repository.py new file mode 100644 index 00000000..745e5c46 --- /dev/null +++ b/src/repositories/skill_repository.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from src.storage.postgres.models_business import Skill +from src.utils.datetime_utils import utc_now_naive + + +class SkillRepository: + def __init__(self, db_session: AsyncSession): + self.db = db_session + + async def list_all(self) -> list[Skill]: + result = await self.db.execute(select(Skill).order_by(Skill.updated_at.desc(), Skill.id.desc())) + return list(result.scalars().all()) + + async def get_by_slug(self, slug: str) -> Skill | None: + result = await self.db.execute(select(Skill).where(Skill.slug == slug)) + return result.scalar_one_or_none() + + async def exists_slug(self, slug: str) -> bool: + return (await self.get_by_slug(slug)) is not None + + async def create( + self, + *, + slug: str, + name: str, + description: str, + tool_dependencies: list[str] | None, + mcp_dependencies: list[str] | None, + skill_dependencies: list[str] | None, + dir_path: str, + created_by: str | None, + ) -> Skill: + now = utc_now_naive() + item = Skill( + slug=slug, + name=name, + description=description, + tool_dependencies=tool_dependencies or [], + mcp_dependencies=mcp_dependencies or [], + skill_dependencies=skill_dependencies or [], + dir_path=dir_path, + created_by=created_by, + updated_by=created_by, + created_at=now, + updated_at=now, + ) + self.db.add(item) + await self.db.commit() + await self.db.refresh(item) + return item + + async def update_dependencies( + self, + item: Skill, + *, + tool_dependencies: list[str], + mcp_dependencies: list[str], + skill_dependencies: list[str], + updated_by: str | None, + ) -> Skill: + item.tool_dependencies = tool_dependencies + item.mcp_dependencies = mcp_dependencies + item.skill_dependencies = skill_dependencies + item.updated_by = updated_by + item.updated_at = utc_now_naive() + await self.db.commit() + await self.db.refresh(item) + return item + + async def update_metadata( + self, + item: Skill, + *, + name: str, + description: str, + updated_by: str | None, + ) -> Skill: + item.name = name + item.description = description + item.updated_by = updated_by + item.updated_at = utc_now_naive() + await self.db.commit() + await self.db.refresh(item) + return item + + async def delete(self, item: Skill) -> None: + await self.db.delete(item) + await self.db.commit() diff --git a/src/services/evaluation_service.py b/src/services/evaluation_service.py index e1b6968b..1c4b42b9 100644 --- a/src/services/evaluation_service.py +++ b/src/services/evaluation_service.py @@ -231,6 +231,29 @@ class EvaluationService: logger.error(f"获取评估基准详情失败: {e}") raise + async def get_benchmark_download_info(self, benchmark_id: str) -> dict[str, str]: + """获取评估基准下载信息""" + row = await self.eval_repo.get_benchmark(benchmark_id) + if row is None: + raise ValueError("Benchmark not found") + + data_file_path = row.data_file_path or "" + if not data_file_path or not os.path.exists(data_file_path): + raise ValueError("Benchmark file not found") + + filename_base = (row.name or "").strip() + if not filename_base: + filename_base = row.benchmark_id + + filename_base = re.sub(r"[\\/:*?\"<>|]+", "_", filename_base).strip() + if not filename_base or filename_base in {".", ".."}: + filename_base = row.benchmark_id + + if not filename_base.endswith(".jsonl"): + filename_base = f"{filename_base}.jsonl" + + return {"file_path": data_file_path, "filename": filename_base} + async def delete_benchmark(self, benchmark_id: str) -> None: """删除评估基准""" try: diff --git a/src/services/mcp_service.py b/src/services/mcp_service.py index d3d9da0e..4385820b 100644 --- a/src/services/mcp_service.py +++ b/src/services/mcp_service.py @@ -35,6 +35,7 @@ _mcp_tools_stats: dict[str, dict[str, int]] = {} # MCP Server configurations (Runtime cache, loaded from DB) MCP_SERVERS: dict[str, dict[str, Any]] = {} +_UNSET = object() # Default MCP Server configurations (Imported to DB on first run) _DEFAULT_MCP_SERVERS = { @@ -130,6 +131,7 @@ async def init_mcp_servers() -> None: url=config.get("url"), command=config.get("command"), args=config.get("args"), + env=config.get("env"), headers=config.get("headers"), timeout=config.get("timeout"), sse_read_timeout=config.get("sse_read_timeout"), @@ -155,6 +157,7 @@ async def init_mcp_servers() -> None: url=config.get("url"), command=config.get("command"), args=config.get("args"), + env=config.get("env"), headers=config.get("headers"), timeout=config.get("timeout"), sse_read_timeout=config.get("sse_read_timeout"), @@ -368,6 +371,7 @@ async def create_mcp_server( url: str = None, command: str = None, args: list = None, + env: dict = None, description: str = None, headers: dict = None, timeout: int = None, @@ -389,6 +393,7 @@ async def create_mcp_server( url=url, command=command, args=args, + env=env, headers=headers, timeout=timeout, sse_read_timeout=sse_read_timeout, @@ -417,6 +422,7 @@ async def update_mcp_server( url: str = None, command: str = None, args: list = None, + env: Any = _UNSET, headers: dict = None, timeout: int = None, sse_read_timeout: int = None, @@ -439,6 +445,8 @@ async def update_mcp_server( server.command = command if args is not None: server.args = args + if env is not _UNSET: + server.env = env if headers is not None: server.headers = headers if timeout is not None: diff --git a/src/services/skill_resolver.py b/src/services/skill_resolver.py new file mode 100644 index 00000000..45041c4b --- /dev/null +++ b/src/services/skill_resolver.py @@ -0,0 +1,226 @@ +from __future__ import annotations + +from typing import TypedDict + +from sqlalchemy.ext.asyncio import AsyncSession + +from src.repositories.skill_repository import SkillRepository +from src.storage.postgres.manager import pg_manager +from src.storage.postgres.models_business import Skill +from src.utils.logging_config import logger + + +class SkillPromptMetadata(TypedDict): + name: str + description: str + path: str + + +class SkillDependencyNode(TypedDict): + tools: list[str] + mcps: list[str] + skills: list[str] + + +class SkillSessionSnapshot(TypedDict): + selected_skills: list[str] + visible_skills: list[str] + prompt_metadata: dict[str, SkillPromptMetadata] + dependency_map: dict[str, SkillDependencyNode] + + +def normalize_selected_skills(selected_skills: list[str] | None) -> list[str]: + return _normalize_string_list(selected_skills) + + +def is_snapshot_match_selected_skills( + snapshot: SkillSessionSnapshot | None, + selected_skills: list[str] | None, +) -> bool: + if not snapshot: + return False + current = snapshot.get("selected_skills") + if not isinstance(current, list): + return False + return current == normalize_selected_skills(selected_skills) + + +async def resolve_session_snapshot( + selected_skills: list[str] | None, + *, + db: AsyncSession | None = None, +) -> SkillSessionSnapshot: + normalized_selected = normalize_selected_skills(selected_skills) + skills = await _list_skills_from_db(db) + prompt_metadata, dependency_map = _build_maps(skills) + visible_skills = expand_skill_closure(normalized_selected, dependency_map) + return { + "selected_skills": normalized_selected, + "visible_skills": visible_skills, + "prompt_metadata": prompt_metadata, + "dependency_map": dependency_map, + } + + +def collect_prompt_metadata( + snapshot: SkillSessionSnapshot | None, + slugs: list[str] | None, +) -> list[SkillPromptMetadata]: + if not snapshot or not slugs: + return [] + + prompt_metadata = snapshot.get("prompt_metadata") or {} + result: list[SkillPromptMetadata] = [] + seen: set[str] = set() + for slug in slugs: + if not isinstance(slug, str): + continue + normalized = slug.strip() + if not normalized or normalized in seen: + continue + seen.add(normalized) + + item = prompt_metadata.get(normalized) + if not item: + logger.debug(f"Skill slug not found in session snapshot, skip prompt metadata: {normalized}") + continue + result.append(dict(item)) + return result + + +def build_dependency_bundle( + snapshot: SkillSessionSnapshot | None, + activated_slugs: list[str] | None, +) -> dict[str, list[str]]: + if not snapshot: + return {"tools": [], "mcps": [], "skills": []} + + dependency_map = snapshot.get("dependency_map") or {} + closure = expand_skill_closure(activated_slugs or [], dependency_map) + tools: list[str] = [] + mcps: list[str] = [] + seen_tools: set[str] = set() + seen_mcps: set[str] = set() + + for slug in closure: + dep = dependency_map.get(slug, {}) + for tool_name in dep.get("tools", []): + if tool_name in seen_tools: + continue + seen_tools.add(tool_name) + tools.append(tool_name) + for mcp_name in dep.get("mcps", []): + if mcp_name in seen_mcps: + continue + seen_mcps.add(mcp_name) + mcps.append(mcp_name) + + return {"tools": tools, "mcps": mcps, "skills": closure} + + +def expand_skill_closure( + slugs: list[str] | None, + dependency_map: dict[str, SkillDependencyNode], +) -> list[str]: + ordered_roots = _normalize_string_list(slugs) + if not ordered_roots: + return [] + + result: list[str] = [] + seen: set[str] = set() + + def dfs(slug: str, stack: set[str]) -> None: + if slug in stack: + logger.warning(f"Cycle detected in skill dependencies, skip: {' -> '.join([*stack, slug])}") + return + if slug in seen: + return + + node = dependency_map.get(slug) + if not node: + logger.warning(f"Skill dependency target not found in DB snapshot, skip: {slug}") + return + + seen.add(slug) + result.append(slug) + next_stack = set(stack) + next_stack.add(slug) + for dep in node.get("skills", []): + dfs(dep, next_stack) + + for root in ordered_roots: + dfs(root, set()) + return result + + +async def get_skill_options_from_db( + *, + db: AsyncSession | None = None, +) -> list[dict[str, str]]: + items = await _list_skills_from_db(db) + return [ + { + "id": item.slug, + "name": item.name, + "description": item.description, + } + for item in items + ] + + +async def get_skill_slug_set_from_db( + *, + db: AsyncSession | None = None, +) -> set[str]: + items = await _list_skills_from_db(db) + return {item.slug for item in items} + + +def _normalize_string_list(values: list[str] | None) -> list[str]: + if not values: + return [] + normalized: list[str] = [] + seen: set[str] = set() + for value in values: + if not isinstance(value, str): + continue + item = value.strip() + if not item or item in seen: + continue + seen.add(item) + normalized.append(item) + return normalized + + +def _build_maps(skills: list[Skill]) -> tuple[dict[str, SkillPromptMetadata], dict[str, SkillDependencyNode]]: + prompt_metadata: dict[str, SkillPromptMetadata] = {} + dependency_map: dict[str, SkillDependencyNode] = {} + for item in skills: + prompt_metadata[item.slug] = { + "name": item.name, + "description": item.description, + "path": f"/skills/{item.slug}/SKILL.md", + } + dependency_map[item.slug] = { + "tools": _normalize_string_list(item.tool_dependencies or []), + "mcps": _normalize_string_list(item.mcp_dependencies or []), + "skills": _normalize_string_list(item.skill_dependencies or []), + } + return prompt_metadata, dependency_map + + +async def _list_skills_from_db(db: AsyncSession | None) -> list[Skill]: + if db is not None: + repo = SkillRepository(db) + return await repo.list_all() + + try: + async with pg_manager.get_async_session_context() as session: + repo = SkillRepository(session) + return await repo.list_all() + except RuntimeError: + # 在非 FastAPI 生命周期场景（如 worker/脚本）按需初始化 + pg_manager.initialize() + async with pg_manager.get_async_session_context() as session: + repo = SkillRepository(session) + return await repo.list_all() diff --git a/src/services/skill_service.py b/src/services/skill_service.py new file mode 100644 index 00000000..1f5744a4 --- /dev/null +++ b/src/services/skill_service.py @@ -0,0 +1,527 @@ +from __future__ import annotations + +import re +import shutil +import tempfile +import uuid +import zipfile +from pathlib import Path, PurePosixPath +from typing import Any + +import yaml +from sqlalchemy.ext.asyncio import AsyncSession + +from src import config as sys_config +from src.repositories.skill_repository import SkillRepository +from src.services.mcp_service import get_mcp_server_names +from src.storage.postgres.models_business import Skill + +SKILL_SLUG_PATTERN = re.compile(r"^[a-z0-9]+(-[a-z0-9]+)*$") +SKILL_NAME_PATTERN = SKILL_SLUG_PATTERN +FRONTMATTER_PATTERN = re.compile(r"^---\s*\n(.*?)\n---\s*\n", re.DOTALL) + +TEXT_FILE_EXTENSIONS = { + ".md", + ".txt", + ".py", + ".js", + ".ts", + ".json", + ".yaml", + ".yml", + ".toml", + ".ini", + ".cfg", + ".conf", + ".xml", + ".html", + ".css", + ".sql", + ".sh", + ".bat", + ".ps1", + ".env", + ".csv", + ".tsv", + ".rst", + ".ipynb", + ".vue", + ".jsx", + ".tsx", +} + +def _normalize_string_list(values: list[str] | None) -> list[str]: + if not values: + return [] + normalized: list[str] = [] + seen: set[str] = set() + for value in values: + if not isinstance(value, str): + continue + item = value.strip() + if not item or item in seen: + continue + seen.add(item) + normalized.append(item) + return normalized + + +def is_valid_skill_slug(slug: str) -> bool: + if not isinstance(slug, str): + return False + return bool(SKILL_SLUG_PATTERN.match(slug.strip())) + + +def validate_skill_slug(slug: str) -> str: + normalized = slug.strip() if isinstance(slug, str) else "" + if not is_valid_skill_slug(normalized): + raise ValueError("无效 skill slug") + return normalized + + +def _get_buildin_tool_names() -> list[str]: + from src.agents.common.tools import get_buildin_tools + + return [tool.name for tool in get_buildin_tools()] + + +def get_skills_root_dir() -> Path: + root = Path(sys_config.save_dir) / "skills" + root.mkdir(parents=True, exist_ok=True) + return root + + +async def get_skill_dependency_options(db: AsyncSession) -> dict[str, list[str]]: + repo = SkillRepository(db) + items = await repo.list_all() + return { + "tools": _get_buildin_tool_names(), + "mcps": get_mcp_server_names(), + "skills": [item.slug for item in items], + } + + +async def list_skills(db: AsyncSession) -> list[Skill]: + repo = SkillRepository(db) + return await repo.list_all() + + +def _validate_dependencies( + *, + slug: str, + tool_dependencies: list[str], + mcp_dependencies: list[str], + skill_dependencies: list[str], + available_skill_slugs: set[str], +) -> tuple[list[str], list[str], list[str]]: + tools = _normalize_string_list(tool_dependencies) + mcps = _normalize_string_list(mcp_dependencies) + skills = _normalize_string_list(skill_dependencies) + + available_tools = set(_get_buildin_tool_names()) + invalid_tools = [name for name in tools if name not in available_tools] + if invalid_tools: + raise ValueError(f"存在无效工具依赖: {', '.join(invalid_tools)}") + + available_mcps = set(get_mcp_server_names()) + invalid_mcps = [name for name in mcps if name not in available_mcps] + if invalid_mcps: + raise ValueError(f"存在无效 MCP 依赖: {', '.join(invalid_mcps)}") + + invalid_skills = [name for name in skills if name not in available_skill_slugs] + if invalid_skills: + raise ValueError(f"存在无效 skill 依赖: {', '.join(invalid_skills)}") + + if slug in skills: + raise ValueError("skill_dependencies 不允许包含自身") + + return tools, mcps, skills + + +async def update_skill_dependencies( + db: AsyncSession, + *, + slug: str, + tool_dependencies: list[str], + mcp_dependencies: list[str], + skill_dependencies: list[str], + updated_by: str | None, +) -> Skill: + item = await get_skill_or_raise(db, slug) + repo = SkillRepository(db) + skill_items = await repo.list_all() + available_skill_slugs = {skill.slug for skill in skill_items} + tools, mcps, skills = _validate_dependencies( + slug=slug, + tool_dependencies=tool_dependencies, + mcp_dependencies=mcp_dependencies, + skill_dependencies=skill_dependencies, + available_skill_slugs=available_skill_slugs, + ) + + return await repo.update_dependencies( + item, + tool_dependencies=tools, + mcp_dependencies=mcps, + skill_dependencies=skills, + updated_by=updated_by, + ) + + +def _validate_skill_name(name: str) -> str: + name = name.strip() + if not name: + raise ValueError("SKILL.md frontmatter 缺少 name") + if len(name) > 128: + raise ValueError("skill name 长度不能超过 128") + if not SKILL_NAME_PATTERN.match(name): + raise ValueError("skill name 必须是小写字母/数字/短横线，且不能连续短横线") + return name + + +def _parse_skill_markdown(content: str) -> tuple[str, str, dict[str, Any]]: + match = FRONTMATTER_PATTERN.match(content) + if not match: + raise ValueError("SKILL.md 缺少有效 frontmatter（--- ... ---）") + + frontmatter_raw = match.group(1) + try: + data = yaml.safe_load(frontmatter_raw) + except yaml.YAMLError as e: + raise ValueError(f"SKILL.md frontmatter YAML 解析失败: {e}") from e + + if not isinstance(data, dict): + raise ValueError("SKILL.md frontmatter 必须是对象") + + name = _validate_skill_name(str(data.get("name", ""))) + description = str(data.get("description", "")).strip() + if not description: + raise ValueError("SKILL.md frontmatter 缺少 description") + + return name, description, data + + +def _rewrite_frontmatter_name(content: str, new_name: str) -> str: + match = FRONTMATTER_PATTERN.match(content) + if not match: + raise ValueError("SKILL.md 缺少有效 frontmatter（--- ... ---）") + + frontmatter_raw = match.group(1) + body = content[match.end() :] + data = yaml.safe_load(frontmatter_raw) + if not isinstance(data, dict): + raise ValueError("SKILL.md frontmatter 必须是对象") + data["name"] = new_name + dumped = yaml.safe_dump(data, sort_keys=False, allow_unicode=True).strip() + return f"---\n{dumped}\n---\n{body}" + + +def _validate_zip_paths(zip_file: zipfile.ZipFile) -> None: + for name in zip_file.namelist(): + pure = PurePosixPath(name) + if pure.is_absolute(): + raise ValueError(f"ZIP 包含不安全绝对路径: {name}") + if ".." in pure.parts: + raise ValueError(f"ZIP 包含路径穿越片段: {name}") + + +async def _generate_available_slug(repo: SkillRepository, base_slug: str) -> str: + root = get_skills_root_dir() + if not await repo.exists_slug(base_slug) and not (root / base_slug).exists(): + return base_slug + + idx = 2 + while True: + candidate = f"{base_slug}-v{idx}" + if not await repo.exists_slug(candidate) and not (root / candidate).exists(): + return candidate + idx += 1 + + +def _resolve_skill_dir(item: Skill) -> Path: + dir_path = Path(item.dir_path) + if dir_path.is_absolute(): + return dir_path + return (Path(sys_config.save_dir) / dir_path).resolve() + + +def _resolve_relative_path(skill_dir: Path, relative_path: str, *, allow_root: bool = False) -> tuple[Path, str]: + rel = (relative_path or "").strip().replace("\\", "/") + rel = rel.lstrip("/") + if not rel and not allow_root: + raise ValueError("path 不能为空") + pure = PurePosixPath(rel) if rel else PurePosixPath(".") + if ".." in pure.parts: + raise ValueError("非法路径：不允许上级路径引用") + + target = (skill_dir / pure).resolve() + try: + target.relative_to(skill_dir) + except ValueError: + raise ValueError("非法路径：越界访问被拒绝") from None + + return target, rel + + +def _is_text_path(path: Path) -> bool: + if path.name == "SKILL.md": + return True + suffix = path.suffix.lower() + return suffix in TEXT_FILE_EXTENSIONS + + +def _build_tree(path: Path, base_dir: Path) -> list[dict[str, Any]]: + children: list[dict[str, Any]] = [] + for child in sorted(path.iterdir(), key=lambda p: (not p.is_dir(), p.name.lower())): + rel = child.relative_to(base_dir).as_posix() + if child.is_dir(): + children.append( + { + "name": child.name, + "path": rel, + "is_dir": True, + "children": _build_tree(child, base_dir), + } + ) + else: + children.append( + { + "name": child.name, + "path": rel, + "is_dir": False, + } + ) + return children + + +async def import_skill_zip( + db: AsyncSession, + *, + filename: str, + file_bytes: bytes, + created_by: str | None, +) -> Skill: + if not filename.lower().endswith(".zip"): + raise ValueError("仅支持上传 .zip 文件") + + repo = SkillRepository(db) + skills_root = get_skills_root_dir() + + with tempfile.TemporaryDirectory(prefix=".skill-import-", dir=str(skills_root.parent)) as temp_root: + temp_root_path = Path(temp_root) + zip_path = temp_root_path / "upload.zip" + extract_dir = temp_root_path / "extract" + stage_dir = temp_root_path / "stage" + extract_dir.mkdir(parents=True, exist_ok=True) + + zip_path.write_bytes(file_bytes) + + with zipfile.ZipFile(zip_path, "r") as zf: + _validate_zip_paths(zf) + zf.extractall(extract_dir) + + skill_md_files = list(extract_dir.rglob("SKILL.md")) + if len(skill_md_files) != 1: + raise ValueError("ZIP 必须且只能包含一个技能（检测到一个 SKILL.md）") + + skill_md_path = skill_md_files[0] + source_skill_dir = skill_md_path.parent + content = skill_md_path.read_text(encoding="utf-8") + parsed_name, parsed_desc, _ = _parse_skill_markdown(content) + + final_slug = await _generate_available_slug(repo, parsed_name) + final_name = parsed_name + if final_slug != parsed_name: + final_name = final_slug + content = _rewrite_frontmatter_name(content, final_name) + skill_md_path.write_text(content, encoding="utf-8") + + shutil.copytree(source_skill_dir, stage_dir) + + temp_target = skills_root / f".{final_slug}.tmp-{uuid.uuid4().hex[:8]}" + if temp_target.exists(): + shutil.rmtree(temp_target) + shutil.move(str(stage_dir), str(temp_target)) + + final_dir = skills_root / final_slug + if final_dir.exists(): + shutil.rmtree(temp_target, ignore_errors=True) + raise ValueError(f"技能目录冲突，请重试: {final_slug}") + temp_target.rename(final_dir) + + try: + item = await repo.create( + slug=final_slug, + name=final_name, + description=parsed_desc, + tool_dependencies=[], + mcp_dependencies=[], + skill_dependencies=[], + dir_path=(Path("skills") / final_slug).as_posix(), + created_by=created_by, + ) + except Exception: + shutil.rmtree(final_dir, ignore_errors=True) + raise + + return item + + +async def get_skill_or_raise(db: AsyncSession, slug: str) -> Skill: + slug = validate_skill_slug(slug) + repo = SkillRepository(db) + item = await repo.get_by_slug(slug) + if not item: + raise ValueError(f"技能 '{slug}' 不存在") + return item + + +async def get_skill_tree(db: AsyncSession, slug: str) -> list[dict[str, Any]]: + item = await get_skill_or_raise(db, slug) + skill_dir = _resolve_skill_dir(item) + if not skill_dir.exists() or not skill_dir.is_dir(): + raise ValueError(f"技能目录不存在: {item.dir_path}") + return _build_tree(skill_dir, skill_dir) + + +async def read_skill_file(db: AsyncSession, slug: str, relative_path: str) -> dict[str, Any]: + item = await get_skill_or_raise(db, slug) + skill_dir = _resolve_skill_dir(item) + target, rel = _resolve_relative_path(skill_dir, relative_path) + if not target.exists() or not target.is_file(): + raise ValueError(f"文件不存在: {relative_path}") + if not _is_text_path(target): + raise ValueError("仅支持读取文本文件") + try: + content = target.read_text(encoding="utf-8") + except UnicodeDecodeError as e: + raise ValueError(f"文件编码不支持（仅支持 UTF-8）: {e}") from e + + return {"path": rel, "content": content} + + +async def create_skill_node( + db: AsyncSession, + *, + slug: str, + relative_path: str, + is_dir: bool, + content: str | None, + updated_by: str | None, +) -> None: + item = await get_skill_or_raise(db, slug) + skill_dir = _resolve_skill_dir(item) + target, _ = _resolve_relative_path(skill_dir, relative_path) + if target.exists(): + raise ValueError("目标已存在") + + if is_dir: + target.mkdir(parents=True, exist_ok=False) + return + + if not _is_text_path(target): + raise ValueError("仅支持创建文本文件") + + parsed_name: str | None = None + parsed_desc: str | None = None + if target.name == "SKILL.md" and target.parent == skill_dir: + parsed_name, parsed_desc, _ = _parse_skill_markdown(content or "") + if parsed_name != item.slug: + raise ValueError("SKILL.md frontmatter.name 必须与 skill slug 一致") + + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content or "", encoding="utf-8") + + if parsed_name is not None and parsed_desc is not None: + repo = SkillRepository(db) + await repo.update_metadata(item, name=parsed_name, description=parsed_desc, updated_by=updated_by) + + +async def update_skill_file( + db: AsyncSession, + *, + slug: str, + relative_path: str, + content: str, + updated_by: str | None, +) -> None: + item = await get_skill_or_raise(db, slug) + skill_dir = _resolve_skill_dir(item) + target, _ = _resolve_relative_path(skill_dir, relative_path) + if not target.exists() or not target.is_file(): + raise ValueError("文件不存在") + if not _is_text_path(target): + raise ValueError("仅支持编辑文本文件") + + parsed_name = None + parsed_desc = None + if target.name == "SKILL.md" and target.parent == skill_dir: + parsed_name, parsed_desc, _ = _parse_skill_markdown(content) + if parsed_name != item.slug: + raise ValueError("SKILL.md frontmatter.name 必须与 skill slug 一致") + + target.write_text(content, encoding="utf-8") + + if parsed_name is not None and parsed_desc is not None: + repo = SkillRepository(db) + await repo.update_metadata(item, name=parsed_name, description=parsed_desc, updated_by=updated_by) + + +async def delete_skill_node(db: AsyncSession, *, slug: str, relative_path: str) -> None: + item = await get_skill_or_raise(db, slug) + skill_dir = _resolve_skill_dir(item) + target, rel = _resolve_relative_path(skill_dir, relative_path, allow_root=False) + if not target.exists(): + raise ValueError("目标不存在") + + if rel == "SKILL.md": + raise ValueError("不允许删除根目录 SKILL.md") + + if target.is_dir(): + shutil.rmtree(target) + else: + target.unlink() + + +async def export_skill_zip(db: AsyncSession, slug: str) -> tuple[str, str]: + item = await get_skill_or_raise(db, slug) + skill_dir = _resolve_skill_dir(item) + if not skill_dir.exists() or not skill_dir.is_dir(): + raise ValueError("技能目录不存在") + + fd, export_path = tempfile.mkstemp(prefix=f"skill-{slug}-", suffix=".zip") + Path(export_path).unlink(missing_ok=True) + export_file = Path(export_path) + try: + with zipfile.ZipFile(export_file, "w", compression=zipfile.ZIP_DEFLATED) as zf: + for p in skill_dir.rglob("*"): + arcname = Path(slug) / p.relative_to(skill_dir) + zf.write(p, arcname.as_posix()) + except Exception: + export_file.unlink(missing_ok=True) + raise + return export_path, f"{slug}.zip" + + +async def delete_skill(db: AsyncSession, *, slug: str) -> None: + repo = SkillRepository(db) + item = await repo.get_by_slug(slug) + if not item: + raise ValueError(f"技能 '{slug}' 不存在") + + skill_dir = _resolve_skill_dir(item) + trash_dir: Path | None = None + + if skill_dir.exists(): + trash_dir = skill_dir.with_name(f".deleted-{slug}-{uuid.uuid4().hex[:8]}") + skill_dir.rename(trash_dir) + + try: + await repo.delete(item) + except Exception: + if trash_dir and trash_dir.exists(): + trash_dir.rename(skill_dir) + raise + + if trash_dir and trash_dir.exists(): + shutil.rmtree(trash_dir, ignore_errors=True) diff --git a/src/storage/db/models.py b/src/storage/db/models.py index c1edfc3e..7c464c05 100644 --- a/src/storage/db/models.py +++ b/src/storage/db/models.py @@ -341,6 +341,7 @@ class MCPServer(Base): url = Column(String(500), nullable=True, comment="服务器 URL（sse/streamable_http）") command = Column(String(500), nullable=True, comment="命令（stdio）") args = Column(JSON, nullable=True, comment="命令参数数组（stdio）") + env = Column(JSON, nullable=True, comment="环境变量（stdio）") headers = Column(JSON, nullable=True, comment="HTTP 请求头") timeout = Column(Integer, nullable=True, comment="HTTP 超时时间（秒）") sse_read_timeout = Column(Integer, nullable=True, comment="SSE 读取超时（秒）") @@ -369,6 +370,7 @@ class MCPServer(Base): "url": self.url, "command": self.command, "args": self.args or [], + "env": self.env or {}, "headers": self.headers or {}, "timeout": self.timeout, "sse_read_timeout": self.sse_read_timeout, @@ -402,6 +404,14 @@ class MCPServer(Base): config["args"] = json.loads(self.args) except json.JSONDecodeError: pass + if self.transport == "stdio" and self.env: + if isinstance(self.env, dict): + config["env"] = self.env + elif isinstance(self.env, str): + try: + config["env"] = json.loads(self.env) + except json.JSONDecodeError: + pass # headers 只用于 sse/streamable_http 传输类型 if self.transport in ("sse", "streamable_http") and self.headers: if isinstance(self.headers, dict): diff --git a/src/storage/postgres/manager.py b/src/storage/postgres/manager.py index a4fa9254..b499a568 100644 --- a/src/storage/postgres/manager.py +++ b/src/storage/postgres/manager.py @@ -164,6 +164,19 @@ class PostgresManager(metaclass=SingletonMeta): for stmt in stmts: await conn.execute(text(stmt)) + async def ensure_business_schema(self): + """确保业务 schema 包含后续新增字段（兼容已存在表）。""" + self._check_initialized() + stmts = [ + "ALTER TABLE IF EXISTS skills ADD COLUMN IF NOT EXISTS tool_dependencies JSONB DEFAULT '[]'::jsonb", + "ALTER TABLE IF EXISTS skills ADD COLUMN IF NOT EXISTS mcp_dependencies JSONB DEFAULT '[]'::jsonb", + "ALTER TABLE IF EXISTS skills ADD COLUMN IF NOT EXISTS skill_dependencies JSONB DEFAULT '[]'::jsonb", + "ALTER TABLE IF EXISTS mcp_servers ADD COLUMN IF NOT EXISTS env JSONB", + ] + async with self.async_engine.begin() as conn: + for stmt in stmts: + await conn.execute(text(stmt)) + @property def is_postgresql(self) -> bool: """检查是否是 PostgreSQL 数据库""" diff --git a/src/storage/postgres/models_business.py b/src/storage/postgres/models_business.py index fd5c9ebe..b7e57478 100644 --- a/src/storage/postgres/models_business.py +++ b/src/storage/postgres/models_business.py @@ -171,6 +171,41 @@ class AgentConfig(Base): } +class Skill(Base): + """Skill 元数据模型（内容存文件系统，索引存数据库）""" + + __tablename__ = "skills" + + id = Column(Integer, primary_key=True, autoincrement=True) + slug = Column(String(128), nullable=False, unique=True, index=True, comment="技能唯一标识（目录名）") + name = Column(String(128), nullable=False, comment="技能名称（来自 SKILL.md frontmatter.name）") + description = Column(Text, nullable=False, comment="技能描述（来自 SKILL.md frontmatter.description）") + tool_dependencies = Column(JSON, nullable=False, default=list, comment="依赖的内置工具名列表") + mcp_dependencies = Column(JSON, nullable=False, default=list, comment="依赖的 MCP 服务名列表") + skill_dependencies = Column(JSON, nullable=False, default=list, comment="依赖的其他 skill slug 列表") + dir_path = Column(String(512), nullable=False, comment="技能目录路径（相对 save_dir）") + created_by = Column(String(64), nullable=True) + updated_by = Column(String(64), nullable=True) + created_at = Column(DateTime, default=utc_now_naive) + updated_at = Column(DateTime, default=utc_now_naive, onupdate=utc_now_naive) + + def to_dict(self) -> dict[str, Any]: + return { + "id": self.id, + "slug": self.slug, + "name": self.name, + "description": self.description, + "tool_dependencies": self.tool_dependencies or [], + "mcp_dependencies": self.mcp_dependencies or [], + "skill_dependencies": self.skill_dependencies or [], + "dir_path": self.dir_path, + "created_by": self.created_by, + "updated_by": self.updated_by, + "created_at": format_utc_datetime(self.created_at), + "updated_at": format_utc_datetime(self.updated_at), + } + + class Conversation(Base): """Conversation table - 对话表""" @@ -381,6 +416,7 @@ class MCPServer(Base): url = Column(String(500), nullable=True, comment="服务器 URL（sse/streamable_http）") command = Column(String(500), nullable=True, comment="命令（stdio）") args = Column(JSON, nullable=True, comment="命令参数数组（stdio）") + env = Column(JSON, nullable=True, comment="环境变量（stdio）") headers = Column(JSON, nullable=True, comment="HTTP 请求头") timeout = Column(Integer, nullable=True, comment="HTTP 超时时间（秒）") sse_read_timeout = Column(Integer, nullable=True, comment="SSE 读取超时（秒）") @@ -409,6 +445,7 @@ class MCPServer(Base): "url": self.url, "command": self.command, "args": self.args or [], + "env": self.env or {}, "headers": self.headers or {}, "timeout": self.timeout, "sse_read_timeout": self.sse_read_timeout, @@ -440,6 +477,14 @@ class MCPServer(Base): config["args"] = json.loads(self.args) except json.JSONDecodeError: pass + if self.transport == "stdio" and self.env: + if isinstance(self.env, dict): + config["env"] = self.env + elif isinstance(self.env, str): + try: + config["env"] = json.loads(self.env) + except json.JSONDecodeError: + pass # headers 只用于 sse/streamable_http 传输类型 if self.transport in ("sse", "streamable_http") and self.headers: if isinstance(self.headers, dict): diff --git a/test/api/test_evaluation_router.py b/test/api/test_evaluation_router.py new file mode 100644 index 00000000..c90e6492 --- /dev/null +++ b/test/api/test_evaluation_router.py @@ -0,0 +1,62 @@ +""" +Integration tests for evaluation router endpoints. +""" + +from __future__ import annotations + +import uuid + +import pytest + +pytestmark = [pytest.mark.asyncio, pytest.mark.integration] + + +async def _upload_test_benchmark(test_client, admin_headers: dict[str, str], db_id: str) -> tuple[str, str]: + benchmark_name = f"pytest_benchmark_{uuid.uuid4().hex[:8]}" + line = '{"query":"什么是单元测试？","gold_answer":"用于验证代码行为的自动化测试"}\n' + + response = await test_client.post( + f"/api/evaluation/databases/{db_id}/benchmarks/upload", + data={"name": benchmark_name, "description": "pytest benchmark for download"}, + files={"file": ("pytest_benchmark.jsonl", line.encode("utf-8"), "application/x-ndjson")}, + headers=admin_headers, + ) + assert response.status_code == 200, response.text + + payload = response.json() + assert payload.get("message") == "success" + benchmark_id = payload.get("data", {}).get("benchmark_id") + assert benchmark_id + return benchmark_id, line + + +async def test_download_benchmark_requires_admin(test_client, standard_user): + response = await test_client.get( + "/api/evaluation/benchmarks/benchmark_fake/download", + headers=standard_user["headers"], + ) + assert response.status_code == 403 + + +async def test_admin_can_download_benchmark(test_client, admin_headers, knowledge_database): + benchmark_id, expected_line = await _upload_test_benchmark(test_client, admin_headers, knowledge_database["db_id"]) + + response = await test_client.get( + f"/api/evaluation/benchmarks/{benchmark_id}/download", + headers=admin_headers, + ) + assert response.status_code == 200, response.text + assert "application/x-ndjson" in response.headers.get("content-type", "") + assert "attachment" in response.headers.get("content-disposition", "").lower() + + content = response.content.decode("utf-8") + assert expected_line.strip() in content + + +async def test_download_benchmark_not_found(test_client, admin_headers): + response = await test_client.get( + f"/api/evaluation/benchmarks/benchmark_not_found_{uuid.uuid4().hex[:8]}/download", + headers=admin_headers, + ) + assert response.status_code == 404, response.text + diff --git a/test/api/test_knowledge_router.py b/test/api/test_knowledge_router.py index 29fa366f..51a41bba 100644 --- a/test/api/test_knowledge_router.py +++ b/test/api/test_knowledge_router.py @@ -40,6 +40,59 @@ async def test_admin_can_manage_knowledge_databases(test_client, admin_headers, assert update_response.json()["database"]["description"] == "Updated by pytest" +async def test_create_database_with_chunk_preset(test_client, admin_headers): + db_name = f"pytest_chunk_preset_{uuid.uuid4().hex[:6]}" + payload = { + "database_name": db_name, + "description": "Chunk preset create test", + "embed_model_name": "siliconflow/BAAI/bge-m3", + "kb_type": "milvus", + "additional_params": {"chunk_preset_id": "book"}, + } + + create_response = await test_client.post("/api/knowledge/databases", json=payload, headers=admin_headers) + assert create_response.status_code == 200, create_response.text + db_id = create_response.json()["db_id"] + + info_response = await test_client.get(f"/api/knowledge/databases/{db_id}", headers=admin_headers) + assert info_response.status_code == 200, info_response.text + assert info_response.json()["additional_params"]["chunk_preset_id"] == "book" + + delete_response = await test_client.delete(f"/api/knowledge/databases/{db_id}", headers=admin_headers) + assert delete_response.status_code == 200, delete_response.text + + +async def test_update_database_additional_params_merge_keeps_chunk_preset( + test_client, admin_headers, knowledge_database +): + db_id = knowledge_database["db_id"] + + first_update = await test_client.put( + f"/api/knowledge/databases/{db_id}", + json={ + "name": knowledge_database["name"], + "description": "update with chunk preset", + "additional_params": {"chunk_preset_id": "qa"}, + }, + headers=admin_headers, + ) + assert first_update.status_code == 200, first_update.text + + second_update = await test_client.put( + f"/api/knowledge/databases/{db_id}", + json={ + "name": knowledge_database["name"], + "description": "update without additional params", + }, + headers=admin_headers, + ) + assert second_update.status_code == 200, second_update.text + + info_response = await test_client.get(f"/api/knowledge/databases/{db_id}", headers=admin_headers) + assert info_response.status_code == 200, info_response.text + assert info_response.json()["additional_params"]["chunk_preset_id"] == "qa" + + async def test_knowledge_routes_enforce_permissions(test_client, standard_user, knowledge_database): db_id = knowledge_database["db_id"] diff --git a/test/test_conversation_service_attachment_state.py b/test/test_conversation_service_attachment_state.py index fb04048a..5155f1ee 100644 --- a/test/test_conversation_service_attachment_state.py +++ b/test/test_conversation_service_attachment_state.py @@ -80,8 +80,8 @@ async def test_sync_thread_attachment_state_updates_graph(monkeypatch: pytest.Mo assert captured["write_config"] == {"configurable": {"thread_id": "thread-1", "user_id": "u1"}} assert captured["write_values"]["attachments"] == attachments assert "/attachments/resume.md" in captured["write_values"]["files"] - assert "/attachments/old.md" not in captured["write_values"]["files"] - assert "/work/result.md" in captured["write_values"]["files"] + assert captured["write_values"]["files"]["/attachments/old.md"] is None + assert "/work/result.md" not in captured["write_values"]["files"] @pytest.mark.asyncio diff --git a/test/test_ragflow_like_chunking.py b/test/test_ragflow_like_chunking.py new file mode 100644 index 00000000..ad270655 --- /dev/null +++ b/test/test_ragflow_like_chunking.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +import os +import sys + +sys.path.append(os.getcwd()) + +from src.knowledge.chunking.ragflow_like.dispatcher import chunk_markdown +from src.knowledge.chunking.ragflow_like.nlp import bullets_category +from src.knowledge.chunking.ragflow_like.presets import ( + CHUNK_ENGINE_VERSION, + get_chunk_preset_options, + map_to_internal_parser_id, + resolve_chunk_processing_params, +) + + +def test_general_maps_to_naive() -> None: + assert map_to_internal_parser_id("general") == "naive" + + +def test_resolve_chunk_processing_params_priority() -> None: + resolved = resolve_chunk_processing_params( + kb_additional_params={ + "chunk_preset_id": "book", + "chunk_parser_config": {"chunk_token_num": 300, "delimiter": "\\n"}, + }, + file_processing_params={ + "chunk_preset_id": "qa", + "chunk_parser_config": {"delimiter": "###"}, + }, + request_params={ + "chunk_preset_id": "laws", + "chunk_parser_config": {"chunk_token_num": 666}, + "chunk_size": 777, + }, + ) + + assert resolved["chunk_preset_id"] == "laws" + assert resolved["chunk_engine_version"] == CHUNK_ENGINE_VERSION + # legacy chunk_size 在当前实现里会映射为 chunk_token_num + assert resolved["chunk_parser_config"]["chunk_token_num"] == 777 + assert resolved["chunk_parser_config"]["delimiter"] == "###" + + +def test_qa_chunking_from_markdown_headings() -> None: + content = """ +# 问题一 +这是答案一。 + +## 子问题 +这是答案二。 +""".strip() + + chunks = chunk_markdown( + markdown_content=content, + file_id="file_1", + filename="faq.md", + processing_params={"chunk_preset_id": "qa", "chunk_parser_config": {}}, + ) + + assert len(chunks) >= 1 + assert "问题：" in chunks[0]["content"] + assert "回答：" in chunks[0]["content"] + + +def test_book_chunking_hierarchical_merge() -> None: + content = """ +第一章 总则 +第一节 适用范围 +本规范适用于测试场景。 +第二节 基本原则 +应当遵循最小改动原则。 +""".strip() + + chunks = chunk_markdown( + markdown_content=content, + file_id="file_2", + filename="book.txt", + processing_params={"chunk_preset_id": "book", "chunk_parser_config": {"chunk_token_num": 256}}, + ) + + assert len(chunks) >= 1 + assert any("第一章" in ck["content"] for ck in chunks) + + +def test_markdown_heading_has_higher_weight_in_bullet_category() -> None: + sections = [ + "# 3.2 个人所得项目及计税、申报方式概括", + "一、关于季节工、临时工等费用税前扣除问题，以下规定继续执行。", + "二、根据现行规定，补贴收入应并入工资薪金所得。", + "（一）从超出国家规定比例支付的补贴，不属于免税福利费。", + ] + + # 命中 markdown 标题模式（BULLET_PATTERN 下标 4）时，应该优先选中该组。 + assert bullets_category(sections) == 4 + + +def test_mid_sentence_bullet_marker_should_not_be_treated_as_heading() -> None: + sections = [ + "根据前述规则：一、这里是句中枚举，不是章节标题，不能被当成层级。", + "延续上文：（二）这里同样是正文中的枚举表达，不是独立标题。", + "## 3.4 交通补贴的个税处理", + ] + assert bullets_category(sections) == 4 + + +def test_chunk_preset_options_include_description() -> None: + options = get_chunk_preset_options() + assert len(options) == 4 + assert all(isinstance(option.get("description"), str) and option["description"] for option in options) diff --git a/test/test_runtime_config_middleware_skills.py b/test/test_runtime_config_middleware_skills.py new file mode 100644 index 00000000..74d43278 --- /dev/null +++ b/test/test_runtime_config_middleware_skills.py @@ -0,0 +1,352 @@ +from __future__ import annotations + +from dataclasses import dataclass +from types import SimpleNamespace +from typing import Any + +import pytest +from langchain_core.messages import SystemMessage, ToolMessage +from langgraph.types import Command + +import src.agents.common.middlewares.runtime_config_middleware as runtime_middleware +from src.agents.common.middlewares.runtime_config_middleware import RuntimeConfigMiddleware + + +@dataclass +class _FakeTool: + name: str + + +@dataclass +class _FakeRequest: + runtime: Any + tools: list[Any] + system_message: SystemMessage + state: dict[str, Any] + + def override(self, **kwargs): + return _FakeRequest( + runtime=kwargs.get("runtime", self.runtime), + tools=kwargs.get("tools", self.tools), + system_message=kwargs.get("system_message", self.system_message), + state=kwargs.get("state", self.state), + ) + + +@dataclass +class _FakeToolCallRequest: + tool_call: dict[str, Any] + runtime: Any + state: dict[str, Any] + + +async def _echo_handler(request): + return request + + +def _build_request(*, skills: list[str], tools: list[str], system_prompt: str = "你是助手", state=None) -> _FakeRequest: + context = SimpleNamespace(system_prompt=system_prompt, skills=skills, tools=[], knowledges=[], mcps=[]) + runtime = SimpleNamespace(context=context) + return _FakeRequest( + runtime=runtime, + tools=[_FakeTool(name=name) for name in tools], + system_message=SystemMessage(content=[{"type": "text", "text": "base"}]), + state=state or {}, + ) + + +def _build_tool_request(*, skills: list[str], visible_skills: list[str], file_path: str) -> _FakeToolCallRequest: + return _FakeToolCallRequest( + tool_call={"name": "read_file", "args": {"file_path": file_path}}, + runtime=SimpleNamespace(context=SimpleNamespace(skills=skills)), + state={ + "skill_session_snapshot": { + "selected_skills": skills, + "visible_skills": visible_skills, + "prompt_metadata": {}, + "dependency_map": {}, + } + }, + ) + + +def _extract_appended_prompt(request: _FakeRequest) -> str: + return request.system_message.content_blocks[-1]["text"] + + +def _build_middleware() -> RuntimeConfigMiddleware: + return RuntimeConfigMiddleware( + enable_model_override=False, + enable_tools_override=False, + enable_system_prompt_override=True, + enable_skills_prompt_override=True, + ) + + +def _build_snapshot(selected: list[str], metadata: dict[str, dict[str, str]] | None = None) -> dict[str, Any]: + return { + "selected_skills": selected, + "visible_skills": selected, + "prompt_metadata": metadata or {}, + "dependency_map": {}, + } + + +@pytest.mark.asyncio +async def test_injects_skills_section_when_skills_configured_and_read_file_available(monkeypatch: pytest.MonkeyPatch): + async def fake_resolve(selected): + assert selected == ["research-report"] + return _build_snapshot( + ["research-report"], + { + "research-report": { + "name": "research-report", + "description": "Write structured research reports", + "path": "/skills/research-report/SKILL.md", + } + }, + ) + + monkeypatch.setattr(runtime_middleware, "resolve_session_snapshot", fake_resolve) + middleware = _build_middleware() + request = _build_request(skills=["research-report"], tools=["read_file"]) + + result = await middleware.awrap_model_call(request, _echo_handler) + prompt = _extract_appended_prompt(result) + + assert "## Skills System" in prompt + assert "**Skills Skills**: `/skills/` (higher priority)" in prompt + assert "- **research-report**: Write structured research reports" in prompt + assert "Read `/skills/research-report/SKILL.md` for full instructions" in prompt + assert "Recognize when a skill applies" in prompt + assert "当前时间：" in prompt + assert "skill_session_snapshot" in result.state + + +@pytest.mark.asyncio +async def test_skips_skills_section_when_context_skills_empty(): + middleware = _build_middleware() + request = _build_request(skills=[], tools=["read_file"]) + + result = await middleware.awrap_model_call(request, _echo_handler) + prompt = _extract_appended_prompt(result) + + assert "## Skills System" not in prompt + + +@pytest.mark.asyncio +async def test_skips_skills_section_without_read_file_and_logs_warning(monkeypatch: pytest.MonkeyPatch): + warnings: list[str] = [] + fake_logger = SimpleNamespace( + debug=lambda *_args, **_kwargs: None, + warning=lambda message: warnings.append(message), + ) + monkeypatch.setattr(runtime_middleware, "logger", fake_logger) + + async def fake_resolve(_selected): + return _build_snapshot( + ["research-report"], + { + "research-report": { + "name": "research-report", + "description": "Write structured research reports", + "path": "/skills/research-report/SKILL.md", + } + }, + ) + + monkeypatch.setattr(runtime_middleware, "resolve_session_snapshot", fake_resolve) + middleware = _build_middleware() + request = _build_request(skills=["research-report"], tools=["write_file"]) + + result = await middleware.awrap_model_call(request, _echo_handler) + prompt = _extract_appended_prompt(result) + + assert "## Skills System" not in prompt + assert any("read_file unavailable" in msg for msg in warnings) + + +@pytest.mark.asyncio +async def test_injects_skills_in_input_order_with_dedup_and_invalid_slug_skipped(monkeypatch: pytest.MonkeyPatch): + async def fake_resolve(_selected): + return _build_snapshot( + ["beta", "missing", "alpha", "beta"], + { + "beta": {"name": "beta", "description": "beta skill", "path": "/skills/beta/SKILL.md"}, + "alpha": {"name": "alpha", "description": "alpha skill", "path": "/skills/alpha/SKILL.md"}, + }, + ) + + monkeypatch.setattr(runtime_middleware, "resolve_session_snapshot", fake_resolve) + middleware = _build_middleware() + request = _build_request(skills=["beta", "missing", "alpha", "beta"], tools=["read_file"]) + + result = await middleware.awrap_model_call(request, _echo_handler) + prompt = _extract_appended_prompt(result) + + beta_line = "- **beta**: beta skill" + alpha_line = "- **alpha**: alpha skill" + assert beta_line in prompt + assert alpha_line in prompt + assert prompt.find(beta_line) < prompt.find(alpha_line) + assert prompt.count(beta_line) == 1 + assert "missing" not in prompt + + +@pytest.mark.asyncio +async def test_awrap_tool_call_activates_skill_when_read_skill_md(): + middleware = _build_middleware() + request = _build_tool_request( + skills=["research-report"], + visible_skills=["research-report"], + file_path="/skills/research-report/SKILL.md", + ) + + async def _handler(_request): + return ToolMessage(content="ok", tool_call_id="tc-1") + + result = await middleware.awrap_tool_call(request, _handler) + assert isinstance(result, Command) + assert result.update["activated_skills"] == ["research-report"] + assert len(result.update["messages"]) == 1 + + +@pytest.mark.asyncio +async def test_awrap_tool_call_skips_invalid_skill_slug_path(): + middleware = _build_middleware() + request = _build_tool_request( + skills=["research-report"], + visible_skills=["research-report"], + file_path="/skills/../SKILL.md", + ) + + async def _handler(_request): + return ToolMessage(content="ok", tool_call_id="tc-1") + + result = await middleware.awrap_tool_call(request, _handler) + assert isinstance(result, ToolMessage) + + +@pytest.mark.asyncio +async def test_awrap_tool_call_merges_with_existing_command_update(): + middleware = _build_middleware() + request = _build_tool_request( + skills=["research-report"], + visible_skills=["research-report"], + file_path="/skills/research-report/SKILL.md", + ) + + async def _handler(_request): + return Command(update={"messages": [ToolMessage(content="ok", tool_call_id="tc-1")], "activated_skills": ["a"]}) + + result = await middleware.awrap_tool_call(request, _handler) + assert isinstance(result, Command) + assert result.update["activated_skills"] == ["a", "research-report"] + + +@pytest.mark.asyncio +async def test_awrap_tool_call_denies_invisible_skill(): + middleware = _build_middleware() + request = _build_tool_request( + skills=["research-report"], + visible_skills=["alpha"], + file_path="/skills/research-report/SKILL.md", + ) + + async def _handler(_request): + return ToolMessage(content="ok", tool_call_id="tc-1") + + result = await middleware.awrap_tool_call(request, _handler) + assert isinstance(result, ToolMessage) + + +@pytest.mark.asyncio +async def test_model_call_injects_dependency_tools_and_mcps_after_activation(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr( + runtime_middleware, + "get_buildin_tools", + lambda: [_FakeTool(name="calculator"), _FakeTool(name="dep-tool")], + ) + monkeypatch.setattr(runtime_middleware, "get_kb_based_tools", lambda db_names=None: []) + monkeypatch.setattr( + runtime_middleware, + "build_dependency_bundle", + lambda _snapshot, activated: {"tools": ["dep-tool"], "mcps": ["mcp-a"], "skills": activated}, + ) + + async def fake_get_enabled_mcp_tools(server_name: str): + if server_name == "mcp-a": + return [_FakeTool(name="mcp_tool")] + return [] + + monkeypatch.setattr(runtime_middleware, "get_enabled_mcp_tools", fake_get_enabled_mcp_tools) + + middleware = RuntimeConfigMiddleware( + extra_tools=[_FakeTool(name="mcp_tool")], + enable_model_override=False, + enable_tools_override=True, + enable_system_prompt_override=False, + enable_skills_prompt_override=False, + ) + + request = _build_request( + skills=[], + tools=["calculator", "dep-tool", "mcp_tool", "read_file"], + state={"activated_skills": ["alpha"], "skill_session_snapshot": _build_snapshot([])}, + ) + + result = await middleware.awrap_model_call(request, _echo_handler) + tool_names = [t.name for t in result.tools] + assert "dep-tool" in tool_names + assert "mcp_tool" in tool_names + assert "calculator" not in tool_names + + +@pytest.mark.asyncio +async def test_model_call_reuses_snapshot_until_skills_changed(monkeypatch: pytest.MonkeyPatch): + called = {"count": 0} + + async def fake_resolve(selected): + called["count"] += 1 + return _build_snapshot(selected) + + monkeypatch.setattr(runtime_middleware, "resolve_session_snapshot", fake_resolve) + + middleware = _build_middleware() + req1 = _build_request(skills=["alpha"], tools=["read_file"], state={}) + res1 = await middleware.awrap_model_call(req1, _echo_handler) + assert called["count"] == 1 + + req2 = _build_request(skills=["alpha"], tools=["read_file"], state=res1.state) + await middleware.awrap_model_call(req2, _echo_handler) + assert called["count"] == 1 + + req3 = _build_request(skills=["beta"], tools=["read_file"], state=res1.state) + await middleware.awrap_model_call(req3, _echo_handler) + assert called["count"] == 2 + + +@pytest.mark.asyncio +async def test_injects_dependency_skills_into_prompt(monkeypatch: pytest.MonkeyPatch): + async def fake_resolve(_selected): + return { + "selected_skills": ["alpha"], + "visible_skills": ["alpha", "beta"], + "prompt_metadata": { + "alpha": {"name": "alpha", "description": "alpha desc", "path": "/skills/alpha/SKILL.md"}, + "beta": {"name": "beta", "description": "beta desc", "path": "/skills/beta/SKILL.md"}, + }, + "dependency_map": { + "alpha": {"tools": [], "mcps": [], "skills": ["beta"]}, + "beta": {"tools": [], "mcps": [], "skills": []}, + }, + } + + monkeypatch.setattr(runtime_middleware, "resolve_session_snapshot", fake_resolve) + middleware = _build_middleware() + request = _build_request(skills=["alpha"], tools=["read_file"]) + + result = await middleware.awrap_model_call(request, _echo_handler) + prompt = _extract_appended_prompt(result) + assert "- **alpha**: alpha desc" in prompt + assert "- **beta**: beta desc" in prompt diff --git a/test/test_skill_resolver.py b/test/test_skill_resolver.py new file mode 100644 index 00000000..aef7585a --- /dev/null +++ b/test/test_skill_resolver.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +import pytest + +from src.services import skill_resolver as resolver +from src.storage.postgres.models_business import Skill + + +def test_expand_skill_closure_and_dependency_bundle(): + dependency_map = { + "alpha": {"tools": ["t1"], "mcps": ["m1"], "skills": ["beta"]}, + "beta": {"tools": ["t2"], "mcps": ["m2"], "skills": ["gamma"]}, + "gamma": {"tools": ["t3"], "mcps": [], "skills": []}, + } + snapshot: resolver.SkillSessionSnapshot = { + "selected_skills": ["alpha"], + "visible_skills": ["alpha", "beta", "gamma"], + "prompt_metadata": {}, + "dependency_map": dependency_map, + } + + closure = resolver.expand_skill_closure(["alpha"], dependency_map) + assert closure == ["alpha", "beta", "gamma"] + + bundle = resolver.build_dependency_bundle(snapshot, ["alpha"]) + assert bundle["skills"] == ["alpha", "beta", "gamma"] + assert bundle["tools"] == ["t1", "t2", "t3"] + assert bundle["mcps"] == ["m1", "m2"] + + +def test_expand_skill_closure_cycle(): + dependency_map = { + "alpha": {"tools": [], "mcps": [], "skills": ["beta"]}, + "beta": {"tools": [], "mcps": [], "skills": ["alpha"]}, + } + assert resolver.expand_skill_closure(["alpha"], dependency_map) == ["alpha", "beta"] + + +def test_collect_prompt_metadata_order_and_dedup(): + snapshot: resolver.SkillSessionSnapshot = { + "selected_skills": ["beta", "alpha"], + "visible_skills": ["beta", "alpha"], + "prompt_metadata": { + "beta": {"name": "beta", "description": "beta skill", "path": "/skills/beta/SKILL.md"}, + "alpha": {"name": "alpha", "description": "alpha skill", "path": "/skills/alpha/SKILL.md"}, + }, + "dependency_map": {}, + } + result = resolver.collect_prompt_metadata(snapshot, ["beta", "missing", "alpha", "beta"]) + assert [item["name"] for item in result] == ["beta", "alpha"] + assert [item["path"] for item in result] == ["/skills/beta/SKILL.md", "/skills/alpha/SKILL.md"] + + +@pytest.mark.asyncio +async def test_resolve_session_snapshot_and_selected_change(monkeypatch: pytest.MonkeyPatch): + async def fake_list_skills(_db=None): + return [ + Skill( + slug="alpha", + name="alpha", + description="a", + tool_dependencies=[], + mcp_dependencies=[], + skill_dependencies=["beta"], + dir_path="skills/alpha", + ), + Skill( + slug="beta", + name="beta", + description="b", + tool_dependencies=[], + mcp_dependencies=[], + skill_dependencies=[], + dir_path="skills/beta", + ), + ] + + monkeypatch.setattr(resolver, "_list_skills_from_db", fake_list_skills) + + snapshot = await resolver.resolve_session_snapshot([" alpha ", "alpha"]) + assert snapshot["selected_skills"] == ["alpha"] + assert snapshot["visible_skills"] == ["alpha", "beta"] + + assert resolver.is_snapshot_match_selected_skills(snapshot, ["alpha"]) is True + assert resolver.is_snapshot_match_selected_skills(snapshot, ["beta"]) is False diff --git a/test/test_skill_router.py b/test/test_skill_router.py new file mode 100644 index 00000000..74fe3c04 --- /dev/null +++ b/test/test_skill_router.py @@ -0,0 +1,165 @@ +from __future__ import annotations + +from fastapi import FastAPI, HTTPException +from fastapi.testclient import TestClient + +from server.routers.skill_router import skills +from server.utils.auth_middleware import get_admin_user, get_db, get_superadmin_user +from src.storage.postgres.models_business import Skill, User + + +def _build_app(*, allow_superadmin: bool) -> FastAPI: + app = FastAPI() + app.include_router(skills, prefix="/api") + + async def fake_db(): + return None + + async def fake_admin_user(): + return User( + username="admin", + user_id="admin", + password_hash="x", + role="admin", + ) + + async def fake_superadmin_user(): + if not allow_superadmin: + raise HTTPException(status_code=403, detail="需要超级管理员权限") + return User( + username="root", + user_id="root", + password_hash="x", + role="superadmin", + ) + + app.dependency_overrides[get_db] = fake_db + app.dependency_overrides[get_admin_user] = fake_admin_user + app.dependency_overrides[get_superadmin_user] = fake_superadmin_user + return app + + +def test_list_skills_route_returns_data(monkeypatch): + async def fake_list_skills(_db): + return [ + Skill( + slug="demo", + name="demo", + description="demo skill", + dir_path="skills/demo", + ) + ] + + monkeypatch.setattr("server.routers.skill_router.list_skills", fake_list_skills) + + app = _build_app(allow_superadmin=True) + client = TestClient(app) + resp = client.get("/api/system/skills") + assert resp.status_code == 200, resp.text + payload = resp.json() + assert payload["success"] is True + assert payload["data"][0]["slug"] == "demo" + + +def test_import_skill_requires_superadmin(): + app = _build_app(allow_superadmin=False) + client = TestClient(app) + + resp = client.post( + "/api/system/skills/import", + files={"file": ("demo.zip", b"not zip", "application/zip")}, + ) + assert resp.status_code == 403 + + +def test_update_skill_file_passes_operator(monkeypatch): + captured: dict[str, str] = {} + + async def fake_update_skill_file(_db, *, slug, relative_path, content, updated_by): + captured["slug"] = slug + captured["relative_path"] = relative_path + captured["content"] = content + captured["updated_by"] = updated_by + + monkeypatch.setattr("server.routers.skill_router.update_skill_file", fake_update_skill_file) + + app = _build_app(allow_superadmin=True) + client = TestClient(app) + + resp = client.put( + "/api/system/skills/demo/file", + json={ + "path": "SKILL.md", + "content": "---\nname: demo\ndescription: demo\n---\n# Demo\n", + }, + ) + assert resp.status_code == 200, resp.text + assert captured["slug"] == "demo" + assert captured["relative_path"] == "SKILL.md" + assert captured["updated_by"] == "root" + + +def test_dependency_options_route(monkeypatch): + async def fake_get_skill_dependency_options(_db): + return { + "tools": ["calculator"], + "mcps": ["mcp-a"], + "skills": ["demo"], + } + + monkeypatch.setattr("server.routers.skill_router.get_skill_dependency_options", fake_get_skill_dependency_options) + + app = _build_app(allow_superadmin=True) + client = TestClient(app) + resp = client.get("/api/system/skills/dependency-options") + assert resp.status_code == 200, resp.text + payload = resp.json() + assert payload["success"] is True + assert payload["data"]["tools"] == ["calculator"] + + +def test_update_skill_dependencies_route(monkeypatch): + captured: dict[str, object] = {} + + async def fake_update_skill_dependencies( + _db, + *, + slug, + tool_dependencies, + mcp_dependencies, + skill_dependencies, + updated_by, + ): + captured["slug"] = slug + captured["tool_dependencies"] = tool_dependencies + captured["mcp_dependencies"] = mcp_dependencies + captured["skill_dependencies"] = skill_dependencies + captured["updated_by"] = updated_by + return Skill( + slug=slug, + name=slug, + description="demo", + dir_path=f"skills/{slug}", + tool_dependencies=tool_dependencies, + mcp_dependencies=mcp_dependencies, + skill_dependencies=skill_dependencies, + ) + + monkeypatch.setattr("server.routers.skill_router.update_skill_dependencies", fake_update_skill_dependencies) + + app = _build_app(allow_superadmin=True) + client = TestClient(app) + resp = client.put( + "/api/system/skills/demo/dependencies", + json={ + "tool_dependencies": ["calculator"], + "mcp_dependencies": ["mcp-a"], + "skill_dependencies": ["other-skill"], + }, + ) + assert resp.status_code == 200, resp.text + assert captured["slug"] == "demo" + assert captured["tool_dependencies"] == ["calculator"] + assert captured["mcp_dependencies"] == ["mcp-a"] + assert captured["skill_dependencies"] == ["other-skill"] + assert captured["updated_by"] == "root" diff --git a/test/test_skill_service.py b/test/test_skill_service.py new file mode 100644 index 00000000..220bdf3d --- /dev/null +++ b/test/test_skill_service.py @@ -0,0 +1,283 @@ +from __future__ import annotations + +import io +import zipfile +from pathlib import Path + +import pytest + +from src.services import skill_service as svc +from src.storage.postgres.models_business import Skill + + +def _build_zip(files: dict[str, str]) -> bytes: + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", compression=zipfile.ZIP_DEFLATED) as zf: + for path, content in files.items(): + zf.writestr(path, content) + return buf.getvalue() + + +def test_parse_skill_markdown_ok(): + content = ( + "---\n" + "name: demo-skill\n" + "description: demo description\n" + "---\n" + "# Demo\n" + ) + name, desc, meta = svc._parse_skill_markdown(content) + assert name == "demo-skill" + assert desc == "demo description" + assert meta["name"] == "demo-skill" + + +def test_parse_skill_markdown_requires_frontmatter(): + with pytest.raises(ValueError, match="frontmatter"): + svc._parse_skill_markdown("# missing") + + +def test_validate_skill_slug(): + assert svc.validate_skill_slug("demo-skill") == "demo-skill" + with pytest.raises(ValueError, match="无效 skill slug"): + svc.validate_skill_slug("../bad") + + +@pytest.mark.asyncio +async def test_get_skill_dependency_options(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(svc, "_get_buildin_tool_names", lambda: ["calculator", "search"]) + monkeypatch.setattr(svc, "get_mcp_server_names", lambda: ["mcp-a", "mcp-b"]) + + class FakeRepo: + def __init__(self, _db): + pass + + async def list_all(self): + return [ + Skill(slug="alpha", name="alpha", description="a", dir_path="skills/alpha"), + Skill(slug="beta", name="beta", description="b", dir_path="skills/beta"), + ] + + monkeypatch.setattr(svc, "SkillRepository", FakeRepo) + + result = await svc.get_skill_dependency_options(None) + assert result["tools"] == ["calculator", "search"] + assert result["mcps"] == ["mcp-a", "mcp-b"] + assert result["skills"] == ["alpha", "beta"] + + +def test_resolve_relative_path_blocks_traversal(tmp_path: Path): + skill_dir = tmp_path / "skill" + skill_dir.mkdir(parents=True, exist_ok=True) + + with pytest.raises(ValueError, match="上级路径"): + svc._resolve_relative_path(skill_dir, "../outside.txt") + + +@pytest.mark.asyncio +async def test_import_skill_zip_conflict_rewrite_name(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(svc.sys_config, "save_dir", str(tmp_path)) + + class FakeRepo: + existing_slugs = {"demo"} + created_item: Skill | None = None + + def __init__(self, _db): + pass + + async def exists_slug(self, slug: str) -> bool: + return slug in self.__class__.existing_slugs + + async def create( + self, + *, + slug: str, + name: str, + description: str, + tool_dependencies: list[str] | None, + mcp_dependencies: list[str] | None, + skill_dependencies: list[str] | None, + dir_path: str, + created_by: str | None, + ) -> Skill: + item = Skill( + slug=slug, + name=name, + description=description, + tool_dependencies=tool_dependencies or [], + mcp_dependencies=mcp_dependencies or [], + skill_dependencies=skill_dependencies or [], + dir_path=dir_path, + created_by=created_by, + updated_by=created_by, + ) + self.__class__.existing_slugs.add(slug) + self.__class__.created_item = item + return item + + monkeypatch.setattr(svc, "SkillRepository", FakeRepo) + + zip_bytes = _build_zip( + { + "demo/SKILL.md": ( + "---\n" + "name: demo\n" + "description: this is demo\n" + "---\n" + "# Demo\n" + ), + "demo/prompts/system.md": "You are demo skill", + } + ) + + item = await svc.import_skill_zip( + None, + filename="demo.zip", + file_bytes=zip_bytes, + created_by="root", + ) + + assert item.slug == "demo-v2" + assert item.name == "demo-v2" + skill_md = (tmp_path / "skills" / "demo-v2" / "SKILL.md").read_text(encoding="utf-8") + assert "name: demo-v2" in skill_md + + +@pytest.mark.asyncio +async def test_update_skill_md_syncs_metadata(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(svc.sys_config, "save_dir", str(tmp_path)) + skill_dir = tmp_path / "skills" / "demo" + skill_dir.mkdir(parents=True, exist_ok=True) + (skill_dir / "SKILL.md").write_text( + "---\nname: demo\ndescription: old\n---\n# old\n", + encoding="utf-8", + ) + + item = Skill( + slug="demo", + name="demo", + description="old", + dir_path="skills/demo", + created_by="root", + updated_by="root", + ) + + async def fake_get_skill_or_raise(_db, _slug: str): + return item + + updates: dict[str, str | None] = {} + + class FakeRepo: + def __init__(self, _db): + pass + + async def update_metadata( + self, + _item: Skill, + *, + name: str, + description: str, + updated_by: str | None, + ) -> Skill: + updates["name"] = name + updates["description"] = description + updates["updated_by"] = updated_by + return item + + monkeypatch.setattr(svc, "get_skill_or_raise", fake_get_skill_or_raise) + monkeypatch.setattr(svc, "SkillRepository", FakeRepo) + + new_content = ( + "---\n" + "name: demo\n" + "description: updated desc\n" + "---\n" + "# updated\n" + ) + await svc.update_skill_file( + None, + slug="demo", + relative_path="SKILL.md", + content=new_content, + updated_by="admin", + ) + + assert updates["name"] == "demo" + assert updates["description"] == "updated desc" + assert updates["updated_by"] == "admin" + saved_content = (skill_dir / "SKILL.md").read_text(encoding="utf-8") + assert "description: updated desc" in saved_content + + +@pytest.mark.asyncio +async def test_update_skill_dependencies(monkeypatch: pytest.MonkeyPatch): + item = Skill( + slug="alpha", + name="alpha", + description="alpha", + dir_path="skills/alpha", + tool_dependencies=[], + mcp_dependencies=[], + skill_dependencies=[], + ) + monkeypatch.setattr(svc, "_get_buildin_tool_names", lambda: ["calculator"]) + monkeypatch.setattr(svc, "get_mcp_server_names", lambda: ["mcp-a"]) + + async def fake_get_skill_or_raise(_db, slug: str): + assert slug == "alpha" + return item + + captured: dict[str, list[str] | str | None] = {} + + class FakeRepo: + def __init__(self, _db): + pass + + async def list_all(self): + return [ + item, + Skill( + slug="beta", + name="beta", + description="beta", + dir_path="skills/beta", + tool_dependencies=[], + mcp_dependencies=[], + skill_dependencies=[], + ), + ] + + async def update_dependencies( + self, + _item: Skill, + *, + tool_dependencies: list[str], + mcp_dependencies: list[str], + skill_dependencies: list[str], + updated_by: str | None, + ): + captured["tool_dependencies"] = tool_dependencies + captured["mcp_dependencies"] = mcp_dependencies + captured["skill_dependencies"] = skill_dependencies + captured["updated_by"] = updated_by + _item.tool_dependencies = tool_dependencies + _item.mcp_dependencies = mcp_dependencies + _item.skill_dependencies = skill_dependencies + return _item + + monkeypatch.setattr(svc, "get_skill_or_raise", fake_get_skill_or_raise) + monkeypatch.setattr(svc, "SkillRepository", FakeRepo) + + updated = await svc.update_skill_dependencies( + None, + slug="alpha", + tool_dependencies=["calculator", "calculator"], + mcp_dependencies=["mcp-a", "mcp-a"], + skill_dependencies=["beta", "beta"], + updated_by="root", + ) + assert captured["tool_dependencies"] == ["calculator"] + assert captured["mcp_dependencies"] == ["mcp-a"] + assert captured["skill_dependencies"] == ["beta"] + assert captured["updated_by"] == "root" + assert updated.skill_dependencies == ["beta"] diff --git a/test/test_skills_backend.py b/test/test_skills_backend.py new file mode 100644 index 00000000..669fec4a --- /dev/null +++ b/test/test_skills_backend.py @@ -0,0 +1,89 @@ +from __future__ import annotations + +from pathlib import Path +from types import SimpleNamespace + +from src.agents.common.backends import skills_backend + + +def _prepare_skills_dir(root: Path) -> None: + (root / "alpha").mkdir(parents=True, exist_ok=True) + (root / "alpha" / "SKILL.md").write_text( + "---\nname: alpha\ndescription: alpha\n---\n# alpha\n", + encoding="utf-8", + ) + (root / "beta").mkdir(parents=True, exist_ok=True) + (root / "beta" / "SKILL.md").write_text( + "---\nname: beta\ndescription: beta\n---\n# beta\n", + encoding="utf-8", + ) + + +def test_selected_skills_backend_readonly_and_visible_only_selected(tmp_path, monkeypatch): + _prepare_skills_dir(tmp_path) + monkeypatch.setattr(skills_backend, "get_skills_root_dir", lambda: tmp_path) + + backend = skills_backend.SelectedSkillsReadonlyBackend(selected_slugs=["alpha"]) + + root_entries = backend.ls_info("/") + paths = sorted(entry.get("path") for entry in root_entries) + assert paths == ["/alpha/"] + + ok_read = backend.read("/alpha/SKILL.md") + assert "alpha" in ok_read + + denied_read = backend.read("/beta/SKILL.md") + assert "Access denied" in denied_read + + write_result = backend.write("/alpha/new.md", "x") + assert write_result.error and "read-only" in write_result.error + + edit_result = backend.edit("/alpha/SKILL.md", "alpha", "changed") + assert edit_result.error and "read-only" in edit_result.error + + upload_result = backend.upload_files([("/alpha/a.txt", b"a")]) + assert len(upload_result) == 1 + assert upload_result[0].error == "permission_denied" + + +def test_composite_backend_mounts_skills_under_prefix(tmp_path, monkeypatch): + _prepare_skills_dir(tmp_path) + monkeypatch.setattr(skills_backend, "get_skills_root_dir", lambda: tmp_path) + + runtime = SimpleNamespace( + context=SimpleNamespace(skills=["alpha"]), + state={ + "skill_session_snapshot": { + "selected_skills": ["alpha"], + "visible_skills": ["alpha", "beta"], + "prompt_metadata": {}, + "dependency_map": {}, + } + }, + ) + composite = skills_backend.create_agent_composite_backend(runtime) + + root = composite.ls_info("/") + all_paths = [entry.get("path") for entry in root] + assert "/skills/" in all_paths + + skills_root = composite.ls_info("/skills/") + skill_paths = sorted(entry.get("path") for entry in skills_root) + assert skill_paths == ["/skills/alpha/", "/skills/beta/"] + + denied = composite.write("/skills/alpha/new.md", "x") + assert denied.error and "read-only" in denied.error + + +def test_composite_backend_fallbacks_to_context_skills_when_snapshot_missing(tmp_path, monkeypatch): + _prepare_skills_dir(tmp_path) + monkeypatch.setattr(skills_backend, "get_skills_root_dir", lambda: tmp_path) + + runtime = SimpleNamespace( + context=SimpleNamespace(skills=["alpha"]), + state={}, + ) + composite = skills_backend.create_agent_composite_backend(runtime) + skills_root = composite.ls_info("/skills/") + skill_paths = sorted(entry.get("path") for entry in skills_root) + assert skill_paths == ["/skills/alpha/"] diff --git a/web/src/apis/index.js b/web/src/apis/index.js index 6be3f1de..6d72bbaf 100644 --- a/web/src/apis/index.js +++ b/web/src/apis/index.js @@ -12,6 +12,7 @@ export * from './tasker' // 任务管理API export * from './mindmap_api' // 思维导图API export * from './department_api' // 部门管理API export * from './mcp_api' // MCP API +export * from './skill_api' // Skills API // 导出基础工具函数 export { diff --git a/web/src/apis/knowledge_api.js b/web/src/apis/knowledge_api.js index 8c23230b..7e78794f 100644 --- a/web/src/apis/knowledge_api.js +++ b/web/src/apis/knowledge_api.js @@ -453,6 +453,15 @@ export const evaluationApi = { return apiAdminDelete(`/api/evaluation/benchmarks/${benchmarkId}`) }, + /** + * 下载评估基准 + * @param {string} benchmarkId - 基准ID + * @returns {Promise} - Response对象 + */ + downloadBenchmark: async (benchmarkId) => { + return apiAdminGet(`/api/evaluation/benchmarks/${benchmarkId}/download`, {}, 'blob') + }, + /** * 自动生成评估基准 * @param {string} dbId - 知识库ID diff --git a/web/src/apis/skill_api.js b/web/src/apis/skill_api.js new file mode 100644 index 00000000..f0bbfbc9 --- /dev/null +++ b/web/src/apis/skill_api.js @@ -0,0 +1,71 @@ +import { + apiAdminGet, + apiSuperAdminDelete, + apiSuperAdminGet, + apiSuperAdminPost, + apiSuperAdminPut +} from './base' + +const BASE_URL = '/api/system/skills' + +export const listSkills = async () => { + return apiAdminGet(BASE_URL) +} + +export const importSkillZip = async (file) => { + const formData = new FormData() + formData.append('file', file) + return apiSuperAdminPost(`${BASE_URL}/import`, formData) +} + +export const getSkillDependencyOptions = async () => { + return apiSuperAdminGet(`${BASE_URL}/dependency-options`) +} + +export const getSkillTree = async (slug) => { + return apiSuperAdminGet(`${BASE_URL}/${encodeURIComponent(slug)}/tree`) +} + +export const getSkillFile = async (slug, path) => { + return apiSuperAdminGet(`${BASE_URL}/${encodeURIComponent(slug)}/file?path=${encodeURIComponent(path)}`) +} + +export const createSkillFile = async (slug, payload) => { + return apiSuperAdminPost(`${BASE_URL}/${encodeURIComponent(slug)}/file`, payload) +} + +export const updateSkillFile = async (slug, payload) => { + return apiSuperAdminPut(`${BASE_URL}/${encodeURIComponent(slug)}/file`, payload) +} + +export const updateSkillDependencies = async (slug, payload) => { + return apiSuperAdminPut(`${BASE_URL}/${encodeURIComponent(slug)}/dependencies`, payload) +} + +export const deleteSkillFile = async (slug, path) => { + return apiSuperAdminDelete(`${BASE_URL}/${encodeURIComponent(slug)}/file?path=${encodeURIComponent(path)}`) +} + +export const exportSkill = async (slug) => { + return apiSuperAdminGet(`${BASE_URL}/${encodeURIComponent(slug)}/export`, {}, 'blob') +} + +export const deleteSkill = async (slug) => { + return apiSuperAdminDelete(`${BASE_URL}/${encodeURIComponent(slug)}`) +} + +export const skillApi = { + listSkills, + importSkillZip, + getSkillDependencyOptions, + getSkillTree, + getSkillFile, + createSkillFile, + updateSkillFile, + updateSkillDependencies, + deleteSkillFile, + exportSkill, + deleteSkill +} + +export default skillApi diff --git a/web/src/components/AgentConfigSidebar.vue b/web/src/components/AgentConfigSidebar.vue index 4f285806..a4affa9b 100644 --- a/web/src/components/AgentConfigSidebar.vue +++ b/web/src/components/AgentConfigSidebar.vue @@ -372,6 +372,7 @@ import ModelSelectorComponent from '@/components/ModelSelectorComponent.vue' import { useAgentStore } from '@/stores/agent' import { useUserStore } from '@/stores/user' import { useDatabaseStore } from '@/stores/database' +import { skillApi } from '@/apis/skill_api' import { storeToRefs } from 'pinia' // Props @@ -396,9 +397,17 @@ const databaseStore = useDatabaseStore() watch( () => props.isOpen, - (val) => { + async (val) => { if (val) { databaseStore.loadDatabases().catch(() => {}) + loadLiveSkillOptions().catch(() => {}) + if (selectedAgentId.value) { + try { + await agentStore.fetchAgentDetail(selectedAgentId.value, true) + } catch (error) { + console.error('刷新智能体配置项失败:', error) + } + } } } ) @@ -423,6 +432,7 @@ const tempSelectedValues = ref([]) const selectionSearchText = ref('') const systemPromptEditMode = ref(false) const activeTab = ref('basic') +const liveSkillOptions = ref([]) const isEmptyConfig = computed(() => { return !selectedAgentId.value || Object.keys(configurableItems.value).length === 0 @@ -443,7 +453,8 @@ const hasOtherConfigs = computed(() => { const isTools = value.template_metadata?.kind === 'mcps' || value.template_metadata?.kind === 'knowledges' || - value.template_metadata?.kind === 'tools' + value.template_metadata?.kind === 'tools' || + value.template_metadata?.kind === 'skills' return !isBasic && !isTools }) @@ -462,6 +473,24 @@ const segmentedOptions = computed(() => { return options }) +const loadLiveSkillOptions = async () => { + if (!userStore.isAdmin) { + liveSkillOptions.value = [] + return + } + try { + const result = await skillApi.listSkills() + const rows = result?.data || [] + liveSkillOptions.value = rows.map((item) => ({ + id: item.slug, + name: item.slug, + description: item.description || '' + })) + } catch (error) { + console.warn('加载实时 Skills 列表失败:', error) + } +} + // 通用选项获取与处理 const getConfigOptions = (value) => { if (value?.template_metadata?.kind === 'tools') { @@ -470,25 +499,28 @@ const getConfigOptions = (value) => { if (value?.template_metadata?.kind === 'knowledges') { return databaseStore.databases || [] } + if (value?.template_metadata?.kind === 'skills') { + return liveSkillOptions.value.length > 0 ? liveSkillOptions.value : value?.options || [] + } return value?.options || [] } const isListConfig = (key, value) => { const isTools = value?.template_metadata?.kind === 'tools' const isList = value?.type === 'list' - return isTools || isList + return isTools || isList || key === 'skills' } const getOptionValue = (option) => { if (typeof option === 'object' && option !== null) { - return option.id || option.value || option.name + return option.id || option.value || option.name || option.db_id || option.slug } return option } const getOptionLabel = (option) => { if (typeof option === 'object' && option !== null) { - return option.name || option.label || option.id + return option.name || option.label || option.id || option.db_id || option.slug } return option } @@ -523,7 +555,9 @@ const shouldShowConfig = (key, value) => { const isTools = value.template_metadata?.kind === 'mcps' || value.template_metadata?.kind === 'knowledges' || - value.template_metadata?.kind === 'tools' + value.template_metadata?.kind === 'tools' || + value.template_metadata?.kind === 'skills' || + key === 'skills' if (activeTab.value === 'basic') { // 基础：System Prompt, LLM Model @@ -626,6 +660,9 @@ const openSelectionModal = async (key) => { console.error('加载知识库列表失败:', error) } } + if (configurableItems.value[key]?.template_metadata?.kind === 'skills') { + await loadLiveSkillOptions() + } const currentValues = agentConfig.value[key] || [] tempSelectedValues.value = [...currentValues] selectionModalOpen.value = true @@ -678,29 +715,14 @@ const validateAndFilterConfig = () => { const configItem = configItems[key] const currentValue = validatedConfig[key] - // 检查工具配置 - if (configItem.template_metadata?.kind === 'tools' && Array.isArray(currentValue)) { - const availableToolIds = availableTools.value - ? Object.values(availableTools.value).map((tool) => tool.id) - : [] - validatedConfig[key] = currentValue.filter((toolId) => availableToolIds.includes(toolId)) + if (Array.isArray(currentValue) && (configItem.template_metadata?.kind === 'tools' || configItem.type === 'list')) { + const options = getConfigOptions(configItem) + const validValues = new Set(options.map((opt) => String(getOptionValue(opt)))) + if (validValues.size === 0) return + validatedConfig[key] = currentValue.filter((value) => validValues.has(String(value))) if (validatedConfig[key].length !== currentValue.length) { - console.warn(`工具配置 ${key} 中包含无效的工具ID，已自动过滤`) - } - } - - // 检查多选配置项 (type === 'list' 且有 options) - else if ( - configItem.type === 'list' && - configItem.options.length > 0 && - Array.isArray(currentValue) - ) { - const validOptions = configItem.options - validatedConfig[key] = currentValue.filter((value) => validOptions.includes(value)) - - if (validatedConfig[key].length !== currentValue.length) { - console.warn(`配置项 ${key} 中包含无效的选项，已自动过滤`) + console.warn(`配置项 ${key} 中包含无效选项，已自动过滤`) } } }) diff --git a/web/src/components/ChunkParamsConfig.vue b/web/src/components/ChunkParamsConfig.vue index 9617e9c8..30e72be5 100644 --- a/web/src/components/ChunkParamsConfig.vue +++ b/web/src/components/ChunkParamsConfig.vue @@ -4,7 +4,27 @@

调整分块参数可以控制文本的切分方式，影响检索质量和文档加载效率。

-
+ + + +

+ 预设策略对齐 RAGFlow：General、QA、Book、Laws。 + 留空时沿用知识库默认策略。 +

+ + +
diff --git a/web/src/components/EvaluationBenchmarks.vue b/web/src/components/EvaluationBenchmarks.vue index 667d9c80..5edf108b 100644 --- a/web/src/components/EvaluationBenchmarks.vue +++ b/web/src/components/EvaluationBenchmarks.vue @@ -48,6 +48,14 @@ + + + @@ -196,6 +204,7 @@ import { UploadOutlined, RobotOutlined, EyeOutlined, + DownloadOutlined, DeleteOutlined, CheckCircleOutlined, CloseCircleOutlined, @@ -225,6 +234,7 @@ const generateModalVisible = ref(false) const previewModalVisible = ref(false) const previewData = ref(null) const previewQuestions = ref([]) +const downloadingBenchmarkMap = reactive({}) const previewPagination = ref({ current: 1, pageSize: 10, @@ -407,6 +417,60 @@ const previewBenchmark = async (benchmark) => { } } +const parseDownloadFilename = (contentDisposition) => { + if (!contentDisposition) return '' + + const utf8Match = contentDisposition.match(/filename\*=UTF-8''([^;]+)/i) + if (utf8Match && utf8Match[1]) { + try { + return decodeURIComponent(utf8Match[1]) + } catch (error) { + console.warn('解析 UTF-8 文件名失败:', error) + } + } + + const asciiMatch = contentDisposition.match(/filename=\"?([^\";]+)\"?/i) + if (asciiMatch && asciiMatch[1]) { + return asciiMatch[1] + } + + return '' +} + +// 下载基准 +const downloadBenchmark = async (benchmark) => { + const benchmarkId = benchmark?.benchmark_id + if (!benchmarkId) return + if (downloadingBenchmarkMap[benchmarkId]) return + + downloadingBenchmarkMap[benchmarkId] = true + try { + const response = await evaluationApi.downloadBenchmark(benchmarkId) + const blob = await response.blob() + const contentDisposition = + response.headers.get('Content-Disposition') || response.headers.get('content-disposition') + const headerFilename = parseDownloadFilename(contentDisposition) + const fallbackFilename = `${benchmark.name || benchmarkId}.jsonl` + const filename = headerFilename || fallbackFilename + + const url = window.URL.createObjectURL(blob) + const link = document.createElement('a') + link.href = url + link.download = filename + document.body.appendChild(link) + link.click() + document.body.removeChild(link) + window.URL.revokeObjectURL(url) + + message.success('下载成功') + } catch (error) { + console.error('下载基准失败:', error) + message.error(`下载失败: ${error.message || '未知错误'}`) + } finally { + delete downloadingBenchmarkMap[benchmarkId] + } +} + // 删除基准 const deleteBenchmark = (benchmark) => { Modal.confirm({ diff --git a/web/src/components/FileTable.vue b/web/src/components/FileTable.vue index c8645954..f2a84494 100644 --- a/web/src/components/FileTable.vue +++ b/web/src/components/FileTable.vue @@ -189,7 +189,14 @@ 确定
- +
@@ -704,8 +711,25 @@ const indexConfigModalTitle = ref('入库参数配置') const indexParams = ref({ chunk_size: 1000, chunk_overlap: 200, - qa_separator: '' + qa_separator: '', + chunk_preset_id: '' }) +const buildIndexParamsPayload = () => { + const payload = {} + if (indexParams.value.chunk_preset_id) { + payload.chunk_preset_id = indexParams.value.chunk_preset_id + } + + if (isLightRAG.value) { + payload.qa_separator = indexParams.value.qa_separator || '' + return payload + } + + return { + ...indexParams.value, + ...payload + } +} const currentIndexFileIds = ref([]) const isBatchIndexOperation = ref(false) @@ -1088,12 +1112,6 @@ const handleBatchIndex = async () => { return } - if (isLightRAG.value) { - await store.indexFiles(validKeys) - selectedRowKeys.value = [] - return - } - currentIndexFileIds.value = [...validKeys] isBatchIndexOperation.value = true indexConfigModalTitle.value = '批量入库参数配置' @@ -1172,11 +1190,6 @@ const handleParseFile = async (record) => { const handleIndexFile = async (record) => { closePopover(record.file_id) - if (isLightRAG.value) { - await store.indexFiles([record.file_id]) - return - } - // 打开参数配置弹窗 currentIndexFileIds.value = [record.file_id] isBatchIndexOperation.value = false @@ -1189,7 +1202,8 @@ const handleIndexFile = async (record) => { Object.assign(indexParams.value, { chunk_size: 1000, chunk_overlap: 200, - qa_separator: '' + qa_separator: '', + chunk_preset_id: '' }) } @@ -1214,7 +1228,7 @@ const handleReindexFile = async (record) => { const handleIndexConfigConfirm = async () => { try { // 调用 indexFiles 接口 (支持 params) - const result = await store.indexFiles(currentIndexFileIds.value, indexParams.value) + const result = await store.indexFiles(currentIndexFileIds.value, buildIndexParamsPayload()) if (result) { currentIndexFileIds.value = [] // 清空选择 @@ -1228,7 +1242,8 @@ const handleIndexConfigConfirm = async () => { Object.assign(indexParams.value, { chunk_size: 1000, chunk_overlap: 200, - qa_separator: '' + qa_separator: '', + chunk_preset_id: '' }) } else { // message.error(`入库失败: ${result.message}`); // store already shows message @@ -1249,7 +1264,8 @@ const handleIndexConfigCancel = () => { Object.assign(indexParams.value, { chunk_size: 1000, chunk_overlap: 200, - qa_separator: '' + qa_separator: '', + chunk_preset_id: '' }) } diff --git a/web/src/components/FileUploadModal.vue b/web/src/components/FileUploadModal.vue index ddd85e19..bf0e6f9c 100644 --- a/web/src/components/FileUploadModal.vue +++ b/web/src/components/FileUploadModal.vue @@ -108,15 +108,17 @@
入库参数配置
- - + +

+ LightRAG 按分隔符预切分，超长片段仍会按 token 大小继续切分。 +

@@ -564,9 +566,26 @@ const autoIndex = ref(false) const indexParams = ref({ chunk_size: 1000, chunk_overlap: 200, - qa_separator: '' + qa_separator: '', + chunk_preset_id: '' }) +const buildAutoIndexParams = () => { + const payload = {} + if (indexParams.value.chunk_preset_id) { + payload.chunk_preset_id = indexParams.value.chunk_preset_id + } + + if (isGraphBased.value) { + payload.qa_separator = indexParams.value.qa_separator || '' + return payload + } + return { + ...indexParams.value, + ...payload + } +} + // 计算属性：是否支持QA分割 const isQaSplitSupported = computed(() => { const type = kbType.value?.toLowerCase() @@ -990,7 +1009,7 @@ const chunkData = async () => { const params = { ...chunkParams.value } if (autoIndex.value) { params.auto_index = true - Object.assign(params, indexParams.value) + Object.assign(params, buildAutoIndexParams()) } // 构造 _preprocessed_map 和 items (minio urls) @@ -1068,7 +1087,7 @@ const chunkData = async () => { const params = { ...chunkParams.value, content_hashes } if (autoIndex.value) { params.auto_index = true - Object.assign(params, indexParams.value) + Object.assign(params, buildAutoIndexParams()) } await store.addFiles({ @@ -1604,17 +1623,6 @@ const chunkData = async () => { border-radius: 6px; } -.lightrag-tip { - display: flex; - align-items: center; - margin-top: 8px; - padding: 8px 12px; - background: var(--main-50); - border-radius: 6px; - font-size: 13px; - color: var(--gray-600); -} - .setting-label .ant-checkbox { margin-right: 8px; } diff --git a/web/src/components/KnowledgeBaseCard.vue b/web/src/components/KnowledgeBaseCard.vue index 67f2a733..833e516a 100644 --- a/web/src/components/KnowledgeBaseCard.vue +++ b/web/src/components/KnowledgeBaseCard.vue @@ -40,6 +40,9 @@ {{ getKbTypeLabel(database.kb_type || 'lightrag') }} {{ database.embed_info?.name || 'N/A' }} + {{ + chunkPresetLabelMap[database.additional_params?.chunk_preset_id || 'general'] || 'General' + }}
@@ -81,6 +84,18 @@ > + + + + + ({ label, value })) +const chunkPresetLabelMap = CHUNK_PRESET_LABEL_MAP +const editPresetDescription = computed(() => getChunkPresetDescription(editForm.chunk_preset_id)) + const rules = { name: [{ required: true, message: '请输入知识库名称' }] } @@ -238,6 +263,7 @@ const showEditModal = () => { editForm.description = database.value.description || '' editForm.auto_generate_questions = database.value.additional_params?.auto_generate_questions || false + editForm.chunk_preset_id = database.value.additional_params?.chunk_preset_id || 'general' // 如果是 LightRAG 类型，加载当前的 LLM 配置 if (database.value.kb_type === 'lightrag') { @@ -283,7 +309,8 @@ const handleEditSubmit = () => { name: editForm.name, description: editForm.description, additional_params: { - auto_generate_questions: editForm.auto_generate_questions + auto_generate_questions: editForm.auto_generate_questions, + chunk_preset_id: editForm.chunk_preset_id || 'general' }, share_config: { is_shared: finalIsShared, @@ -428,4 +455,16 @@ const deleteDatabase = () => { align-items: center; flex-wrap: wrap; } + +.chunk-preset-label { + display: inline-flex; + align-items: center; + gap: 6px; +} + +.chunk-preset-help-icon { + color: var(--gray-500); + cursor: help; + font-size: 14px; +} diff --git a/web/src/components/McpEnvEditor.vue b/web/src/components/McpEnvEditor.vue new file mode 100644 index 00000000..77600c49 --- /dev/null +++ b/web/src/components/McpEnvEditor.vue @@ -0,0 +1,145 @@ + + + + + diff --git a/web/src/components/McpServerDetailModal.vue b/web/src/components/McpServerDetailModal.vue index 631ca69d..fb9fead6 100644 --- a/web/src/components/McpServerDetailModal.vue +++ b/web/src/components/McpServerDetailModal.vue @@ -78,6 +78,10 @@ +
+ + 
{{ JSON.stringify(server.env, null, 2) }}
+
diff --git a/web/src/components/McpServersComponent.vue b/web/src/components/McpServersComponent.vue index 0066a723..ff237394 100644 --- a/web/src/components/McpServersComponent.vue +++ b/web/src/components/McpServersComponent.vue @@ -214,7 +214,7 @@ -