From 8e17f44ec2efbf5a7d198d882e2ad85d2bc6ec69 Mon Sep 17 00:00:00 2001 From: Wenjie Zhang Date: Sat, 16 May 2026 13:36:35 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=B0=86kb=20=E4=BB=8E=E6=B2=99?= =?UTF-8?q?=E7=9B=92=E6=98=A0=E5=B0=84=E4=B8=AD=E7=A7=BB=E9=99=A4=EF=BC=8C?= =?UTF-8?q?=E5=8F=96=E8=80=8C=E4=BB=A3=E4=B9=8B=E7=9A=84=E6=98=AF=E4=B8=80?= =?UTF-8?q?=E4=B8=AA=E6=96=B0=E7=9A=84=20Open=20=E5=B7=A5=E5=85=B7?= =?UTF-8?q?=EF=BC=8C=E5=8F=AF=E4=BB=A5=E6=8F=90=E4=BE=9B=E8=87=AA=E7=94=B1?= =?UTF-8?q?=E4=B8=94=E5=8F=AF=E6=8E=A7=E7=9A=84=E6=96=87=E4=BB=B6=E8=AE=BF?= =?UTF-8?q?=E9=97=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ARCHITECTURE.md | 8 +- .../package/yuxi/agents/backends/__init__.py | 12 +- .../package/yuxi/agents/backends/composite.py | 11 - .../agents/backends/knowledge_base_backend.py | 646 ------------------ .../yuxi/agents/buildin/chatbot/prompt.py | 6 +- .../package/yuxi/agents/toolkits/kbs/tools.py | 50 +- .../yuxi/services/knowledge_fs_service.py | 169 ----- .../services/viewer_filesystem_service.py | 36 +- backend/package/yuxi/utils/paths.py | 2 - .../api/test_viewer_filesystem_router.py | 26 +- .../backends/test_knowledge_base_backend.py | 307 +-------- .../unit/backends/test_sandbox_backends.py | 2 +- backend/test/unit/toolkits/test_kbs_tools.py | 86 +-- docs/agents/sandbox-architecture.md | 16 +- docs/develop-guides/changelog.md | 1 + docs/develop-guides/roadmap.md | 4 +- web/src/components/AgentPanel.vue | 26 +- 17 files changed, 98 insertions(+), 1310 deletions(-) delete mode 100644 backend/package/yuxi/services/knowledge_fs_service.py diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 8c181243..f7e612ba 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -26,7 +26,7 @@ Yuxi 是一个面向 RAG、知识图谱和多智能体工作流的知识库平 `backend/package/yuxi` 是后端主体： -- `agents` 定义 LangGraph 智能体体系。`BaseAgent` 是智能体基类，`BaseContext` 是运行配置上下文；`buildin` 放内置智能体；`middlewares` 负责把知识库、Skills、MCP、附件、运行配置等能力挂到运行时；`toolkits` 放工具注册与内置工具；`backends` 对接沙盒、知识库和 Skills 等外部执行/资源后端。 +- `agents` 定义 LangGraph 智能体体系。`BaseAgent` 是智能体基类，`BaseContext` 是运行配置上下文；`buildin` 放内置智能体；`middlewares` 负责把知识库、Skills、MCP、附件、运行配置等能力挂到运行时；`toolkits` 放工具注册与内置工具；`backends` 对接沙盒和 Skills 等外部执行/资源后端。 - `services` 是用例层，负责串联 repositories、agents、knowledge、storage 和外部系统。聊天、运行队列、文件视图、Skills、MCP、SubAgents、评估等跨模块流程都从这里找入口。 - `repositories` 是数据库访问边界，封装业务对象和知识库元数据的 SQLAlchemy 查询。不要让路由绕过 repository 直接操作模型，除非已有局部模式要求这样做。 - `storage` 放持久化基础设施。`storage/postgres` 管理业务表、知识库表和 LangGraph checkpoint 所需连接池；`storage/minio` 管理对象存储。 @@ -56,8 +56,8 @@ Yuxi 是一个面向 RAG、知识图谱和多智能体工作流的知识库平 2. `web/src/apis` 调用 `/api/chat` 相关接口。 3. `server/routers/chat_router.py` 进入后端，委托 `yuxi.services.chat_service` 或 `agent_run_service`。 4. 服务层读取 conversation、agent config、tools、skills、knowledge 等配置，必要时创建后台 run。 -5. `worker-dev` 执行 LangGraph 智能体；中间件按上下文挂载知识库、工具、Skills、MCP、附件与沙盒能力。 -6. 运行事件写入 Redis，最终状态和业务记录写入 Postgres；文件和产物落到 `saves`、MinIO 或沙盒映射目录。 +5. `worker-dev` 执行 LangGraph 智能体；中间件按上下文挂载知识库工具、Skills、MCP、附件与沙盒能力。 +6. 运行事件写入 Redis，最终状态和业务记录写入 Postgres；文件和产物落到 `saves`、MinIO 或沙盒用户数据目录。 7. 前端通过 SSE/轮询消费运行事件，渲染消息、工具调用、引用来源、产物卡片和文件预览。 ## 架构不变量 @@ -65,7 +65,7 @@ Yuxi 是一个面向 RAG、知识图谱和多智能体工作流的知识库平 - Docker Compose 是开发环境的事实来源。开发时优先检查容器、日志和热重载，不要默认要求本地裸跑服务。 - HTTP 路由层应保持薄；领域流程放在 `yuxi.services`，持久化查询放在 `yuxi.repositories`。 - 前端 API 调用应集中在 `web/src/apis`，组件不要散落拼接后端 URL。 -- 智能体能力通过 context、middleware、toolkits、backends 组合，不要把知识库、MCP、Skills 或沙盒逻辑硬编码进单个页面或路由。 +- 智能体能力通过 context、middleware、toolkits、backends 组合；知识库通过工具访问，不要把知识库、MCP、Skills 或沙盒逻辑硬编码进单个页面或路由。 - LITE 模式必须允许跳过知识库、图谱、评估等重依赖能力；新增相关接口或初始化逻辑时要尊重这个边界。 - 沙盒虚拟路径以 `SANDBOX_VIRTUAL_PATH_PREFIX` 为边界，用户可见路径与宿主机真实路径不要混用。 - 面向用户或外部系统的输入在边界校验；内部服务之间优先信任已有类型、仓储和框架约束，避免为了假设场景堆叠防御代码。 diff --git a/backend/package/yuxi/agents/backends/__init__.py b/backend/package/yuxi/agents/backends/__init__.py index d7849076..b20552a1 100644 --- a/backend/package/yuxi/agents/backends/__init__.py +++ b/backend/package/yuxi/agents/backends/__init__.py @@ -1,13 +1,7 @@ from deepagents.backends import CompositeBackend, StateBackend from .composite import create_agent_composite_backend -from .knowledge_base_backend import ( - KBS_PATH, - KnowledgeBaseReadonlyBackend, - build_knowledge_base_filepath_map, - resolve_file_relative_virtual_path, - resolve_visible_knowledge_bases_for_context, -) +from .knowledge_base_backend import resolve_visible_knowledge_bases_for_context from .sandbox import ( IDLE_CHECK_INTERVAL, LARGE_TOOL_RESULTS_DIR, @@ -37,10 +31,6 @@ from .skills_backend import SelectedSkillsReadonlyBackend __all__ = [ "CompositeBackend", - "KBS_PATH", - "KnowledgeBaseReadonlyBackend", - "build_knowledge_base_filepath_map", - "resolve_file_relative_virtual_path", "StateBackend", "SelectedSkillsReadonlyBackend", "create_agent_composite_backend", diff --git a/backend/package/yuxi/agents/backends/composite.py b/backend/package/yuxi/agents/backends/composite.py index 0b7149d6..939a024a 100644 --- a/backend/package/yuxi/agents/backends/composite.py +++ b/backend/package/yuxi/agents/backends/composite.py @@ -10,7 +10,6 @@ from deepagents.backends.protocol import FileInfo from yuxi.agents.middlewares.skills_middleware import normalize_selected_skills -from .knowledge_base_backend import KnowledgeBaseReadonlyBackend from .sandbox import ProvisionerSandboxBackend from .skills_backend import SelectedSkillsReadonlyBackend @@ -111,23 +110,13 @@ def _extract_user_id(runtime) -> str: raise ValueError("user_id is required in runtime configurable context") -def _get_visible_knowledge_bases_from_runtime(runtime) -> list[dict]: - context = getattr(runtime, "context", None) - selected = getattr(context, "_visible_knowledge_bases", None) - if isinstance(selected, list): - return selected - return [] - - def create_agent_composite_backend(runtime) -> CompositeBackend: visible_skills = _get_visible_skills_from_runtime(runtime) thread_id = _extract_thread_id(runtime) user_id = _extract_user_id(runtime) - visible_kbs = _get_visible_knowledge_bases_from_runtime(runtime) return CustomCompositeBackend( default=ProvisionerSandboxBackend(thread_id=thread_id, user_id=user_id, visible_skills=visible_skills), routes={ "/skills/": SelectedSkillsReadonlyBackend(selected_slugs=visible_skills), - "/home/gem/kbs/": KnowledgeBaseReadonlyBackend(visible_kbs=visible_kbs), }, ) diff --git a/backend/package/yuxi/agents/backends/knowledge_base_backend.py b/backend/package/yuxi/agents/backends/knowledge_base_backend.py index 0d08f6a1..21b6eca1 100644 --- a/backend/package/yuxi/agents/backends/knowledge_base_backend.py +++ b/backend/package/yuxi/agents/backends/knowledge_base_backend.py @@ -1,124 +1,8 @@ from __future__ import annotations -import fnmatch -import re -from collections import defaultdict -from dataclasses import dataclass -from pathlib import Path, PurePosixPath from typing import Any -from deepagents.backends import FilesystemBackend -from deepagents.backends.protocol import EditResult, FileDownloadResponse, FileInfo, FileUploadResponse, WriteResult - -from yuxi import config as conf from yuxi import knowledge_base -from yuxi.knowledge.utils.kb_utils import is_minio_url, parse_minio_url -from yuxi.storage.minio import get_minio_client - -KBS_PATH = "/home/gem/kbs" -_INVALID_SEGMENT_RE = re.compile(r"[\\/\x00-\x1f\x7f]+") -_WHITESPACE_RE = re.compile(r"\s+") - - -@dataclass(frozen=True) -class _MaterializedFile: - virtual_path: str - cache_path: Path - source_path: str - modified_at: str | None = None - - -@dataclass(frozen=True) -class _ResolvedVirtualNode: - db_id: str - file_id: str - path: str - parent_path: str - name: str - is_folder: bool - - -@dataclass(frozen=True) -class _KnowledgeBaseVirtualLayout: - kb_virtual_names: dict[str, str] - files_by_db: dict[str, dict[str, dict[str, Any]]] - nodes_by_db: dict[str, list[_ResolvedVirtualNode]] - nodes_by_file_id: dict[str, _ResolvedVirtualNode] - source_filepaths: dict[str, str] - parsed_filepaths: dict[str, str] - - -def _normalize_virtual_path(path: str | None) -> str: - raw = str(path or "").strip() or "/" - normalized = "/" + raw.lstrip("/") - pure = PurePosixPath(normalized) - if ".." in pure.parts: - raise ValueError("path traversal is not allowed") - return str(pure) - - -def _sanitize_segment(value: str | None, fallback: str) -> str: - cleaned = str(value or "").strip() - cleaned = _INVALID_SEGMENT_RE.sub("_", cleaned) - cleaned = _WHITESPACE_RE.sub(" ", cleaned).strip() - if not cleaned or cleaned in {".", ".."}: - return fallback - return cleaned - - -def _candidate_name(file_meta: dict[str, Any]) -> str: - filename = str(file_meta.get("filename") or "").strip() - if filename: - return filename - - original = str(file_meta.get("original_filename") or "").strip() - if original: - return original - - for key in ("path", "markdown_file"): - raw = str(file_meta.get(key) or "").strip() - if raw: - try: - parsed = PurePosixPath(raw) - name = parsed.name - except Exception: # noqa: BLE001 - name = "" - if name: - return name - - return str(file_meta.get("file_id") or "") - - -def _unique_name(base_name: str, *, stable_id: str, used_names: set[str]) -> str: - if base_name not in used_names: - used_names.add(base_name) - return base_name - - suffix = f"__{stable_id[:8]}" - candidate = f"{base_name}{suffix}" - while candidate in used_names: - suffix = f"__{stable_id}" - candidate = f"{base_name}{suffix}" - used_names.add(candidate) - return candidate - - -def _materialize_text_view(content: bytes, file_path: str, *, offset: int = 0, limit: int = 2000) -> str: - if not content: - return "System reminder: File exists but has empty contents" - - if b"\x00" in content: - return f"Error: File '{file_path}' is binary and cannot be rendered as text" - - try: - text = content.decode("utf-8") - except UnicodeDecodeError: - return f"Error: File '{file_path}' is binary and cannot be rendered as text" - - lines = text.splitlines() - start = max(0, int(offset)) - selected = lines[start : start + int(limit)] - return "\n".join(f"{start + idx + 1:6d}\t{line}" for idx, line in enumerate(selected)) async def resolve_visible_knowledge_bases_for_context(context) -> list[dict[str, Any]]: @@ -142,533 +26,3 @@ async def resolve_visible_knowledge_bases_for_context(context) -> list[dict[str, setattr(context, "_visible_knowledge_bases", databases) return databases - - -def _all_files_meta() -> dict[str, dict[str, Any]]: - aggregated: dict[str, dict[str, Any]] = {} - for kb_instance in getattr(knowledge_base, "kb_instances", {}).values(): - for file_id, meta in getattr(kb_instance, "files_meta", {}).items(): - aggregated[str(file_id)] = meta - return aggregated - - -def _resolve_kb_virtual_names(visible_kbs: list[dict[str, Any]]) -> dict[str, str]: - kb_name_candidates = { - str(db.get("db_id") or db.get("name") or f"kb-{index}"): _sanitize_segment( - db.get("name"), - str(db.get("db_id") or f"kb-{index}"), - ) - for index, db in enumerate(visible_kbs) - } - used_root_names: set[str] = set() - kb_virtual_names: dict[str, str] = {} - sorted_kbs = sorted( - visible_kbs, - key=lambda item: (str(item.get("name") or ""), str(item.get("db_id") or "")), - ) - for db in sorted_kbs: - db_id = str(db.get("db_id") or "") - if not db_id: - continue - base_name = kb_name_candidates.get(db_id) or db_id - kb_virtual_names[db_id] = _unique_name(base_name, stable_id=db_id, used_names=used_root_names) - return kb_virtual_names - - -def _resolve_db_virtual_nodes( - *, - db_id: str, - kb_root: str, - records: dict[str, dict[str, Any]], -) -> list[_ResolvedVirtualNode]: - # Keep sibling deduplication local to each parent directory so the generated tree is deterministic. - children_by_parent: dict[str | None, list[tuple[str, dict[str, Any]]]] = defaultdict(list) - for file_id, meta in records.items(): - parent_id = str(meta.get("parent_id") or "").strip() or None - children_by_parent[parent_id].append((file_id, meta)) - - resolved_nodes: list[_ResolvedVirtualNode] = [] - - def walk(parent_id: str | None, parent_path: str) -> None: - used_names: set[str] = set() - siblings = children_by_parent.get(parent_id, []) - sorted_siblings = sorted( - siblings, - key=lambda item: (_sanitize_segment(_candidate_name(item[1]), item[0]), item[0]), - ) - for file_id, meta in sorted_siblings: - base_name = _sanitize_segment(_candidate_name(meta), file_id) - unique_name = _unique_name(base_name, stable_id=file_id, used_names=used_names) - child_path = f"{parent_path.rstrip('/')}/{unique_name}" if parent_path != "/" else f"/{unique_name}" - node = _ResolvedVirtualNode( - db_id=db_id, - file_id=file_id, - path=child_path, - parent_path=parent_path, - name=unique_name, - is_folder=bool(meta.get("is_folder")), - ) - resolved_nodes.append(node) - if node.is_folder: - walk(file_id, child_path) - - walk(None, kb_root) - return resolved_nodes - - -def _build_parsed_filepath_map( - *, - nodes_by_file_id: dict[str, _ResolvedVirtualNode], - files_by_db: dict[str, dict[str, dict[str, Any]]], - kb_virtual_names: dict[str, str], -) -> dict[str, str]: - # Parsed files mirror the source tree and only add a fixed `/parsed` segment plus `.md` suffix. - parsed_paths: dict[str, str] = {} - for file_id, node in nodes_by_file_id.items(): - if node.is_folder: - continue - - record = files_by_db.get(node.db_id, {}).get(file_id, {}) - if not str(record.get("markdown_file") or "").strip(): - continue - - kb_root = f"/{kb_virtual_names[node.db_id]}" - parsed_root = f"{kb_root}/parsed" - parsed_parent = ( - f"{parsed_root}{node.parent_path[len(kb_root) :]}" if node.parent_path.startswith(kb_root) else parsed_root - ) - parsed_name = _sanitize_segment(f"{node.name}.md", f"{file_id}.md") - parsed_path = f"{parsed_parent.rstrip('/')}/{parsed_name}" if parsed_parent != "/" else f"/{parsed_name}" - parsed_paths[file_id] = f"{KBS_PATH}{parsed_path}" - - return parsed_paths - - -def _resolve_virtual_layout( - *, - visible_kbs: list[dict[str, Any]] | None, - files_meta: dict[str, dict[str, Any]] | None = None, -) -> _KnowledgeBaseVirtualLayout: - # Single source of truth for the virtual KB tree. - # The readonly backend and query_kb filepath injection must stay fully aligned. - kb_virtual_names = _resolve_kb_virtual_names(list(visible_kbs or [])) - if not kb_virtual_names: - return _KnowledgeBaseVirtualLayout( - kb_virtual_names={}, - files_by_db={}, - nodes_by_db={}, - nodes_by_file_id={}, - source_filepaths={}, - parsed_filepaths={}, - ) - - source_files = _all_files_meta() if files_meta is None else files_meta - files_by_db: dict[str, dict[str, dict[str, Any]]] = defaultdict(dict) - for file_id, meta in source_files.items(): - db_id = str(meta.get("database_id") or "") - if db_id in kb_virtual_names: - files_by_db[db_id][str(file_id)] = meta - - nodes_by_db: dict[str, list[_ResolvedVirtualNode]] = {} - nodes_by_file_id: dict[str, _ResolvedVirtualNode] = {} - source_filepaths: dict[str, str] = {} - - for db_id, kb_virtual_name in kb_virtual_names.items(): - kb_root = f"/{kb_virtual_name}" - records = files_by_db.get(db_id, {}) - nodes = _resolve_db_virtual_nodes(db_id=db_id, kb_root=kb_root, records=records) - nodes_by_db[db_id] = nodes - for node in nodes: - nodes_by_file_id[node.file_id] = node - if not node.is_folder: - source_filepaths[node.file_id] = f"{KBS_PATH}{node.path}" - - return _KnowledgeBaseVirtualLayout( - kb_virtual_names=kb_virtual_names, - files_by_db=dict(files_by_db), - nodes_by_db=nodes_by_db, - nodes_by_file_id=nodes_by_file_id, - source_filepaths=source_filepaths, - parsed_filepaths=_build_parsed_filepath_map( - nodes_by_file_id=nodes_by_file_id, - files_by_db=files_by_db, - kb_virtual_names=kb_virtual_names, - ), - ) - - -def resolve_file_relative_virtual_path( - *, - file_id: str, - visible_kbs: list[dict[str, Any]] | None, - files_meta: dict[str, dict[str, Any]] | None = None, -) -> str | None: - normalized_id = str(file_id or "").strip() - if not normalized_id: - return None - - layout = _resolve_virtual_layout( - visible_kbs=visible_kbs, - files_meta=files_meta, - ) - node = layout.nodes_by_file_id.get(normalized_id) - if node is None or node.is_folder: - return None - return node.path - - -def build_knowledge_base_filepath_map( - *, - visible_kbs: list[dict[str, Any]] | None, - files_meta: dict[str, dict[str, Any]] | None = None, -) -> dict[str, str]: - layout = _resolve_virtual_layout( - visible_kbs=visible_kbs, - files_meta=files_meta, - ) - return layout.source_filepaths.copy() - - -def _inject_kb_filepaths( - chunks: list[dict[str, Any]], - filepath_map: dict[str, str], - parsed_filepath_map: dict[str, str], -) -> list[dict[str, Any]]: - if not filepath_map and not parsed_filepath_map: - return chunks - - for chunk in chunks: - if not isinstance(chunk, dict): - continue - - metadata = chunk.get("metadata") - if metadata is None: - metadata = {} - if not isinstance(metadata, dict): - continue - - file_id = str(metadata.get("file_id") or chunk.get("file_id") or "").strip() - if not file_id: - continue - - if not metadata.get("filepath"): - filepath = filepath_map.get(file_id) - if filepath: - metadata["filepath"] = filepath - - if not metadata.get("parsed_path"): - parsed_path = parsed_filepath_map.get(file_id) - if parsed_path: - metadata["parsed_path"] = parsed_path - - chunk["metadata"] = metadata - - return chunks - - -async def inject_filepaths_into_retrieval_result( - *, - retrieval_chunks: list[dict[str, Any]], - visible_kbs: list[dict[str, Any]] | None, - target_db_id: str | None, - target_kb_name: str | None = None, -) -> list[dict[str, Any]]: - scope_kbs = list(visible_kbs or []) - if not scope_kbs and target_db_id: - scope_kbs = [{"db_id": target_db_id, "name": target_kb_name or target_db_id}] - - layout = _resolve_virtual_layout(visible_kbs=scope_kbs) - return _inject_kb_filepaths( - retrieval_chunks, - layout.source_filepaths, - layout.parsed_filepaths, - ) - - -class KnowledgeBaseReadonlyBackend(FilesystemBackend): - def __init__(self, *, visible_kbs: list[dict[str, Any]] | None, cache_root: Path | str | None = None): - self._cache_root = Path(cache_root or (Path(conf.save_dir) / "knowledge_base_data" / "kb-cache")).resolve() - self._cache_root.mkdir(parents=True, exist_ok=True) - super().__init__(root_dir=self._cache_root, virtual_mode=True) - self._visible_kbs = list(visible_kbs or []) - self._entries_by_dir: dict[str, list[FileInfo]] = defaultdict(list) - self._dir_paths: set[str] = {"/"} - self._files: dict[str, _MaterializedFile] = {} - self._all_files: list[FileInfo] = [] - self._build_virtual_tree() - - def has_entries(self) -> bool: - return bool(self._visible_kbs) - - def _build_virtual_tree(self) -> None: - layout = _resolve_virtual_layout(visible_kbs=self._visible_kbs) - - for db_id, kb_virtual_name in layout.kb_virtual_names.items(): - kb_root = f"/{kb_virtual_name}" - self._add_entry("/", kb_root, is_dir=True) - records = layout.files_by_db.get(db_id, {}) - self._build_source_tree(db_id=db_id, records=records, nodes=layout.nodes_by_db.get(db_id, [])) - self._build_parsed_tree(db_id=db_id, kb_root=kb_root, records=records) - - for path, entries in list(self._entries_by_dir.items()): - entries.sort(key=lambda item: str(item.get("path") or "")) - self._entries_by_dir[path] = entries - self._all_files = sorted(self._all_files, key=lambda item: str(item.get("path") or "")) - - def _build_source_tree( - self, - *, - db_id: str, - records: dict[str, dict[str, Any]], - nodes: list[_ResolvedVirtualNode], - ) -> None: - resolved_parent_paths: dict[str, str] = {} - resolved_source_names: dict[str, str] = {} - - for node in sorted(nodes, key=lambda item: item.path): - meta = records.get(node.file_id, {}) - modified_at = meta.get("updated_at") or meta.get("created_at") - if node.is_folder: - self._add_entry(node.parent_path, node.path, is_dir=True, modified_at=modified_at) - continue - - self._add_entry( - node.parent_path, - node.path, - is_dir=False, - size=int(meta.get("size") or 0), - modified_at=modified_at, - ) - resolved_parent_paths[node.file_id] = node.parent_path - resolved_source_names[node.file_id] = node.name - cache_path = self._cache_root / db_id / "source" / node.file_id / node.name - self._files[node.path] = _MaterializedFile( - virtual_path=node.path, - cache_path=cache_path, - source_path=str(meta.get("path") or ""), - modified_at=modified_at, - ) - self._all_files.append( - { - "path": node.path, - "is_dir": False, - "size": int(meta.get("size") or 0), - "modified_at": str(modified_at or ""), - } - ) - - self._resolved_parent_paths = getattr(self, "_resolved_parent_paths", {}) - self._resolved_parent_paths[db_id] = resolved_parent_paths - self._resolved_source_names = getattr(self, "_resolved_source_names", {}) - self._resolved_source_names[db_id] = resolved_source_names - - def _build_parsed_tree(self, *, db_id: str, kb_root: str, records: dict[str, dict[str, Any]]) -> None: - parsed_records = { - file_id: meta - for file_id, meta in records.items() - if not meta.get("is_folder") and str(meta.get("markdown_file") or "").strip() - } - if not parsed_records: - return - - parsed_root = f"{kb_root}/parsed" - self._add_entry(kb_root, parsed_root, is_dir=True) - parent_paths = getattr(self, "_resolved_parent_paths", {}).get(db_id, {}) - source_names = getattr(self, "_resolved_source_names", {}).get(db_id, {}) - - grouped: dict[str, list[tuple[str, dict[str, Any], str]]] = defaultdict(list) - for file_id, meta in parsed_records.items(): - source_parent = parent_paths.get(file_id, kb_root) - parsed_parent = ( - f"{parsed_root}{source_parent[len(kb_root) :]}" if source_parent.startswith(kb_root) else parsed_root - ) - source_name = source_names.get(file_id) or _sanitize_segment(meta.get("filename"), file_id) - safe_name = source_name or file_id - base_name = _sanitize_segment(f"{safe_name}.md", f"{file_id}.md") - grouped[parsed_parent].append((file_id, meta, base_name)) - - for parsed_parent, items in grouped.items(): - current = PurePosixPath(parsed_parent) - while str(current) not in {"", "."} and str(current) != "/": - current_str = str(current) - parent_str = str(current.parent) if str(current.parent) != "." else "/" - if current_str not in self._dir_paths: - self._add_entry(parent_str, current_str, is_dir=True) - current = current.parent - - used_names: set[str] = set() - for file_id, meta, base_name in sorted(items, key=lambda item: (item[2], item[0])): - unique_name = _unique_name(base_name, stable_id=file_id, used_names=used_names) - file_path = f"{parsed_parent.rstrip('/')}/{unique_name}" if parsed_parent != "/" else f"/{unique_name}" - self._add_entry( - parsed_parent, - file_path, - is_dir=False, - size=0, - modified_at=meta.get("updated_at") or meta.get("created_at"), - ) - cache_path = self._cache_root / db_id / "parsed" / f"{file_id}.md" - self._files[file_path] = _MaterializedFile( - virtual_path=file_path, - cache_path=cache_path, - source_path=str(meta.get("markdown_file") or ""), - modified_at=meta.get("updated_at") or meta.get("created_at"), - ) - self._all_files.append( - { - "path": file_path, - "is_dir": False, - "size": 0, - "modified_at": str(meta.get("updated_at") or meta.get("created_at") or ""), - } - ) - - def _add_entry( - self, - parent_path: str, - child_path: str, - *, - is_dir: bool, - size: int = 0, - modified_at: str | None = None, - ) -> None: - normalized_parent = _normalize_virtual_path(parent_path) - normalized_child = _normalize_virtual_path(child_path) - entry_path = f"{normalized_child}/" if is_dir else normalized_child - entry: FileInfo = { - "path": entry_path, - "is_dir": is_dir, - "size": int(size or 0), - "modified_at": str(modified_at or ""), - } - if entry_path not in {str(item.get("path")) for item in self._entries_by_dir[normalized_parent]}: - self._entries_by_dir[normalized_parent].append(entry) - if is_dir: - self._dir_paths.add(normalized_child) - self._entries_by_dir.setdefault(normalized_child, []) - - def _ensure_local_file(self, descriptor: _MaterializedFile) -> Path: - if descriptor.cache_path.exists(): - return descriptor.cache_path - - source = descriptor.source_path.strip() - if not source: - raise FileNotFoundError(descriptor.virtual_path) - if not is_minio_url(source): - raise FileNotFoundError(descriptor.virtual_path) - - bucket_name, object_name = parse_minio_url(source) - payload = get_minio_client().download_file(bucket_name, object_name) - descriptor.cache_path.parent.mkdir(parents=True, exist_ok=True) - descriptor.cache_path.write_bytes(payload) - return descriptor.cache_path - - def ls_info(self, path: str) -> list[FileInfo]: - normalized_path = _normalize_virtual_path(path) - return list(self._entries_by_dir.get(normalized_path, [])) - - def read(self, file_path: str, offset: int = 0, limit: int = 2000) -> str: - normalized_path = _normalize_virtual_path(file_path) - descriptor = self._files.get(normalized_path) - if descriptor is None: - if normalized_path in self._dir_paths: - return f"Error: Path '{file_path}' is a directory" - return f"Error: File '{file_path}' not found" - - try: - content = self._ensure_local_file(descriptor).read_bytes() - except FileNotFoundError: - return f"Error: File '{file_path}' not found" - except Exception as exc: # noqa: BLE001 - detail = str(exc).strip() or "unknown error" - return f"Error: Failed to read '{file_path}': {detail}" - - return _materialize_text_view(content, file_path, offset=offset, limit=limit) - - def glob_info(self, pattern: str, path: str = "/") -> list[FileInfo]: - normalized_path = _normalize_virtual_path(path) - if ".." in PurePosixPath(pattern).parts: - raise ValueError("Path traversal not allowed in glob pattern") - normalized_pattern = pattern.lstrip("/") or "*" - prefix = "/" if normalized_path == "/" else f"{normalized_path.rstrip('/')}/" - - matches: list[FileInfo] = [] - for item in self._all_files: - item_path = str(item.get("path") or "") - if normalized_path != "/" and not item_path.startswith(prefix): - continue - relative = item_path[len(prefix) :] if normalized_path != "/" else item_path.lstrip("/") - if fnmatch.fnmatch(relative, normalized_pattern): - matches.append(dict(item)) - return matches - - def grep_raw(self, pattern: str, path: str | None = None, glob: str | None = None) -> list[dict[str, Any]] | str: - normalized_path = _normalize_virtual_path(path or "/") - prefix = "/" if normalized_path == "/" else f"{normalized_path.rstrip('/')}/" - if normalized_path in self._files: - targets = [normalized_path] - else: - targets = [ - item["path"] - for item in self._all_files - if normalized_path == "/" or str(item["path"]).startswith(prefix) - ] - - matches: list[dict[str, Any]] = [] - for target in targets: - display_target = str(target) - relative = display_target.lstrip("/") if normalized_path == "/" else display_target[len(prefix) :] - if glob and not fnmatch.fnmatch(relative, glob): - continue - descriptor = self._files.get(display_target) - if descriptor is None: - continue - try: - content = self._ensure_local_file(descriptor).read_bytes() - if b"\x00" in content: - continue - text = content.decode("utf-8") - except Exception: # noqa: BLE001 - continue - for line_num, line in enumerate(text.splitlines(), start=1): - if pattern in line: - matches.append({"path": display_target, "line": line_num, "text": line}) - return matches - - def write(self, file_path: str, content: str) -> WriteResult: - return WriteResult(error="Knowledge base path is read-only.") - - def edit(self, file_path: str, old_string: str, new_string: str, replace_all: bool = False) -> EditResult: - return EditResult(error="Knowledge base path is read-only.") - - def upload_files(self, files: list[tuple[str, bytes]]) -> list[FileUploadResponse]: - return [FileUploadResponse(path=path, error="permission_denied") for path, _content in files] - - def download_files(self, paths: list[str]) -> list[FileDownloadResponse]: - responses: list[FileDownloadResponse] = [] - for path in paths: - try: - normalized_path = _normalize_virtual_path(path) - except ValueError: - responses.append(FileDownloadResponse(path=path, content=None, error="invalid_path")) - continue - - descriptor = self._files.get(normalized_path) - if descriptor is None: - if normalized_path in self._dir_paths: - responses.append(FileDownloadResponse(path=path, content=None, error="is_directory")) - else: - responses.append(FileDownloadResponse(path=path, content=None, error="file_not_found")) - continue - - try: - content = self._ensure_local_file(descriptor).read_bytes() - except FileNotFoundError: - responses.append(FileDownloadResponse(path=path, content=None, error="file_not_found")) - continue - except ValueError: - responses.append(FileDownloadResponse(path=path, content=None, error="invalid_path")) - continue - - responses.append(FileDownloadResponse(path=path, content=content, error=None)) - return responses diff --git a/backend/package/yuxi/agents/buildin/chatbot/prompt.py b/backend/package/yuxi/agents/buildin/chatbot/prompt.py index ae885b17..e57e03a9 100644 --- a/backend/package/yuxi/agents/buildin/chatbot/prompt.py +++ b/backend/package/yuxi/agents/buildin/chatbot/prompt.py @@ -1,5 +1,4 @@ from yuxi.utils.paths import ( - VIRTUAL_KBS_PATH, VIRTUAL_PATH_OUTPUTS, VIRTUAL_PATH_PREFIX, VIRTUAL_PATH_UPLOADS, @@ -25,9 +24,8 @@ PROMPT = f""" 非必要不写入其他路径 <| 知识库访问 |> -当 query_kb 中没有找到相关的内容，或者需要进一步基于检索到的内容获取更加详细的上下文的时候，还可以直接访问知识库文件系统 -（路径为 {VIRTUAL_KBS_PATH}）来获取信息。 -源文件可能无法直接读取，可以在 {VIRTUAL_KBS_PATH}//parsed/ 中找到解析后的 markdown 文件。 +当 query_kb 中没有找到相关内容，或者需要进一步基于检索结果获取更详细上下文时， +使用 open_kb_document 按 resource_id 和 file_id 打开知识库文档。 """ # 效果不好，暂时不启用 diff --git a/backend/package/yuxi/agents/toolkits/kbs/tools.py b/backend/package/yuxi/agents/toolkits/kbs/tools.py index 02ee8a46..6a40a551 100644 --- a/backend/package/yuxi/agents/toolkits/kbs/tools.py +++ b/backend/package/yuxi/agents/toolkits/kbs/tools.py @@ -41,24 +41,19 @@ async def list_kbs(dummy: str, runtime: ToolRuntime) -> str: # Now has 2 params enabled_kb_names = getattr(runtime_context, "knowledges", None) - # 获取用户可访问的知识库列表（包含名称和描述） try: - result = await knowledge_base.get_databases_by_raw_id(user_id) - all_kbs = result.get("databases", []) + from yuxi.agents.backends.knowledge_base_backend import resolve_visible_knowledge_bases_for_context + + available_kbs = await resolve_visible_knowledge_bases_for_context(runtime_context) except Exception as e: logger.error(f"获取用户知识库列表失败: {e}") return f"获取知识库列表失败: {str(e)}" - all_kb_names = [kb["name"] for kb in all_kbs] + all_kb_names = [kb["name"] for kb in available_kbs] logger.debug(f"用户 {user_id} 可访问的知识库列表: {all_kb_names}") logger.debug(f"用户 {user_id} 当前对话启用的知识库列表: {enabled_kb_names}") - if enabled_kb_names is None: - available_kbs = all_kbs - else: - available_kbs = [kb for kb in all_kbs if kb["name"] in enabled_kb_names] - if not available_kbs: return "当前没有可访问的知识库" @@ -188,7 +183,7 @@ async def _resolve_visible_knowledge_bases_for_query(runtime: ToolRuntime | None return await resolve_visible_knowledge_bases_for_context(context) except Exception as exc: # noqa: BLE001 - logger.warning(f"解析会话可见知识库失败，跳过 filepath 注入: {exc}") + logger.warning(f"解析会话可见知识库失败: {exc}") return [] @@ -197,21 +192,21 @@ def _find_query_target( kb_name: str, retrievers: dict[str, Any], visible_kbs: list[dict[str, Any]], -) -> tuple[str | None, dict[str, Any] | None, str | None]: +) -> tuple[dict[str, Any] | None, str | None]: if not visible_kbs: - return None, None, "无法获取当前会话可访问的知识库" + return None, "无法获取当前会话可访问的知识库" matched_kbs = [db for db in visible_kbs if str(db.get("name") or "").strip() == kb_name] if not matched_kbs: - return None, None, f"知识库 '{kb_name}' 不存在或当前会话未启用" + return None, f"知识库 '{kb_name}' 不存在或当前会话未启用" if len(matched_kbs) > 1: - return None, None, f"知识库 '{kb_name}' 存在重名，请先调整名称后重试" + return None, f"知识库 '{kb_name}' 存在重名，请先调整名称后重试" target_db_id = str(matched_kbs[0].get("db_id") or "") target_info = retrievers.get(target_db_id) if target_info is None: - return None, None, f"知识库 '{kb_name}' 不存在" - return target_db_id, target_info, None + return None, f"知识库 '{kb_name}' 不存在" + return target_info, None def _normalize_retrieval_result_metadata(result: Any) -> Any: @@ -265,7 +260,7 @@ async def query_kb(kb_name: str, query_text: str, file_name: str | None = None, visible_kbs = await _resolve_visible_knowledge_bases_for_query(runtime) - target_db_id, target_info, target_error = _find_query_target( + target_info, target_error = _find_query_target( kb_name=kb_name, retrievers=retrievers, visible_kbs=visible_kbs, @@ -273,9 +268,6 @@ async def query_kb(kb_name: str, query_text: str, file_name: str | None = None, if target_error: return target_error - metadata = target_info.get("metadata") if isinstance(target_info, dict) else None - kb_type = str((metadata or {}).get("kb_type") or "").strip().lower() - try: retriever = target_info["retriever"] kwargs = {} @@ -287,23 +279,7 @@ async def query_kb(kb_name: str, query_text: str, file_name: str | None = None, else: result = retriever(query_text, **kwargs) - result = _normalize_retrieval_result_metadata(result) - - if kb_type != "milvus": - return result - - if not isinstance(result, list): - return f"知识库 '{kb_name}' 返回结果不是 Milvus chunks 列表，无法注入文件路径" - - from yuxi.agents.backends.knowledge_base_backend import inject_filepaths_into_retrieval_result - - # 只有 Milvus 结果的 file_id 对应本地文件系统，可补充沙盒可读路径。 - return await inject_filepaths_into_retrieval_result( - retrieval_chunks=result, - visible_kbs=visible_kbs, - target_db_id=target_db_id, - target_kb_name=kb_name, - ) + return _normalize_retrieval_result_metadata(result) except Exception as e: logger.error(f"检索失败: {e}") diff --git a/backend/package/yuxi/services/knowledge_fs_service.py b/backend/package/yuxi/services/knowledge_fs_service.py deleted file mode 100644 index 4d7c9cb5..00000000 --- a/backend/package/yuxi/services/knowledge_fs_service.py +++ /dev/null @@ -1,169 +0,0 @@ -from __future__ import annotations - -import hashlib -import json -import re -import tempfile -from pathlib import Path -from typing import Any - -from yuxi.config import config -from yuxi.knowledge import knowledge_base -from yuxi.knowledge.utils.kb_utils import parse_minio_url -from yuxi.repositories.knowledge_file_repository import KnowledgeFileRepository -from yuxi.storage.minio import get_minio_client - -KBS_PATH = "/home/gem/kbs" -PARSED_DIR_NAME = "parsed" -_INVALID_MOUNT_NAME_CHARS = re.compile(r'[\\/:*?"<>|\x00-\x1f]') -_MULTISPACE = re.compile(r"\s+") - - -def get_kb_cache_root() -> Path: - return Path(config.save_dir) / "knowledge_base_data" / "kb-cache" - - -def normalize_knowledge_mount_name(name: str) -> str: - normalized = _MULTISPACE.sub(" ", str(name or "").strip()) - if _INVALID_MOUNT_NAME_CHARS.search(normalized): - raise ValueError("知识库名称包含不能映射为目录名的非法字符") - normalized = normalized.strip(" .") - if not normalized or normalized in {".", ".."}: - raise ValueError("知识库名称不能映射为有效目录名") - if "/" in normalized or "\\" in normalized: - raise ValueError("知识库名称不能包含路径分隔符") - return normalized - - -def validate_knowledge_mount_name(name: str) -> str: - return normalize_knowledge_mount_name(name) - - -def _normalize_selected_knowledges(selected: list[str] | None) -> list[str]: - normalized: list[str] = [] - seen: set[str] = set() - for item in selected or []: - if not isinstance(item, str): - continue - value = item.strip() - if not value: - continue - key = value.casefold() - if key in seen: - continue - seen.add(key) - normalized.append(value) - return normalized - - -def _derive_parsed_filename(filename: str, file_id: str, used_names: set[str]) -> str: - raw_name = (filename or file_id or "file").strip() or file_id or "file" - suffix = Path(raw_name).suffix - stem = Path(raw_name).name[: -len(suffix)] if suffix else Path(raw_name).name - candidate = f"{stem or file_id}.md" - lowered = candidate.casefold() - if lowered in used_names: - candidate = f"{stem or file_id}__{file_id}.md" - lowered = candidate.casefold() - used_names.add(lowered) - return candidate - - -def _serialize_file_record(record) -> dict[str, Any]: - return { - "file_id": record.file_id, - "db_id": record.db_id, - "parent_id": record.parent_id, - "filename": record.filename, - "original_filename": record.original_filename, - "file_type": record.file_type, - "path": record.path, - "minio_url": record.minio_url, - "markdown_file": record.markdown_file, - "status": record.status, - "content_hash": record.content_hash, - "file_size": record.file_size, - "content_type": record.content_type, - "processing_params": record.processing_params, - "is_folder": bool(record.is_folder), - } - - -async def build_visible_knowledge_mounts( - *, - user_id: str, - selected_knowledges: list[str] | None, -) -> list[dict[str, Any]]: - accessible = (await knowledge_base.get_databases_by_user_id(user_id)).get("databases", []) - selected = _normalize_selected_knowledges(selected_knowledges) - selected_keys = {item.casefold() for item in selected} - - file_repo = KnowledgeFileRepository() - mounts: list[dict[str, Any]] = [] - used_mount_names: dict[str, str] = {} - - for db in accessible: - db_id = str(db.get("db_id") or "").strip() - db_name = str(db.get("name") or db_id).strip() - if not db_id or not db_name: - continue - if selected_keys and db_name.casefold() not in selected_keys and db_id.casefold() not in selected_keys: - continue - - mount_name = normalize_knowledge_mount_name(db_name) - conflict_db_id = used_mount_names.get(mount_name.casefold()) - if conflict_db_id and conflict_db_id != db_id: - raise ValueError(f"知识库名称映射冲突: '{db_name}' -> '{mount_name}'") - used_mount_names[mount_name.casefold()] = db_id - - records = await file_repo.list_by_db_id(db_id) - mounts.append( - { - "db_id": db_id, - "db_name": db_name, - "mount_name": mount_name, - "files": [_serialize_file_record(record) for record in records], - } - ) - - mounts.sort(key=lambda item: item["mount_name"].casefold()) - return mounts - - -def cache_minio_object(*, source_url: str, metadata: dict[str, Any] | None = None) -> Path: - bucket_name, object_name = parse_minio_url(source_url) - minio_client = get_minio_client() - stat = minio_client.client.stat_object(bucket_name=bucket_name, object_name=object_name) - etag = str(getattr(stat, "etag", "") or "") - last_modified = getattr(stat, "last_modified", None) - version_key = etag or (last_modified.isoformat() if last_modified else "") - cache_key = hashlib.sha256(f"{bucket_name}:{object_name}:{version_key}".encode()).hexdigest() - - suffix = Path(object_name).suffix - objects_root = get_kb_cache_root() / "objects" - manifests_root = get_kb_cache_root() / "manifests" - objects_root.mkdir(parents=True, exist_ok=True) - manifests_root.mkdir(parents=True, exist_ok=True) - - cached_path = objects_root / f"{cache_key}{suffix}" - if not cached_path.exists(): - payload = minio_client.download_file(bucket_name=bucket_name, object_name=object_name) - with tempfile.NamedTemporaryFile(dir=objects_root, delete=False) as tmp: - tmp.write(payload) - tmp_path = Path(tmp.name) - tmp_path.replace(cached_path) - - manifest_path = manifests_root / f"{cache_key}.json" - if not manifest_path.exists(): - manifest = { - "bucket_name": bucket_name, - "object_name": object_name, - "etag": etag, - "last_modified": last_modified.isoformat() if last_modified else None, - "source_url": source_url, - "cached_path": str(cached_path), - "metadata": metadata or {}, - } - manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8") - - return cached_path diff --git a/backend/package/yuxi/services/viewer_filesystem_service.py b/backend/package/yuxi/services/viewer_filesystem_service.py index bea968dd..9514f7a4 100644 --- a/backend/package/yuxi/services/viewer_filesystem_service.py +++ b/backend/package/yuxi/services/viewer_filesystem_service.py @@ -12,7 +12,6 @@ import aiofiles from fastapi import HTTPException, UploadFile from fastapi.responses import FileResponse, StreamingResponse from sqlalchemy.ext.asyncio import AsyncSession -from yuxi.agents.backends import KBS_PATH, KnowledgeBaseReadonlyBackend, resolve_visible_knowledge_bases_for_context from yuxi.agents.backends.sandbox import ( SKILLS_PATH, USER_DATA_PATH, @@ -90,6 +89,7 @@ _PROTECTED_USER_DATA_ROOTS = frozenset( VIRTUAL_PATH_OUTPUTS, } ) +_LEGACY_KBS_PATH = "/home/gem/kbs" def _detect_preview_type(path: str, raw_content: bytes) -> tuple[str, bool, str | None]: @@ -139,7 +139,7 @@ def _normalize_path(path: str | None) -> str: normalized = (path or "/").strip() or "/" if not normalized.startswith("/"): normalized = f"/{normalized}" - return normalized.rstrip("/") if normalized not in {"/", KBS_PATH, SKILLS_PATH, USER_DATA_PATH} else normalized + return normalized.rstrip("/") if normalized not in {"/", SKILLS_PATH, USER_DATA_PATH} else normalized def _is_path_within(path: Path, root: Path) -> bool: @@ -180,10 +180,6 @@ def _is_skills_path(path: str) -> bool: return path == SKILLS_PATH or path.startswith(f"{SKILLS_PATH}/") -def _is_kbs_path(path: str) -> bool: - return path == KBS_PATH or path.startswith(f"{KBS_PATH}/") - - def _is_in_home_gem(path: str) -> bool: """检查路径是否在 /home/gem/ 下但不在虚拟挂载点内""" if not path.startswith("/home/gem/"): @@ -193,7 +189,7 @@ def _is_in_home_gem(path: str) -> bool: return False if path.startswith(f"{SKILLS_PATH}/") or path == SKILLS_PATH: return False - if path.startswith(f"{KBS_PATH}/") or path == KBS_PATH: + if path == _LEGACY_KBS_PATH or path.startswith(f"{_LEGACY_KBS_PATH}/"): return False return True @@ -204,12 +200,6 @@ def _strip_skills_prefix(path: str) -> str: return path[len(SKILLS_PATH) :] or "/" -def _strip_kbs_prefix(path: str) -> str: - if path == KBS_PATH: - return "/" - return path[len(KBS_PATH) :] or "/" - - def _remap_prefixed_entry(entry: dict, prefix: str) -> dict: raw_path = str(entry.get("path") or "") is_dir = bool(entry.get("is_dir", False)) @@ -358,11 +348,9 @@ async def _resolve_viewer_state( agent_id=agent_id, agent_config_id=agent_config_id, ) - visible_kbs = await resolve_visible_knowledge_bases_for_context(runtime_context) selected_skills = normalize_selected_skills(getattr(runtime_context, "skills", None) or []) skills_backend = SelectedSkillsReadonlyBackend(selected_slugs=selected_skills) - kb_backend = KnowledgeBaseReadonlyBackend(visible_kbs=visible_kbs) - return sandbox_backend, skills_backend, kb_backend, selected_skills + return sandbox_backend, skills_backend, selected_skills async def list_viewer_filesystem_tree( @@ -378,7 +366,7 @@ async def list_viewer_filesystem_tree( raise HTTPException(status_code=422, detail="thread_id 不能为空") normalized_path = _normalize_path(path) - sandbox_backend, skills_backend, kb_backend, selected_skills = await _resolve_viewer_state( + sandbox_backend, skills_backend, selected_skills = await _resolve_viewer_state( thread_id=thread_id, agent_id=agent_id, agent_config_id=agent_config_id, @@ -395,8 +383,6 @@ async def list_viewer_filesystem_tree( ) if selected_skills: entries.append({"path": f"{SKILLS_PATH}/", "name": "skills", "is_dir": True, "size": 0, "modified_at": ""}) - if kb_backend.has_entries(): - entries.append({"path": f"{KBS_PATH}/", "name": "kbs", "is_dir": True, "size": 0, "modified_at": ""}) return {"entries": _sort_entries(entries)} @@ -419,10 +405,6 @@ async def list_viewer_filesystem_tree( entries = await asyncio.to_thread(skills_backend.ls_info, _strip_skills_prefix(normalized_path)) remapped = [_remap_prefixed_entry(entry, SKILLS_PATH) for entry in entries] return {"entries": _sort_entries(remapped)} - if _is_kbs_path(normalized_path): - entries = await asyncio.to_thread(kb_backend.ls_info, _strip_kbs_prefix(normalized_path)) - remapped = [_remap_prefixed_entry(entry, KBS_PATH) for entry in entries] - return {"entries": _sort_entries(remapped)} except PermissionError as e: raise HTTPException(status_code=400, detail=str(e)) from e except ValueError as e: @@ -444,7 +426,7 @@ async def read_viewer_file_content( raise HTTPException(status_code=422, detail="thread_id 不能为空") normalized_path = _normalize_path(path) - sandbox_backend, skills_backend, kb_backend, _selected_skills = await _resolve_viewer_state( + sandbox_backend, skills_backend, _selected_skills = await _resolve_viewer_state( thread_id=thread_id, agent_id=agent_id, agent_config_id=agent_config_id, @@ -476,8 +458,6 @@ async def read_viewer_file_content( } elif _is_skills_path(normalized_path): responses = await asyncio.to_thread(skills_backend.download_files, [_strip_skills_prefix(normalized_path)]) - elif _is_kbs_path(normalized_path): - responses = await asyncio.to_thread(kb_backend.download_files, [_strip_kbs_prefix(normalized_path)]) elif _is_in_home_gem(normalized_path): # /home/gem/ 下的其他文件（如 workspace 目录） responses = await asyncio.to_thread(sandbox_backend.download_files, [normalized_path]) @@ -537,7 +517,7 @@ async def download_viewer_file( db: AsyncSession, ) -> StreamingResponse: normalized_path = _normalize_path(path) - sandbox_backend, skills_backend, kb_backend, _selected_skills = await _resolve_viewer_state( + sandbox_backend, skills_backend, _selected_skills = await _resolve_viewer_state( thread_id=thread_id, agent_id=agent_id, agent_config_id=agent_config_id, @@ -562,8 +542,6 @@ async def download_viewer_file( if _is_skills_path(normalized_path): responses = await asyncio.to_thread(skills_backend.download_files, [_strip_skills_prefix(normalized_path)]) - elif _is_kbs_path(normalized_path): - responses = await asyncio.to_thread(kb_backend.download_files, [_strip_kbs_prefix(normalized_path)]) elif _is_in_home_gem(normalized_path): # /home/gem/ 下的其他文件（如 workspace 目录） responses = await asyncio.to_thread(sandbox_backend.download_files, [normalized_path]) diff --git a/backend/package/yuxi/utils/paths.py b/backend/package/yuxi/utils/paths.py index 304b5514..cfa62093 100644 --- a/backend/package/yuxi/utils/paths.py +++ b/backend/package/yuxi/utils/paths.py @@ -9,7 +9,6 @@ WORKSPACE_AGENTS_PROMPT_FILE_NAME = "AGENTS.md" UPLOADS_DIR_NAME = "uploads" OUTPUTS_DIR_NAME = "outputs" VIRTUAL_SKILLS_PATH = "/home/gem/skills" -VIRTUAL_KBS_PATH = "/home/gem/kbs" VIRTUAL_PATH_WORKSPACE = (Path(VIRTUAL_PATH_PREFIX) / WORKSPACE_DIR_NAME).as_posix() VIRTUAL_PATH_UPLOADS = (Path(VIRTUAL_PATH_PREFIX) / UPLOADS_DIR_NAME).as_posix() @@ -26,5 +25,4 @@ __all__ = [ "VIRTUAL_PATH_UPLOADS", "VIRTUAL_PATH_OUTPUTS", "VIRTUAL_SKILLS_PATH", - "VIRTUAL_KBS_PATH", ] diff --git a/backend/test/integration/api/test_viewer_filesystem_router.py b/backend/test/integration/api/test_viewer_filesystem_router.py index 97d2f23e..6c2841d1 100644 --- a/backend/test/integration/api/test_viewer_filesystem_router.py +++ b/backend/test/integration/api/test_viewer_filesystem_router.py @@ -620,7 +620,7 @@ async def test_viewer_create_directory_rejects_invalid_names(test_client, standa assert response.status_code == 422, response.text -async def test_viewer_tree_root_hides_kbs_namespace_when_no_database_is_visible(test_client, standard_user): +async def test_viewer_tree_root_hides_kbs_namespace(test_client, standard_user): headers = standard_user["headers"] thread_id = await _create_thread_for_user(test_client, headers) @@ -633,13 +633,11 @@ async def test_viewer_tree_root_hides_kbs_namespace_when_no_database_is_visible( entries = response.json().get("entries", []) paths = {entry.get("path") for entry in entries} - if "/home/gem/kbs/" in paths: - pytest.skip("Current integration database has visible knowledge bases.") assert "/home/gem/user-data/" in paths assert "/home/gem/kbs/" not in paths -async def test_viewer_kbs_namespace_is_empty_when_no_database_is_visible(test_client, standard_user): +async def test_viewer_rejects_kbs_namespace(test_client, standard_user): headers = standard_user["headers"] thread_id = await _create_thread_for_user(test_client, headers) @@ -648,8 +646,18 @@ async def test_viewer_kbs_namespace_is_empty_when_no_database_is_visible(test_cl params={"thread_id": thread_id, "path": "/home/gem/kbs"}, headers=headers, ) - assert tree_response.status_code == 200, tree_response.text - entries = tree_response.json().get("entries", []) - if entries: - pytest.skip("Current integration database has visible knowledge bases.") - assert entries == [] + assert tree_response.status_code == 400, tree_response.text + + file_response = await test_client.get( + "/api/viewer/filesystem/file", + params={"thread_id": thread_id, "path": "/home/gem/kbs/demo.md"}, + headers=headers, + ) + assert file_response.status_code == 400, file_response.text + + download_response = await test_client.get( + "/api/viewer/filesystem/download", + params={"thread_id": thread_id, "path": "/home/gem/kbs/demo.md"}, + headers=headers, + ) + assert download_response.status_code == 400, download_response.text diff --git a/backend/test/unit/backends/test_knowledge_base_backend.py b/backend/test/unit/backends/test_knowledge_base_backend.py index d5c910a7..39f4ae4a 100644 --- a/backend/test/unit/backends/test_knowledge_base_backend.py +++ b/backend/test/unit/backends/test_knowledge_base_backend.py @@ -3,93 +3,7 @@ from __future__ import annotations from types import SimpleNamespace import pytest -from yuxi.agents.backends.knowledge_base_backend import ( - KBS_PATH, - KnowledgeBaseReadonlyBackend, - build_knowledge_base_filepath_map, - inject_filepaths_into_retrieval_result, - resolve_file_relative_virtual_path, - resolve_visible_knowledge_bases_for_context, -) - - -def test_knowledge_base_backend_builds_virtual_tree_and_materializes_files(monkeypatch, tmp_path) -> None: - visible_kbs = [ - {"db_id": "db-1", "name": "FAQ"}, - {"db_id": "db-2", "name": "FAQ"}, - ] - files_meta = { - "folder-api": { - "file_id": "folder-api", - "database_id": "db-1", - "parent_id": None, - "filename": "API", - "is_folder": True, - "created_at": "2026-03-26T00:00:00Z", - }, - "file-pdf": { - "file_id": "file-pdf", - "database_id": "db-1", - "parent_id": "folder-api", - "filename": "auth-guide.pdf", - "path": "http://minio/kb-source/db-1/upload/auth-guide.pdf", - "markdown_file": "http://minio/kb-parsed/db-1/parsed/file-pdf.md", - "size": 12, - "is_folder": False, - "created_at": "2026-03-26T00:00:00Z", - }, - "file-pdf-2": { - "file_id": "file-pdf-2", - "database_id": "db-1", - "parent_id": "folder-api", - "filename": "auth-guide.pdf", - "path": "http://minio/kb-source/db-1/upload/auth-guide-2.pdf", - "size": 7, - "is_folder": False, - "created_at": "2026-03-26T00:00:00Z", - }, - "file-db2": { - "file_id": "file-db2", - "database_id": "db-2", - "parent_id": None, - "filename": "overview.txt", - "path": "http://minio/kb-source/db-2/upload/overview.txt", - "size": 5, - "is_folder": False, - "created_at": "2026-03-26T00:00:00Z", - }, - } - monkeypatch.setattr("yuxi.agents.backends.knowledge_base_backend._all_files_meta", lambda: files_meta) - - class _FakeMinio: - def download_file(self, bucket_name: str, object_name: str) -> bytes: - return f"{bucket_name}:{object_name}".encode() - - monkeypatch.setattr("yuxi.agents.backends.knowledge_base_backend.get_minio_client", lambda: _FakeMinio()) - - backend = KnowledgeBaseReadonlyBackend(visible_kbs=visible_kbs, cache_root=tmp_path) - - root_entries = {entry["path"] for entry in backend.ls_info("/")} - assert "/FAQ/" in root_entries - assert "/FAQ__db-2/" in root_entries - - faq_entries = {entry["path"] for entry in backend.ls_info("/FAQ")} - assert "/FAQ/API/" in faq_entries - assert "/FAQ/parsed/" in faq_entries - - api_entries = {entry["path"] for entry in backend.ls_info("/FAQ/API")} - assert "/FAQ/API/auth-guide.pdf" in api_entries - assert any(path.startswith("/FAQ/API/auth-guide.pdf__file-pdf") for path in api_entries) - - parsed_entries = {entry["path"] for entry in backend.ls_info("/FAQ/parsed/API")} - assert "/FAQ/parsed/API/auth-guide.pdf.md" in parsed_entries - - responses = backend.download_files(["/FAQ/API/auth-guide.pdf", "/FAQ/parsed/API/auth-guide.pdf.md"]) - assert responses[0].content == b"kb-source:db-1/upload/auth-guide.pdf" - assert responses[1].content == b"kb-parsed:db-1/parsed/file-pdf.md" - - assert "kb-source" in backend.read("/FAQ/API/auth-guide.pdf") - assert backend.write("/FAQ/API/new.txt", "x").error == "Knowledge base path is read-only." +from yuxi.agents.backends.knowledge_base_backend import resolve_visible_knowledge_bases_for_context @pytest.mark.asyncio @@ -110,216 +24,41 @@ async def test_resolve_visible_knowledge_bases_for_context_filters_by_enabled_na assert getattr(context, "_visible_knowledge_bases") == visible -def test_build_knowledge_base_filepath_map_matches_virtual_tree(monkeypatch, tmp_path) -> None: - visible_kbs = [ - {"db_id": "db-1", "name": "FAQ"}, - {"db_id": "db-2", "name": "FAQ"}, - ] - files_meta = { - "folder-api": { - "file_id": "folder-api", - "database_id": "db-1", - "parent_id": None, - "filename": "API", - "is_folder": True, - "created_at": "2026-03-26T00:00:00Z", - }, - "file-pdf": { - "file_id": "file-pdf", - "database_id": "db-1", - "parent_id": "folder-api", - "filename": "auth-guide.pdf", - "path": "http://minio/kb-source/db-1/upload/auth-guide.pdf", - "size": 12, - "is_folder": False, - "created_at": "2026-03-26T00:00:00Z", - }, - "file-pdf-2": { - "file_id": "file-pdf-2", - "database_id": "db-1", - "parent_id": "folder-api", - "filename": "auth-guide.pdf", - "path": "http://minio/kb-source/db-1/upload/auth-guide-2.pdf", - "size": 7, - "is_folder": False, - "created_at": "2026-03-26T00:00:00Z", - }, - "file-db2": { - "file_id": "file-db2", - "database_id": "db-2", - "parent_id": None, - "filename": "overview.txt", - "path": "http://minio/kb-source/db-2/upload/overview.txt", - "size": 5, - "is_folder": False, - "created_at": "2026-03-26T00:00:00Z", - }, - } - monkeypatch.setattr("yuxi.agents.backends.knowledge_base_backend._all_files_meta", lambda: files_meta) - - filepath_map = build_knowledge_base_filepath_map(visible_kbs=visible_kbs, files_meta=files_meta) - backend = KnowledgeBaseReadonlyBackend(visible_kbs=visible_kbs, cache_root=tmp_path) - - assert filepath_map["file-pdf"] == f"{KBS_PATH}/FAQ/API/auth-guide.pdf" - assert filepath_map["file-db2"] == f"{KBS_PATH}/FAQ__db-2/overview.txt" - assert filepath_map["file-pdf-2"].startswith(f"{KBS_PATH}/FAQ/API/auth-guide.pdf__") - - mapped_virtual_paths = {path[len(KBS_PATH) :] for path in filepath_map.values()} - for virtual_path in mapped_virtual_paths: - assert virtual_path in backend._files - - -def test_resolve_file_relative_virtual_path_by_file_id(monkeypatch) -> None: - visible_kbs = [{"db_id": "db-1", "name": "食品 相关文献"}] - files_meta = { - "folder-1": { - "file_id": "folder-1", - "database_id": "db-1", - "parent_id": None, - "filename": "课程", - "is_folder": True, - "created_at": "2026-03-26T00:00:00Z", - }, - "file_79c496": { - "file_id": "file_79c496", - "database_id": "db-1", - "parent_id": "folder-1", - "filename": "营养与食品课程中的思政建设研究.pdf", - "path": "http://minio/knowledgebases/db-1/uploads/file_79c496.pdf", - "is_folder": False, - "size": 100, - "created_at": "2026-03-26T00:00:00Z", - }, - } - monkeypatch.setattr("yuxi.agents.backends.knowledge_base_backend._all_files_meta", lambda: files_meta) - - relative_path = resolve_file_relative_virtual_path(file_id="file_79c496", visible_kbs=visible_kbs) - absolute_map = build_knowledge_base_filepath_map(visible_kbs=visible_kbs, files_meta=files_meta) - - assert relative_path == "/食品 相关文献/课程/营养与食品课程中的思政建设研究.pdf" - assert absolute_map["file_79c496"] == f"{KBS_PATH}{relative_path}" - - @pytest.mark.asyncio -async def test_inject_filepaths_into_retrieval_result_injects_by_file_id(monkeypatch) -> None: - retrieval_chunks = [ - { - "content": "auth guide", - "metadata": { - "file_id": "file-1", - "source": "auth-guide.pdf", - }, - } - ] +async def test_resolve_visible_knowledge_bases_for_context_defaults_to_all_accessible(monkeypatch) -> None: + databases = [{"db_id": "db-1", "name": "Alpha"}, {"db_id": "db-2", "name": "Beta"}] + + async def _fake_get_databases_by_raw_id(user_id: int) -> dict: + assert user_id == 7 + return {"databases": databases} monkeypatch.setattr( - "yuxi.agents.backends.knowledge_base_backend._resolve_virtual_layout", - lambda **kwargs: SimpleNamespace( - source_filepaths={"file-1": f"{KBS_PATH}/FAQ/auth-guide.pdf"}, - parsed_filepaths={"file-1": f"{KBS_PATH}/FAQ/parsed/auth-guide.pdf.md"}, - ), + "yuxi.agents.backends.knowledge_base_backend.knowledge_base.get_databases_by_raw_id", + _fake_get_databases_by_raw_id, ) - result = await inject_filepaths_into_retrieval_result( - retrieval_chunks=retrieval_chunks, - visible_kbs=[{"db_id": "db-1", "name": "FAQ"}], - target_db_id="db-1", - ) + context = SimpleNamespace(user_id="7", knowledges=None) + visible = await resolve_visible_knowledge_bases_for_context(context) - assert result[0]["metadata"]["filepath"] == f"{KBS_PATH}/FAQ/auth-guide.pdf" - assert result[0]["metadata"]["parsed_path"] == f"{KBS_PATH}/FAQ/parsed/auth-guide.pdf.md" + assert visible == databases + assert getattr(context, "_visible_knowledge_bases") == databases @pytest.mark.asyncio -async def test_inject_filepaths_into_retrieval_result_injects_parsed_path_when_markdown_exists(monkeypatch) -> None: - files_meta = { - "file-1": { - "file_id": "file-1", - "database_id": "db-1", - "parent_id": None, - "filename": "auth-guide.pdf", - "path": "http://minio/kb-source/db-1/upload/auth-guide.pdf", - "markdown_file": "http://minio/kb-parsed/db-1/parsed/file-1.md", - "is_folder": False, - "created_at": "2026-03-26T00:00:00Z", - } - } - monkeypatch.setattr("yuxi.agents.backends.knowledge_base_backend._all_files_meta", lambda: files_meta) +async def test_resolve_visible_knowledge_bases_for_context_handles_missing_user() -> None: + context = SimpleNamespace(user_id="", knowledges=None) - retrieval_chunks = [{"content": "auth guide", "metadata": {"file_id": "file-1"}}] - result = await inject_filepaths_into_retrieval_result( - retrieval_chunks=retrieval_chunks, - visible_kbs=[{"db_id": "db-1", "name": "FAQ"}], - target_db_id="db-1", - ) + visible = await resolve_visible_knowledge_bases_for_context(context) - assert result[0]["metadata"]["filepath"] == f"{KBS_PATH}/FAQ/auth-guide.pdf" - assert result[0]["metadata"]["parsed_path"] == f"{KBS_PATH}/FAQ/parsed/auth-guide.pdf.md" + assert visible == [] + assert getattr(context, "_visible_knowledge_bases") == [] @pytest.mark.asyncio -async def test_inject_filepaths_into_retrieval_result_does_not_use_filename_fallback(monkeypatch) -> None: - retrieval_chunks = [ - { - "id": "chunk-1", - "metadata": { - "source": "auth-guide.pdf", - }, - } - ] +async def test_resolve_visible_knowledge_bases_for_context_handles_invalid_user() -> None: + context = SimpleNamespace(user_id="not-a-number", knowledges=None) - monkeypatch.setattr( - "yuxi.agents.backends.knowledge_base_backend._resolve_virtual_layout", - lambda **kwargs: SimpleNamespace( - source_filepaths={ - "file-1": f"{KBS_PATH}/FAQ/API/auth-guide.pdf", - "file-2": f"{KBS_PATH}/FAQ/API/another.pdf", - }, - parsed_filepaths={ - "file-1": f"{KBS_PATH}/FAQ/parsed/API/auth-guide.pdf.md", - "file-2": f"{KBS_PATH}/FAQ/parsed/API/another.pdf.md", - }, - ), - ) + visible = await resolve_visible_knowledge_bases_for_context(context) - result = await inject_filepaths_into_retrieval_result( - retrieval_chunks=retrieval_chunks, - visible_kbs=[{"db_id": "db-1", "name": "FAQ"}], - target_db_id="db-1", - ) - - assert "filepath" not in result[0]["metadata"] - assert "parsed_path" not in result[0]["metadata"] - - -@pytest.mark.asyncio -async def test_inject_filepaths_into_retrieval_result_requires_explicit_file_id(monkeypatch) -> None: - retrieval_chunks = [ - { - "content": "chunk", - "metadata": { - "source": "http://172.19.13.5:9000/knowledgebases/kb-1/parsed/file_79c496.md", - }, - } - ] - - monkeypatch.setattr( - "yuxi.agents.backends.knowledge_base_backend._resolve_virtual_layout", - lambda **kwargs: SimpleNamespace( - source_filepaths={ - "file_79c496": f"{KBS_PATH}/食品 相关文献/课程/营养与食品课程中的思政建设研究.pdf", - }, - parsed_filepaths={ - "file_79c496": f"{KBS_PATH}/食品 相关文献/parsed/课程/营养与食品课程中的思政建设研究.pdf.md", - }, - ), - ) - - result = await inject_filepaths_into_retrieval_result( - retrieval_chunks=retrieval_chunks, - visible_kbs=[{"db_id": "db-1", "name": "食品 相关文献"}], - target_db_id="db-1", - ) - - assert "filepath" not in result[0]["metadata"] - assert "parsed_path" not in result[0]["metadata"] + assert visible == [] + assert getattr(context, "_visible_knowledge_bases") == [] diff --git a/backend/test/unit/backends/test_sandbox_backends.py b/backend/test/unit/backends/test_sandbox_backends.py index 893ea918..afc2906b 100644 --- a/backend/test/unit/backends/test_sandbox_backends.py +++ b/backend/test/unit/backends/test_sandbox_backends.py @@ -39,7 +39,7 @@ def test_create_agent_composite_backend_uses_provisioner_default(monkeypatch): assert isinstance(backend.default, ProvisionerSandboxBackend) assert backend.default._visible_skills == ["reporter"] assert "/skills/" in backend.routes - assert "/home/gem/kbs/" in backend.routes + assert "/home/gem/kbs/" not in backend.routes def test_create_agent_composite_backend_requires_thread_id(): diff --git a/backend/test/unit/toolkits/test_kbs_tools.py b/backend/test/unit/toolkits/test_kbs_tools.py index b6851063..e4dbdb5d 100644 --- a/backend/test/unit/toolkits/test_kbs_tools.py +++ b/backend/test/unit/toolkits/test_kbs_tools.py @@ -62,7 +62,7 @@ def _build_test_window(content: str, offset: int = 0, limit: int = 800) -> dict: @pytest.mark.asyncio -async def test_query_kb_injects_filepath_into_chunk_metadata(monkeypatch) -> None: +async def test_query_kb_returns_milvus_chunks_without_sandbox_paths(monkeypatch) -> None: async def _fake_retriever(query_text: str, **kwargs): assert query_text == "auth" return [ @@ -92,23 +92,20 @@ async def test_query_kb_injects_filepath_into_chunk_metadata(monkeypatch) -> Non monkeypatch.setattr(tools, "_resolve_visible_knowledge_bases_for_query", _fake_visible_kbs) - async def _fake_inject(*, retrieval_chunks, visible_kbs, target_db_id, target_kb_name=None): - assert visible_kbs == [{"db_id": "db-1", "name": "FAQ"}] - assert target_db_id == "db-1" - retrieval_chunks[0]["metadata"]["filepath"] = "/home/gem/kbs/FAQ/API/auth-guide.pdf" - retrieval_chunks[0]["metadata"]["parsed_path"] = "/home/gem/kbs/FAQ/parsed/API/auth-guide.pdf.md" - return retrieval_chunks - - monkeypatch.setattr( - "yuxi.agents.backends.knowledge_base_backend.inject_filepaths_into_retrieval_result", - _fake_inject, - ) - runtime = SimpleNamespace(context=SimpleNamespace()) result = await _run_query_kb(kb_name="FAQ", query_text="auth", runtime=runtime) - assert result[0]["metadata"]["filepath"] == "/home/gem/kbs/FAQ/API/auth-guide.pdf" - assert result[0]["metadata"]["parsed_path"] == "/home/gem/kbs/FAQ/parsed/API/auth-guide.pdf.md" + assert result == [ + { + "content": "auth guide", + "metadata": { + "file_id": "file-1", + "source": "auth-guide.pdf", + }, + } + ] + assert "filepath" not in result[0]["metadata"] + assert "parsed_path" not in result[0]["metadata"] @pytest.mark.asyncio @@ -142,10 +139,6 @@ async def test_query_kb_allows_dify_knowledge_base(monkeypatch) -> None: return [{"db_id": "db-1", "name": "FAQ"}] monkeypatch.setattr(tools, "_resolve_visible_knowledge_bases_for_query", _fake_visible_kbs) - monkeypatch.setattr( - "yuxi.agents.backends.knowledge_base_backend.inject_filepaths_into_retrieval_result", - pytest.fail, - ) runtime = SimpleNamespace(context=SimpleNamespace()) result = await _run_query_kb(kb_name="FAQ", query_text="auth", runtime=runtime) @@ -163,7 +156,7 @@ async def test_query_kb_allows_dify_knowledge_base(monkeypatch) -> None: @pytest.mark.asyncio -async def test_query_kb_returns_lightrag_result_without_filepath_injection(monkeypatch) -> None: +async def test_query_kb_returns_lightrag_result_without_path_injection(monkeypatch) -> None: async def _fake_retriever(query_text: str, **kwargs): assert query_text == "auth" return "LightRAG context" @@ -184,10 +177,6 @@ async def test_query_kb_returns_lightrag_result_without_filepath_injection(monke return [{"db_id": "db-1", "name": "FAQ"}] monkeypatch.setattr(tools, "_resolve_visible_knowledge_bases_for_query", _fake_visible_kbs) - monkeypatch.setattr( - "yuxi.agents.backends.knowledge_base_backend.inject_filepaths_into_retrieval_result", - pytest.fail, - ) runtime = SimpleNamespace(context=SimpleNamespace()) result = await _run_query_kb(kb_name="FAQ", query_text="auth", runtime=runtime) @@ -195,55 +184,6 @@ async def test_query_kb_returns_lightrag_result_without_filepath_injection(monke assert result == "LightRAG context" -@pytest.mark.asyncio -async def test_query_kb_uses_backend_filepath_injector(monkeypatch) -> None: - async def _fake_retriever(query_text: str, **kwargs): - assert query_text == "auth" - return [ - { - "content": "auth guide", - "metadata": { - "file_id": "file-1", - "source": "auth-guide.pdf", - }, - } - ] - - monkeypatch.setattr( - tools.knowledge_base, - "get_retrievers", - lambda: { - "db-1": { - "name": "FAQ", - "retriever": _fake_retriever, - "metadata": {"kb_type": "milvus"}, - } - }, - ) - - async def _fake_visible_kbs(runtime): - return [{"db_id": "db-1", "name": "FAQ"}] - - async def _fake_inject(*, retrieval_chunks, visible_kbs, target_db_id, target_kb_name=None): - assert visible_kbs == [{"db_id": "db-1", "name": "FAQ"}] - assert target_db_id == "db-1" - retrieval_chunks[0]["metadata"]["filepath"] = "/home/gem/kbs/FAQ/auth-guide.pdf" - retrieval_chunks[0]["metadata"]["parsed_path"] = "/home/gem/kbs/FAQ/parsed/auth-guide.pdf.md" - return retrieval_chunks - - monkeypatch.setattr(tools, "_resolve_visible_knowledge_bases_for_query", _fake_visible_kbs) - monkeypatch.setattr( - "yuxi.agents.backends.knowledge_base_backend.inject_filepaths_into_retrieval_result", - _fake_inject, - ) - - runtime = SimpleNamespace(context=SimpleNamespace()) - result = await _run_query_kb(kb_name="FAQ", query_text="auth", runtime=runtime) - - assert result[0]["metadata"]["filepath"] == "/home/gem/kbs/FAQ/auth-guide.pdf" - assert result[0]["metadata"]["parsed_path"] == "/home/gem/kbs/FAQ/parsed/auth-guide.pdf.md" - - @pytest.mark.asyncio async def test_query_kb_normalizes_file_metadata_for_open(monkeypatch) -> None: async def _fake_retriever(query_text: str, **kwargs): diff --git a/docs/agents/sandbox-architecture.md b/docs/agents/sandbox-architecture.md index 91d1c4af..3cae430f 100644 --- a/docs/agents/sandbox-architecture.md +++ b/docs/agents/sandbox-architecture.md @@ -6,9 +6,9 @@ 我们在 Yuxi 里引入沙盒，不是为了让架构更“重”，而是因为 Agent 一旦从纯文本对话进入真实执行阶段，就一定会碰到一组很具体的运行时需求：执行命令、读写文件、处理用户上传附件、产出可下载结果，以及在受控目录里保留中间过程文件。如果把这些能力直接放进 API 进程本身，权限边界、租户隔离、环境一致性和后续运维成本都会迅速恶化。 -从设计目标上看，沙盒这一层主要解决三件事。第一，给 Agent 一个可写、可执行、可回收的独立运行空间，而不是让它直接操作应用主进程。第二，把模型可见文件系统整理成稳定的命名空间，例如 `/home/gem/user-data`、`/home/gem/skills`、`/home/gem/kbs`，这样 prompt、工具、viewer 和 artifact 下载接口可以共享同一套路径语义。第三，让这套能力既能在本地 Docker 开发环境里稳定工作，也能在需要时切到 Kubernetes 这类更适合多实例部署的承载方式。 +从设计目标上看，沙盒这一层主要解决三件事。第一，给 Agent 一个可写、可执行、可回收的独立运行空间，而不是让它直接操作应用主进程。第二，把模型可见文件系统整理成稳定的命名空间，例如 `/home/gem/user-data` 和 `/home/gem/skills`，这样 prompt、工具、viewer 和 artifact 下载接口可以共享同一套路径语义。第三，让这套能力既能在本地 Docker 开发环境里稳定工作，也能在需要时切到 Kubernetes 这类更适合多实例部署的承载方式。 -这份文档说明当前项目中“沙盒”这一层到底是什么、为什么同时会看到 Docker 和 Kubernetes、默认开发环境实际启用的是哪一种模式，以及沙盒如何和 `skills`、知识库、附件、工作区文件系统组合在一起工作。内容以当前仓库实现为准，我们重点解释真实调用链、配置入口、路径语义和运维边界，而不是抽象地介绍容器技术。 +这份文档说明当前项目中“沙盒”这一层到底是什么、为什么同时会看到 Docker 和 Kubernetes、默认开发环境实际启用的是哪一种模式，以及沙盒如何和 `skills`、附件、工作区文件系统组合在一起工作。内容以当前仓库实现为准，我们重点解释真实调用链、配置入口、路径语义和运维边界，而不是抽象地介绍容器技术。 ## 一、先说明白：Docker 和 K8s 在这里是什么关系 @@ -114,7 +114,7 @@ services: ## 八、当前项目的沙盒文件系统是如何设计的 -从模型和工具调用的视角看，Yuxi 主要向 Agent 暴露三类路径：`/home/gem/user-data`、`/home/gem/skills` 和 `/home/gem/kbs`。其中 `user-data` 是可写的用户工作区，`skills` 是只读的技能目录，`kbs` 是只读的知识库映射目录。 +从模型和工具调用的视角看，Yuxi 主要向 Agent 暴露两类路径：`/home/gem/user-data` 和 `/home/gem/skills`。其中 `user-data` 是可写的用户工作区，`skills` 是只读的技能目录。知识库不再映射为沙盒文件系统路径，模型应通过知识库工具检索和打开文档。 在宿主机侧，和线程相关的数据主要放在 `saves` 目录下。当前可读的目录结构可以概括为下面这样： @@ -149,7 +149,7 @@ Yuxi 不会把整个容器文件系统都开放给 Agent 或 viewer。当前 vie `/home/gem/skills` 是只读目录。它不是简单地把 `saves/skills` 整个暴露进去，而是先根据当前线程可见的 skill 列表，把这些技能从全局 skills 根目录同步复制到 `saves/threads//skills`，再把这个线程目录只读挂进沙盒。这样做的结果是，不同线程看到的 skill 集可能不同，而且模型永远不能在运行时修改 skills 内容。 -`/home/gem/kbs` 也是只读目录。它不是物理直挂一个宿主机目录，而是由 `KnowledgeBaseReadonlyBackend` 动态组织出来的一棵虚拟树。这个树只暴露“当前用户可访问的知识库”和“当前 Agent 上启用的知识库”的交集，并且会同时组织源文件和解析后的 Markdown 视图。对于模型来说，这个目录更像一个只读文件系统投影，而不是原始磁盘路径。 +知识库访问不属于沙盒文件系统暴露规则。当前 Agent 可见知识库仍由用户权限和 Agent 配置共同决定，但只通过 `query_kb`、`open_kb_document` 等工具访问，不提供沙盒目录投影。 ## 十、skills、知识库、附件是怎么和沙盒结合的 @@ -157,7 +157,7 @@ skills 的结合方式分成两层。第一层是提示词层，`SkillsMiddlewar 附件的结合方式更偏向“先落盘，再把路径告诉模型”。用户上传文件后，系统会先把原始文件写入 `saves/threads//user-data/uploads`。如果该文件可以被解析，系统还会额外生成一个 Markdown 副本，写到 `saves/threads//user-data/uploads/attachments/.md`。随后，LangGraph state 中会维护一份 `uploads` 列表，`AttachmentMiddleware` 会把这些可读路径注入系统提示，告诉模型优先用 `read_file` 去读取这些路径。因此，附件并不是“作为消息大段内联塞给模型”，而是被转换成沙盒文件系统中的路径对象。 -知识库则是另一种只读投影。它不会被复制到每个线程目录，而是按当前运行上下文动态生成 `/home/gem/kbs` 虚拟树。模型既可以通过专门的知识库工具检索，也可以在某些需要高精度定位原始内容的场景下直接遍历 `/home/gem/kbs//...`。内置 prompt 里已经明确提到，解析后的 Markdown 通常位于 `parsed` 视图下，这样模型在工具检索不足时还有一个明确的文件系统后备路径。 +知识库不再与沙盒文件系统结合。它不会被复制到每个线程目录，也不会生成虚拟目录；模型通过专门的知识库工具检索，并在需要更完整上下文时用 `open_kb_document` 按 `resource_id` 和 `file_id` 打开文档内容。 ## 十一、当前推荐如何使用 Docker 沙盒 @@ -186,9 +186,9 @@ curl http://localhost:8002/health ## 十二、如何理解文件管理与暴露边界 -从产品行为上看，viewer 文件系统和 artifact 下载接口优先走的是宿主机路径解析，而不是无条件透传到沙盒容器内部。这么设计有两个直接收益。第一，浏览 `/` 或 `/home/gem/user-data` 这样的树形入口时，不需要为了只读查看而冷启动沙盒。第二，权限边界更好做，因为 `resolve_virtual_path` 会把用户可见路径严格限制在预定义的 `user-data`、`skills`、`kbs` 命名空间内。 +从产品行为上看，viewer 文件系统和 artifact 下载接口优先走的是宿主机路径解析，而不是无条件透传到沙盒容器内部。这么设计有两个直接收益。第一，浏览 `/` 或 `/home/gem/user-data` 这样的树形入口时，不需要为了只读查看而冷启动沙盒。第二，权限边界更好做，因为 `resolve_virtual_path` 会把用户可见路径严格限制在预定义的 `user-data` 和 `skills` 命名空间内。 -从工程上看，当前实现更像“双层文件系统”。对 Agent 执行来说，真正工作的对象是远程沙盒进程暴露的文件 API；对 viewer、附件下载和一部分 artifact 查看来说，系统会优先在宿主机侧解析虚拟路径，再用本地文件读取或只读 backend 下载内容。这也是为什么你会看到既有 `ProvisionerSandboxBackend`，又有 `viewer_filesystem_service`、`SelectedSkillsReadonlyBackend`、`KnowledgeBaseReadonlyBackend` 这样的配套实现。 +从工程上看，当前实现更像“双层文件系统”。对 Agent 执行来说，真正工作的对象是远程沙盒进程暴露的文件 API；对 viewer、附件下载和一部分 artifact 查看来说，系统会优先在宿主机侧解析虚拟路径，再用本地文件读取或只读 backend 下载内容。这也是为什么你会看到既有 `ProvisionerSandboxBackend`，又有 `viewer_filesystem_service`、`SelectedSkillsReadonlyBackend` 这样的配套实现。 ## 十三、环境变量配置与传递链 @@ -302,4 +302,4 @@ CHECK_YUXI_SANDBOX_ENV_EXISTS=True 如果怀疑是 Docker 地址不可达，重点检查 `SANDBOX_DOCKER_SANDBOX_HOST` 和随机映射端口是否从 `api` 容器可访问。可以在 `api` 容器内直接 `curl` provisioner 返回的 `sandbox_url`。如果怀疑是 Kubernetes 地址不可达，重点检查 `NODE_HOST` 和 NodePort 的外部连通性，因为当前实现并不是通过集群内部 Service 名称回连。 -如果怀疑是文件看得到但模型读不到，或者模型写了但 viewer 看不到，优先把问题拆成两层：一层是宿主机路径是否存在于 `saves/...` 下，另一层是该路径是否真的被当前线程沙盒挂载并暴露到了 `/home/gem/user-data`、`/home/gem/skills` 或 `/home/gem/kbs`。只要先分清“宿主机侧文件语义”和“沙盒侧运行时挂载语义”，定位问题通常会快很多。 +如果怀疑是文件看得到但模型读不到，或者模型写了但 viewer 看不到，优先把问题拆成两层：一层是宿主机路径是否存在于 `saves/...` 下，另一层是该路径是否真的被当前线程沙盒挂载并暴露到了 `/home/gem/user-data` 或 `/home/gem/skills`。只要先分清“宿主机侧文件语义”和“沙盒侧运行时挂载语义”，定位问题通常会快很多。 diff --git a/docs/develop-guides/changelog.md b/docs/develop-guides/changelog.md index d1bd0c01..f772ab12 100644 --- a/docs/develop-guides/changelog.md +++ b/docs/develop-guides/changelog.md @@ -31,6 +31,7 @@ - 调整 Milvus 混合检索实现：集合 schema 增加 BM25 稀疏向量字段、BM25 函数和中文 analyzer 配置 - 重构 MCP 运行时配置加载模型：移除 `MCP_SERVERS` 作为运行正确性前提的设计，改为每次直接从数据库读取最新 MCP 配置 - 为知识库检索工具补充 `metadata.filepath` 注入：在 `query_kb` 统一出口基于会话可见知识库构建 `file_id -> /home/gem/kbs/...` 映射并回填 Milvus 检索结果 +- 移除知识库沙盒文件系统映射：Agent 不再通过 `/home/gem/kbs` 遍历知识库文件，继续通过 `query_kb` 和 `open_kb_document` 检索与打开文档。 ## v0.6.0 (2026-04-01) diff --git a/docs/develop-guides/roadmap.md b/docs/develop-guides/roadmap.md index 6c152c5e..bf5a9d73 100644 --- a/docs/develop-guides/roadmap.md +++ b/docs/develop-guides/roadmap.md @@ -18,8 +18,7 @@ - 拓宽检索的知识源，统一多知识源（channel），目前已知知识库/知识图谱/网页，可拓展：个人知识库、数据库、历史对话等 - 前置任务，多知识库并行检索（扩展 query_kb） - 新增 query_keywords 工具，专门用于基于关键词命中的排序，也结合词频（和 BM25 的区别？） -- 调研将当前知识库映射为虚拟文件系统工具的可行性，先明确文件树映射、权限边界、内容读取与 Agent 工具调用形态，再决定是否实现 -- 参考 AgenticRAG 方案扩展当前 Search 工具：等待虚拟文件系统构建完成后，改进 Search 返回递增文件序列 ID，新增 Find 与 Open 能力；Summary 暂缓 +- 参考 AgenticRAG 方案扩展当前 Search 工具：基于知识库工具返回的 resource_id/file_id 改进 Search 返回递增文件序列 ID，完善 Find 与 Open 能力；Summary 暂缓 - 评估，基于 Agent 的评估，这里应该是结合 Langfuse 实现 ### Bugs @@ -40,6 +39,7 @@ - 下放扩展管理权限：普通管理员现在可进入扩展管理并完整管理 Tools、MCP、SubAgent、Skills；同步放开 Skill 管理接口权限并补充权限测试。 - 调整 Agent 知识库默认选择：未显式配置知识库时默认启用当前用户可访问的全部知识库，显式保存空列表仍表示不启用知识库。 +- 移除知识库沙盒文件系统映射：不再通过 `/home/gem/kbs` 暴露知识库文件树，Agent 继续使用 `query_kb` 与 `open_kb_document` 访问知识库内容。 - 优化评估基准自动生成：仅支持 commonrag/Milvus 知识库，默认参考 chunks 数量改为 1；多 chunk 场景复用知识库向量检索选择相似 chunks，不再对全量 chunks 重新计算 embedding，并移除前端 Embedding 模型选择。 - 修复知识库文档入库状态回退：当已解析文件缺失 `markdown_file` 解析产物时，索引流程会将文件状态恢复为未解析，便于重新解析而不是停留在索引失败。 - 优化 Agent 输入框文件 mention：用户级 workspace 文件候选改为从独立 workspace API 递归加载，不再依赖 active thread；插入时仍转换为 `/home/gem/user-data/workspace/` 沙盒虚拟路径，并修复附件上传后未立即刷新 mention 候选的问题。 diff --git a/web/src/components/AgentPanel.vue b/web/src/components/AgentPanel.vue index f0e75c6b..3e927dbd 100644 --- a/web/src/components/AgentPanel.vue +++ b/web/src/components/AgentPanel.vue @@ -154,7 +154,7 @@ const props = defineProps({ const emit = defineEmits(['refresh', 'resize', 'resizing']) const INLINE_PREVIEW_MIN_WIDTH = 920 -const DEFAULT_EXPANDED_ROOT_DIRECTORY_NAME = 'user-data' +const DISPLAY_ROOT_DIRECTORY_NAME = 'user-data' const panelRef = ref(null) const modalVisible = ref(false) @@ -303,12 +303,6 @@ const loadDirectoryChildren = async (directoryPath) => { return sortEntries(res?.entries || []).map((entry) => createTreeNode(entry)) } -const findDefaultExpandedRootNode = (nodes) => { - return nodes.find( - (node) => !node.isLeaf && buildDisplayName(node.key) === DEFAULT_EXPANDED_ROOT_DIRECTORY_NAME - ) -} - const refreshFileSystem = async () => { if (!props.threadId) { dynamicTreeData.value = [] @@ -327,21 +321,13 @@ const refreshFileSystem = async () => { props.agentConfigId ) if (res?.entries) { - const rootNodes = sortEntries(res.entries).map((entry) => createTreeNode(entry)) - const defaultExpandedNode = findDefaultExpandedRootNode(rootNodes) + const displayRootEntry = res.entries.find( + (entry) => entry?.is_dir && entry.name === DISPLAY_ROOT_DIRECTORY_NAME + ) - dynamicTreeData.value = rootNodes - expandedKeys.value = defaultExpandedNode ? [defaultExpandedNode.key] : [] + dynamicTreeData.value = displayRootEntry ? await loadDirectoryChildren(displayRootEntry.path) : [] + expandedKeys.value = [] selectedKeys.value = [] - - if (defaultExpandedNode) { - try { - const children = await loadDirectoryChildren(defaultExpandedNode.key) - dynamicTreeData.value = updateTreeChildren(rootNodes, defaultExpandedNode.key, children) - } catch (error) { - console.error('Failed to load default expanded directory', error) - } - } } else { dynamicTreeData.value = [] }