diff --git a/AGENTS.md b/AGENTS.md index 055c53a6..6545c7c2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,6 +13,8 @@ Don't add error handling, fallbacks, or validation for scenarios that can't happ Don't create helpers, utilities, or abstractions for one-time operations. Don't design for hypothetical future requirements. The right amount of complexity is the minimum needed for the current task. Reuse existing abstractions where possible and follow the DRY principle. +To ensure readability, it is necessary to add essential comments at key points, particularly to explain the functionality of a function and the design intent. + ## 开发与调试工作流 (Development & Debugging Workflow) 本项目完全通过 Docker Compose 进行管理。所有开发和调试都应在运行的容器环境中进行。使用 `docker compose up -d` 命令进行构建和启动。 diff --git a/CLAUDE.md b/CLAUDE.md index 055c53a6..6545c7c2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -13,6 +13,8 @@ Don't add error handling, fallbacks, or validation for scenarios that can't happ Don't create helpers, utilities, or abstractions for one-time operations. Don't design for hypothetical future requirements. The right amount of complexity is the minimum needed for the current task. Reuse existing abstractions where possible and follow the DRY principle. +To ensure readability, it is necessary to add essential comments at key points, particularly to explain the functionality of a function and the design intent. + ## 开发与调试工作流 (Development & Debugging Workflow) 本项目完全通过 Docker Compose 进行管理。所有开发和调试都应在运行的容器环境中进行。使用 `docker compose up -d` 命令进行构建和启动。 diff --git a/backend/package/yuxi/agents/backends/__init__.py b/backend/package/yuxi/agents/backends/__init__.py index cdf6730e..d7849076 100644 --- a/backend/package/yuxi/agents/backends/__init__.py +++ b/backend/package/yuxi/agents/backends/__init__.py @@ -1,7 +1,13 @@ from deepagents.backends import CompositeBackend, StateBackend from .composite import create_agent_composite_backend -from .knowledge_base_backend import KBS_PATH, KnowledgeBaseReadonlyBackend, resolve_visible_knowledge_bases_for_context +from .knowledge_base_backend import ( + KBS_PATH, + KnowledgeBaseReadonlyBackend, + build_knowledge_base_filepath_map, + resolve_file_relative_virtual_path, + resolve_visible_knowledge_bases_for_context, +) from .sandbox import ( IDLE_CHECK_INTERVAL, LARGE_TOOL_RESULTS_DIR, @@ -33,6 +39,8 @@ __all__ = [ "CompositeBackend", "KBS_PATH", "KnowledgeBaseReadonlyBackend", + "build_knowledge_base_filepath_map", + "resolve_file_relative_virtual_path", "StateBackend", "SelectedSkillsReadonlyBackend", "create_agent_composite_backend", diff --git a/backend/package/yuxi/agents/backends/knowledge_base_backend.py b/backend/package/yuxi/agents/backends/knowledge_base_backend.py index 72f8eacf..729a942d 100644 --- a/backend/package/yuxi/agents/backends/knowledge_base_backend.py +++ b/backend/package/yuxi/agents/backends/knowledge_base_backend.py @@ -28,6 +28,26 @@ class _MaterializedFile: modified_at: str | None = None +@dataclass(frozen=True) +class _ResolvedVirtualNode: + db_id: str + file_id: str + path: str + parent_path: str + name: str + is_folder: bool + + +@dataclass(frozen=True) +class _KnowledgeBaseVirtualLayout: + kb_virtual_names: dict[str, str] + files_by_db: dict[str, dict[str, dict[str, Any]]] + nodes_by_db: dict[str, list[_ResolvedVirtualNode]] + nodes_by_file_id: dict[str, _ResolvedVirtualNode] + source_filepaths: dict[str, str] + parsed_filepaths: dict[str, str] + + def _normalize_virtual_path(path: str | None) -> str: raw = str(path or "").strip() or "/" normalized = "/" + raw.lstrip("/") @@ -132,6 +152,239 @@ def _all_files_meta() -> dict[str, dict[str, Any]]: return aggregated +def _resolve_kb_virtual_names(visible_kbs: list[dict[str, Any]]) -> dict[str, str]: + kb_name_candidates = { + str(db.get("db_id") or db.get("name") or f"kb-{index}"): _sanitize_segment( + db.get("name"), + str(db.get("db_id") or f"kb-{index}"), + ) + for index, db in enumerate(visible_kbs) + } + used_root_names: set[str] = set() + kb_virtual_names: dict[str, str] = {} + sorted_kbs = sorted( + visible_kbs, + key=lambda item: (str(item.get("name") or ""), str(item.get("db_id") or "")), + ) + for db in sorted_kbs: + db_id = str(db.get("db_id") or "") + if not db_id: + continue + base_name = kb_name_candidates.get(db_id) or db_id + kb_virtual_names[db_id] = _unique_name(base_name, stable_id=db_id, used_names=used_root_names) + return kb_virtual_names + + +def _resolve_db_virtual_nodes( + *, + db_id: str, + kb_root: str, + records: dict[str, dict[str, Any]], +) -> list[_ResolvedVirtualNode]: + # Keep sibling deduplication local to each parent directory so the generated tree is deterministic. + children_by_parent: dict[str | None, list[tuple[str, dict[str, Any]]]] = defaultdict(list) + for file_id, meta in records.items(): + parent_id = str(meta.get("parent_id") or "").strip() or None + children_by_parent[parent_id].append((file_id, meta)) + + resolved_nodes: list[_ResolvedVirtualNode] = [] + + def walk(parent_id: str | None, parent_path: str) -> None: + used_names: set[str] = set() + siblings = children_by_parent.get(parent_id, []) + sorted_siblings = sorted( + siblings, + key=lambda item: (_sanitize_segment(_candidate_name(item[1]), item[0]), item[0]), + ) + for file_id, meta in sorted_siblings: + base_name = _sanitize_segment(_candidate_name(meta), file_id) + unique_name = _unique_name(base_name, stable_id=file_id, used_names=used_names) + child_path = f"{parent_path.rstrip('/')}/{unique_name}" if parent_path != "/" else f"/{unique_name}" + node = _ResolvedVirtualNode( + db_id=db_id, + file_id=file_id, + path=child_path, + parent_path=parent_path, + name=unique_name, + is_folder=bool(meta.get("is_folder")), + ) + resolved_nodes.append(node) + if node.is_folder: + walk(file_id, child_path) + + walk(None, kb_root) + return resolved_nodes + + +def _build_parsed_filepath_map( + *, + nodes_by_file_id: dict[str, _ResolvedVirtualNode], + files_by_db: dict[str, dict[str, dict[str, Any]]], + kb_virtual_names: dict[str, str], +) -> dict[str, str]: + # Parsed files mirror the source tree and only add a fixed `/parsed` segment plus `.md` suffix. + parsed_paths: dict[str, str] = {} + for file_id, node in nodes_by_file_id.items(): + if node.is_folder: + continue + + record = files_by_db.get(node.db_id, {}).get(file_id, {}) + if not str(record.get("markdown_file") or "").strip(): + continue + + kb_root = f"/{kb_virtual_names[node.db_id]}" + parsed_root = f"{kb_root}/parsed" + parsed_parent = ( + f"{parsed_root}{node.parent_path[len(kb_root) :]}" if node.parent_path.startswith(kb_root) else parsed_root + ) + parsed_name = _sanitize_segment(f"{node.name}.md", f"{file_id}.md") + parsed_path = f"{parsed_parent.rstrip('/')}/{parsed_name}" if parsed_parent != "/" else f"/{parsed_name}" + parsed_paths[file_id] = f"{KBS_PATH}{parsed_path}" + + return parsed_paths + + +def _resolve_virtual_layout( + *, + visible_kbs: list[dict[str, Any]] | None, + files_meta: dict[str, dict[str, Any]] | None = None, +) -> _KnowledgeBaseVirtualLayout: + # Single source of truth for the virtual KB tree. + # The readonly backend and query_kb filepath injection must stay fully aligned. + kb_virtual_names = _resolve_kb_virtual_names(list(visible_kbs or [])) + if not kb_virtual_names: + return _KnowledgeBaseVirtualLayout( + kb_virtual_names={}, + files_by_db={}, + nodes_by_db={}, + nodes_by_file_id={}, + source_filepaths={}, + parsed_filepaths={}, + ) + + source_files = _all_files_meta() if files_meta is None else files_meta + files_by_db: dict[str, dict[str, dict[str, Any]]] = defaultdict(dict) + for file_id, meta in source_files.items(): + db_id = str(meta.get("database_id") or "") + if db_id in kb_virtual_names: + files_by_db[db_id][str(file_id)] = meta + + nodes_by_db: dict[str, list[_ResolvedVirtualNode]] = {} + nodes_by_file_id: dict[str, _ResolvedVirtualNode] = {} + source_filepaths: dict[str, str] = {} + + for db_id, kb_virtual_name in kb_virtual_names.items(): + kb_root = f"/{kb_virtual_name}" + records = files_by_db.get(db_id, {}) + nodes = _resolve_db_virtual_nodes(db_id=db_id, kb_root=kb_root, records=records) + nodes_by_db[db_id] = nodes + for node in nodes: + nodes_by_file_id[node.file_id] = node + if not node.is_folder: + source_filepaths[node.file_id] = f"{KBS_PATH}{node.path}" + + return _KnowledgeBaseVirtualLayout( + kb_virtual_names=kb_virtual_names, + files_by_db=dict(files_by_db), + nodes_by_db=nodes_by_db, + nodes_by_file_id=nodes_by_file_id, + source_filepaths=source_filepaths, + parsed_filepaths=_build_parsed_filepath_map( + nodes_by_file_id=nodes_by_file_id, + files_by_db=files_by_db, + kb_virtual_names=kb_virtual_names, + ), + ) + + +def resolve_file_relative_virtual_path( + *, + file_id: str, + visible_kbs: list[dict[str, Any]] | None, + files_meta: dict[str, dict[str, Any]] | None = None, +) -> str | None: + normalized_id = str(file_id or "").strip() + if not normalized_id: + return None + + layout = _resolve_virtual_layout( + visible_kbs=visible_kbs, + files_meta=files_meta, + ) + node = layout.nodes_by_file_id.get(normalized_id) + if node is None or node.is_folder: + return None + return node.path + + +def build_knowledge_base_filepath_map( + *, + visible_kbs: list[dict[str, Any]] | None, + files_meta: dict[str, dict[str, Any]] | None = None, +) -> dict[str, str]: + layout = _resolve_virtual_layout( + visible_kbs=visible_kbs, + files_meta=files_meta, + ) + return layout.source_filepaths.copy() + + +def _inject_kb_filepaths( + chunks: list[dict[str, Any]], + filepath_map: dict[str, str], + parsed_filepath_map: dict[str, str], +) -> list[dict[str, Any]]: + if not filepath_map and not parsed_filepath_map: + return chunks + + for chunk in chunks: + if not isinstance(chunk, dict): + continue + + metadata = chunk.get("metadata") + if metadata is None: + metadata = {} + if not isinstance(metadata, dict): + continue + + file_id = str(metadata.get("file_id") or chunk.get("file_id") or "").strip() + if not file_id: + continue + + if not metadata.get("filepath"): + filepath = filepath_map.get(file_id) + if filepath: + metadata["filepath"] = filepath + + if not metadata.get("parsed_path"): + parsed_path = parsed_filepath_map.get(file_id) + if parsed_path: + metadata["parsed_path"] = parsed_path + + chunk["metadata"] = metadata + + return chunks + + +async def inject_filepaths_into_retrieval_result( + *, + retrieval_chunks: list[dict[str, Any]], + visible_kbs: list[dict[str, Any]] | None, + target_db_id: str | None, + target_kb_name: str | None = None, +) -> list[dict[str, Any]]: + scope_kbs = list(visible_kbs or []) + if not scope_kbs and target_db_id: + scope_kbs = [{"db_id": target_db_id, "name": target_kb_name or target_db_id}] + + layout = _resolve_virtual_layout(visible_kbs=scope_kbs) + return _inject_kb_filepaths( + retrieval_chunks, + layout.source_filepaths, + layout.parsed_filepaths, + ) + + class KnowledgeBaseReadonlyBackend(FilesystemBackend): def __init__(self, *, visible_kbs: list[dict[str, Any]] | None, cache_root: Path | str | None = None): self._cache_root = Path(cache_root or (Path(conf.save_dir) / "knowledge_base_data" / "kb-cache")).resolve() @@ -148,37 +401,13 @@ class KnowledgeBaseReadonlyBackend(FilesystemBackend): return bool(self._visible_kbs) def _build_virtual_tree(self) -> None: - kb_name_candidates = { - str(db.get("db_id") or db.get("name") or f"kb-{index}"): _sanitize_segment( - db.get("name"), - str(db.get("db_id") or f"kb-{index}"), - ) - for index, db in enumerate(self._visible_kbs) - } - used_root_names: set[str] = set() - kb_virtual_names: dict[str, str] = {} - sorted_kbs = sorted( - self._visible_kbs, - key=lambda item: (str(item.get("name") or ""), str(item.get("db_id") or "")), - ) - for db in sorted_kbs: - db_id = str(db.get("db_id") or "") - if not db_id: - continue - base_name = kb_name_candidates.get(db_id) or db_id - kb_virtual_names[db_id] = _unique_name(base_name, stable_id=db_id, used_names=used_root_names) + layout = _resolve_virtual_layout(visible_kbs=self._visible_kbs) - files_by_db: dict[str, dict[str, dict[str, Any]]] = defaultdict(dict) - for file_id, meta in _all_files_meta().items(): - db_id = str(meta.get("database_id") or "") - if db_id in kb_virtual_names: - files_by_db[db_id][str(file_id)] = meta - - for db_id, kb_virtual_name in kb_virtual_names.items(): + for db_id, kb_virtual_name in layout.kb_virtual_names.items(): kb_root = f"/{kb_virtual_name}" self._add_entry("/", kb_root, is_dir=True) - records = files_by_db.get(db_id, {}) - self._build_source_tree(db_id=db_id, kb_root=kb_root, records=records) + records = layout.files_by_db.get(db_id, {}) + self._build_source_tree(db_id=db_id, records=records, nodes=layout.nodes_by_db.get(db_id, [])) self._build_parsed_tree(db_id=db_id, kb_root=kb_root, records=records) for path, entries in list(self._entries_by_dir.items()): @@ -186,62 +415,48 @@ class KnowledgeBaseReadonlyBackend(FilesystemBackend): self._entries_by_dir[path] = entries self._all_files = sorted(self._all_files, key=lambda item: str(item.get("path") or "")) - def _build_source_tree(self, *, db_id: str, kb_root: str, records: dict[str, dict[str, Any]]) -> None: - children_by_parent: dict[str | None, list[dict[str, Any]]] = defaultdict(list) - for file_id, meta in records.items(): - parent_id = meta.get("parent_id") - children_by_parent[str(parent_id) if parent_id else None].append({"file_id": file_id, "meta": meta}) - + def _build_source_tree( + self, + *, + db_id: str, + records: dict[str, dict[str, Any]], + nodes: list[_ResolvedVirtualNode], + ) -> None: resolved_parent_paths: dict[str, str] = {} resolved_source_names: dict[str, str] = {} - def walk(parent_id: str | None, parent_path: str) -> None: - siblings = children_by_parent.get(parent_id, []) - used_names: set[str] = set() - candidates: list[tuple[dict[str, Any], str]] = [] - for item in siblings: - file_id = item["file_id"] - meta = item["meta"] - base_name = _sanitize_segment(_candidate_name(meta), file_id) - candidates.append((item, base_name)) + for node in sorted(nodes, key=lambda item: item.path): + meta = records.get(node.file_id, {}) + modified_at = meta.get("updated_at") or meta.get("created_at") + if node.is_folder: + self._add_entry(node.parent_path, node.path, is_dir=True, modified_at=modified_at) + continue - for item, base_name in sorted(candidates, key=lambda pair: (pair[1], pair[0]["file_id"])): - file_id = item["file_id"] - meta = item["meta"] - unique_name = _unique_name(base_name, stable_id=file_id, used_names=used_names) - child_path = f"{parent_path.rstrip('/')}/{unique_name}" if parent_path != "/" else f"/{unique_name}" - if meta.get("is_folder"): - modified_at = meta.get("updated_at") or meta.get("created_at") - self._add_entry(parent_path, child_path, is_dir=True, modified_at=modified_at) - walk(file_id, child_path) - continue + self._add_entry( + node.parent_path, + node.path, + is_dir=False, + size=int(meta.get("size") or 0), + modified_at=modified_at, + ) + resolved_parent_paths[node.file_id] = node.parent_path + resolved_source_names[node.file_id] = node.name + cache_path = self._cache_root / db_id / "source" / node.file_id / node.name + self._files[node.path] = _MaterializedFile( + virtual_path=node.path, + cache_path=cache_path, + source_path=str(meta.get("path") or ""), + modified_at=modified_at, + ) + self._all_files.append( + { + "path": node.path, + "is_dir": False, + "size": int(meta.get("size") or 0), + "modified_at": str(modified_at or ""), + } + ) - self._add_entry( - parent_path, - child_path, - is_dir=False, - size=int(meta.get("size") or 0), - modified_at=meta.get("updated_at") or meta.get("created_at"), - ) - resolved_parent_paths[file_id] = parent_path - resolved_source_names[file_id] = unique_name - cache_path = self._cache_root / db_id / "source" / file_id / unique_name - self._files[child_path] = _MaterializedFile( - virtual_path=child_path, - cache_path=cache_path, - source_path=str(meta.get("path") or ""), - modified_at=meta.get("updated_at") or meta.get("created_at"), - ) - self._all_files.append( - { - "path": child_path, - "is_dir": False, - "size": int(meta.get("size") or 0), - "modified_at": str(meta.get("updated_at") or meta.get("created_at") or ""), - } - ) - - walk(None, kb_root) self._resolved_parent_paths = getattr(self, "_resolved_parent_paths", {}) self._resolved_parent_paths[db_id] = resolved_parent_paths self._resolved_source_names = getattr(self, "_resolved_source_names", {}) @@ -268,8 +483,8 @@ class KnowledgeBaseReadonlyBackend(FilesystemBackend): f"{parsed_root}{source_parent[len(kb_root) :]}" if source_parent.startswith(kb_root) else parsed_root ) source_name = source_names.get(file_id) or _sanitize_segment(meta.get("filename"), file_id) - source_stem = PurePosixPath(source_name).stem or source_name or file_id - base_name = _sanitize_segment(f"{source_stem}.md", f"{file_id}.md") + safe_name = source_name or file_id + base_name = _sanitize_segment(f"{safe_name}.md", f"{file_id}.md") grouped[parsed_parent].append((file_id, meta, base_name)) for parsed_parent, items in grouped.items(): diff --git a/backend/package/yuxi/agents/toolkits/kbs/tools.py b/backend/package/yuxi/agents/toolkits/kbs/tools.py index 5221d4df..3302cc6c 100644 --- a/backend/package/yuxi/agents/toolkits/kbs/tools.py +++ b/backend/package/yuxi/agents/toolkits/kbs/tools.py @@ -160,6 +160,53 @@ class QueryKBInput(BaseModel): ) +async def _resolve_visible_knowledge_bases_for_query(runtime: ToolRuntime | None) -> list[dict[str, Any]]: + if runtime is None: + return [] + + context = getattr(runtime, "context", None) + if context is None: + return [] + + visible_kbs = getattr(context, "_visible_knowledge_bases", None) + if isinstance(visible_kbs, list): + return visible_kbs + + try: + from yuxi.agents.backends.knowledge_base_backend import resolve_visible_knowledge_bases_for_context + + return await resolve_visible_knowledge_bases_for_context(context) + except Exception as exc: # noqa: BLE001 + logger.warning(f"解析会话可见知识库失败，跳过 filepath 注入: {exc}") + return [] + + +def _find_query_target( + *, + kb_name: str, + retrievers: dict[str, Any], + visible_kbs: list[dict[str, Any]], +) -> tuple[str | None, dict[str, Any] | None, str | None]: + if visible_kbs: + matched_kbs = [db for db in visible_kbs if str(db.get("name") or "").strip() == kb_name] + if not matched_kbs: + return None, None, f"知识库 '{kb_name}' 不存在或当前会话未启用" + if len(matched_kbs) > 1: + return None, None, f"知识库 '{kb_name}' 存在重名，请先调整名称后重试" + + target_db_id = str(matched_kbs[0].get("db_id") or "") + target_info = retrievers.get(target_db_id) + if target_info is None: + return None, None, f"知识库 '{kb_name}' 不存在" + return target_db_id, target_info, None + + for db_id, info in retrievers.items(): + if info["name"] == kb_name: + return str(db_id), info, None + + return None, None, f"知识库 '{kb_name}' 不存在" + + @tool(args_schema=QueryKBInput) async def query_kb(kb_name: str, query_text: str, file_name: str | None = None, runtime: ToolRuntime = None) -> Any: """在指定知识库中检索内容 @@ -182,15 +229,20 @@ async def query_kb(kb_name: str, query_text: str, file_name: str | None = None, # 获取所有检索器 retrievers = knowledge_base.get_retrievers() - # 查找对应的知识库 - target_info = None - for db_id, info in retrievers.items(): - if info["name"] == kb_name: - target_info = info - break + visible_kbs = await _resolve_visible_knowledge_bases_for_query(runtime) - if not target_info: - return f"知识库 '{kb_name}' 不存在" + target_db_id, target_info, target_error = _find_query_target( + kb_name=kb_name, + retrievers=retrievers, + visible_kbs=visible_kbs, + ) + if target_error: + return target_error + + metadata = target_info.get("metadata") if isinstance(target_info, dict) else None + kb_type = str((metadata or {}).get("kb_type") or "").strip().lower() + if kb_type != "milvus": + return f"知识库 '{kb_name}' 不是 Milvus 类型，当前 query_kb 仅支持 Milvus" try: retriever = target_info["retriever"] @@ -203,7 +255,17 @@ async def query_kb(kb_name: str, query_text: str, file_name: str | None = None, else: result = retriever(query_text, **kwargs) - return result + if not isinstance(result, list): + return f"知识库 '{kb_name}' 返回结果不是 Milvus chunks 列表，当前 query_kb 仅支持 Milvus" + + from yuxi.agents.backends.knowledge_base_backend import inject_filepaths_into_retrieval_result + + return await inject_filepaths_into_retrieval_result( + retrieval_chunks=result, + visible_kbs=visible_kbs, + target_db_id=target_db_id, + target_kb_name=kb_name, + ) except Exception as e: logger.error(f"检索失败: {e}") diff --git a/backend/test/unit/backends/test_knowledge_base_backend.py b/backend/test/unit/backends/test_knowledge_base_backend.py index 7af0fe2f..d5c910a7 100644 --- a/backend/test/unit/backends/test_knowledge_base_backend.py +++ b/backend/test/unit/backends/test_knowledge_base_backend.py @@ -4,7 +4,11 @@ from types import SimpleNamespace import pytest from yuxi.agents.backends.knowledge_base_backend import ( + KBS_PATH, KnowledgeBaseReadonlyBackend, + build_knowledge_base_filepath_map, + inject_filepaths_into_retrieval_result, + resolve_file_relative_virtual_path, resolve_visible_knowledge_bases_for_context, ) @@ -78,9 +82,9 @@ def test_knowledge_base_backend_builds_virtual_tree_and_materializes_files(monke assert any(path.startswith("/FAQ/API/auth-guide.pdf__file-pdf") for path in api_entries) parsed_entries = {entry["path"] for entry in backend.ls_info("/FAQ/parsed/API")} - assert "/FAQ/parsed/API/auth-guide.md" in parsed_entries + assert "/FAQ/parsed/API/auth-guide.pdf.md" in parsed_entries - responses = backend.download_files(["/FAQ/API/auth-guide.pdf", "/FAQ/parsed/API/auth-guide.md"]) + responses = backend.download_files(["/FAQ/API/auth-guide.pdf", "/FAQ/parsed/API/auth-guide.pdf.md"]) assert responses[0].content == b"kb-source:db-1/upload/auth-guide.pdf" assert responses[1].content == b"kb-parsed:db-1/parsed/file-pdf.md" @@ -104,3 +108,218 @@ async def test_resolve_visible_knowledge_bases_for_context_filters_by_enabled_na assert visible == [{"db_id": "db-2", "name": "Beta"}] assert getattr(context, "_visible_knowledge_bases") == visible + + +def test_build_knowledge_base_filepath_map_matches_virtual_tree(monkeypatch, tmp_path) -> None: + visible_kbs = [ + {"db_id": "db-1", "name": "FAQ"}, + {"db_id": "db-2", "name": "FAQ"}, + ] + files_meta = { + "folder-api": { + "file_id": "folder-api", + "database_id": "db-1", + "parent_id": None, + "filename": "API", + "is_folder": True, + "created_at": "2026-03-26T00:00:00Z", + }, + "file-pdf": { + "file_id": "file-pdf", + "database_id": "db-1", + "parent_id": "folder-api", + "filename": "auth-guide.pdf", + "path": "http://minio/kb-source/db-1/upload/auth-guide.pdf", + "size": 12, + "is_folder": False, + "created_at": "2026-03-26T00:00:00Z", + }, + "file-pdf-2": { + "file_id": "file-pdf-2", + "database_id": "db-1", + "parent_id": "folder-api", + "filename": "auth-guide.pdf", + "path": "http://minio/kb-source/db-1/upload/auth-guide-2.pdf", + "size": 7, + "is_folder": False, + "created_at": "2026-03-26T00:00:00Z", + }, + "file-db2": { + "file_id": "file-db2", + "database_id": "db-2", + "parent_id": None, + "filename": "overview.txt", + "path": "http://minio/kb-source/db-2/upload/overview.txt", + "size": 5, + "is_folder": False, + "created_at": "2026-03-26T00:00:00Z", + }, + } + monkeypatch.setattr("yuxi.agents.backends.knowledge_base_backend._all_files_meta", lambda: files_meta) + + filepath_map = build_knowledge_base_filepath_map(visible_kbs=visible_kbs, files_meta=files_meta) + backend = KnowledgeBaseReadonlyBackend(visible_kbs=visible_kbs, cache_root=tmp_path) + + assert filepath_map["file-pdf"] == f"{KBS_PATH}/FAQ/API/auth-guide.pdf" + assert filepath_map["file-db2"] == f"{KBS_PATH}/FAQ__db-2/overview.txt" + assert filepath_map["file-pdf-2"].startswith(f"{KBS_PATH}/FAQ/API/auth-guide.pdf__") + + mapped_virtual_paths = {path[len(KBS_PATH) :] for path in filepath_map.values()} + for virtual_path in mapped_virtual_paths: + assert virtual_path in backend._files + + +def test_resolve_file_relative_virtual_path_by_file_id(monkeypatch) -> None: + visible_kbs = [{"db_id": "db-1", "name": "食品 相关文献"}] + files_meta = { + "folder-1": { + "file_id": "folder-1", + "database_id": "db-1", + "parent_id": None, + "filename": "课程", + "is_folder": True, + "created_at": "2026-03-26T00:00:00Z", + }, + "file_79c496": { + "file_id": "file_79c496", + "database_id": "db-1", + "parent_id": "folder-1", + "filename": "营养与食品课程中的思政建设研究.pdf", + "path": "http://minio/knowledgebases/db-1/uploads/file_79c496.pdf", + "is_folder": False, + "size": 100, + "created_at": "2026-03-26T00:00:00Z", + }, + } + monkeypatch.setattr("yuxi.agents.backends.knowledge_base_backend._all_files_meta", lambda: files_meta) + + relative_path = resolve_file_relative_virtual_path(file_id="file_79c496", visible_kbs=visible_kbs) + absolute_map = build_knowledge_base_filepath_map(visible_kbs=visible_kbs, files_meta=files_meta) + + assert relative_path == "/食品 相关文献/课程/营养与食品课程中的思政建设研究.pdf" + assert absolute_map["file_79c496"] == f"{KBS_PATH}{relative_path}" + + +@pytest.mark.asyncio +async def test_inject_filepaths_into_retrieval_result_injects_by_file_id(monkeypatch) -> None: + retrieval_chunks = [ + { + "content": "auth guide", + "metadata": { + "file_id": "file-1", + "source": "auth-guide.pdf", + }, + } + ] + + monkeypatch.setattr( + "yuxi.agents.backends.knowledge_base_backend._resolve_virtual_layout", + lambda **kwargs: SimpleNamespace( + source_filepaths={"file-1": f"{KBS_PATH}/FAQ/auth-guide.pdf"}, + parsed_filepaths={"file-1": f"{KBS_PATH}/FAQ/parsed/auth-guide.pdf.md"}, + ), + ) + + result = await inject_filepaths_into_retrieval_result( + retrieval_chunks=retrieval_chunks, + visible_kbs=[{"db_id": "db-1", "name": "FAQ"}], + target_db_id="db-1", + ) + + assert result[0]["metadata"]["filepath"] == f"{KBS_PATH}/FAQ/auth-guide.pdf" + assert result[0]["metadata"]["parsed_path"] == f"{KBS_PATH}/FAQ/parsed/auth-guide.pdf.md" + + +@pytest.mark.asyncio +async def test_inject_filepaths_into_retrieval_result_injects_parsed_path_when_markdown_exists(monkeypatch) -> None: + files_meta = { + "file-1": { + "file_id": "file-1", + "database_id": "db-1", + "parent_id": None, + "filename": "auth-guide.pdf", + "path": "http://minio/kb-source/db-1/upload/auth-guide.pdf", + "markdown_file": "http://minio/kb-parsed/db-1/parsed/file-1.md", + "is_folder": False, + "created_at": "2026-03-26T00:00:00Z", + } + } + monkeypatch.setattr("yuxi.agents.backends.knowledge_base_backend._all_files_meta", lambda: files_meta) + + retrieval_chunks = [{"content": "auth guide", "metadata": {"file_id": "file-1"}}] + result = await inject_filepaths_into_retrieval_result( + retrieval_chunks=retrieval_chunks, + visible_kbs=[{"db_id": "db-1", "name": "FAQ"}], + target_db_id="db-1", + ) + + assert result[0]["metadata"]["filepath"] == f"{KBS_PATH}/FAQ/auth-guide.pdf" + assert result[0]["metadata"]["parsed_path"] == f"{KBS_PATH}/FAQ/parsed/auth-guide.pdf.md" + + +@pytest.mark.asyncio +async def test_inject_filepaths_into_retrieval_result_does_not_use_filename_fallback(monkeypatch) -> None: + retrieval_chunks = [ + { + "id": "chunk-1", + "metadata": { + "source": "auth-guide.pdf", + }, + } + ] + + monkeypatch.setattr( + "yuxi.agents.backends.knowledge_base_backend._resolve_virtual_layout", + lambda **kwargs: SimpleNamespace( + source_filepaths={ + "file-1": f"{KBS_PATH}/FAQ/API/auth-guide.pdf", + "file-2": f"{KBS_PATH}/FAQ/API/another.pdf", + }, + parsed_filepaths={ + "file-1": f"{KBS_PATH}/FAQ/parsed/API/auth-guide.pdf.md", + "file-2": f"{KBS_PATH}/FAQ/parsed/API/another.pdf.md", + }, + ), + ) + + result = await inject_filepaths_into_retrieval_result( + retrieval_chunks=retrieval_chunks, + visible_kbs=[{"db_id": "db-1", "name": "FAQ"}], + target_db_id="db-1", + ) + + assert "filepath" not in result[0]["metadata"] + assert "parsed_path" not in result[0]["metadata"] + + +@pytest.mark.asyncio +async def test_inject_filepaths_into_retrieval_result_requires_explicit_file_id(monkeypatch) -> None: + retrieval_chunks = [ + { + "content": "chunk", + "metadata": { + "source": "http://172.19.13.5:9000/knowledgebases/kb-1/parsed/file_79c496.md", + }, + } + ] + + monkeypatch.setattr( + "yuxi.agents.backends.knowledge_base_backend._resolve_virtual_layout", + lambda **kwargs: SimpleNamespace( + source_filepaths={ + "file_79c496": f"{KBS_PATH}/食品 相关文献/课程/营养与食品课程中的思政建设研究.pdf", + }, + parsed_filepaths={ + "file_79c496": f"{KBS_PATH}/食品 相关文献/parsed/课程/营养与食品课程中的思政建设研究.pdf.md", + }, + ), + ) + + result = await inject_filepaths_into_retrieval_result( + retrieval_chunks=retrieval_chunks, + visible_kbs=[{"db_id": "db-1", "name": "食品 相关文献"}], + target_db_id="db-1", + ) + + assert "filepath" not in result[0]["metadata"] + assert "parsed_path" not in result[0]["metadata"] diff --git a/backend/test/unit/toolkits/test_kbs_tools.py b/backend/test/unit/toolkits/test_kbs_tools.py new file mode 100644 index 00000000..755df5ad --- /dev/null +++ b/backend/test/unit/toolkits/test_kbs_tools.py @@ -0,0 +1,156 @@ +from __future__ import annotations + +import inspect +from types import SimpleNamespace + +import pytest + +from yuxi.agents.toolkits.kbs import tools + + +def _query_kb_callable(): + callback = getattr(tools.query_kb, "coroutine", None) + if callback is not None: + return callback + + callback = getattr(tools.query_kb, "func", None) + if callback is not None: + return callback + + raise AssertionError("query_kb tool has no callable entry") + + +async def _run_query_kb(**kwargs): + callback = _query_kb_callable() + result = callback(**kwargs) + if inspect.isawaitable(result): + return await result + return result + + +@pytest.mark.asyncio +async def test_query_kb_injects_filepath_into_chunk_metadata(monkeypatch) -> None: + async def _fake_retriever(query_text: str, **kwargs): + assert query_text == "auth" + return [ + { + "content": "auth guide", + "metadata": { + "file_id": "file-1", + "source": "auth-guide.pdf", + }, + } + ] + + monkeypatch.setattr( + tools.knowledge_base, + "get_retrievers", + lambda: { + "db-1": { + "name": "FAQ", + "retriever": _fake_retriever, + "metadata": {"kb_type": "milvus"}, + } + }, + ) + + async def _fake_visible_kbs(runtime): + return [{"db_id": "db-1", "name": "FAQ"}] + + monkeypatch.setattr(tools, "_resolve_visible_knowledge_bases_for_query", _fake_visible_kbs) + + async def _fake_inject(*, retrieval_chunks, visible_kbs, target_db_id, target_kb_name=None): + assert visible_kbs == [{"db_id": "db-1", "name": "FAQ"}] + assert target_db_id == "db-1" + retrieval_chunks[0]["metadata"]["filepath"] = "/home/gem/kbs/FAQ/API/auth-guide.pdf" + retrieval_chunks[0]["metadata"]["parsed_path"] = "/home/gem/kbs/FAQ/parsed/API/auth-guide.pdf.md" + return retrieval_chunks + + monkeypatch.setattr( + "yuxi.agents.backends.knowledge_base_backend.inject_filepaths_into_retrieval_result", + _fake_inject, + ) + + runtime = SimpleNamespace(context=SimpleNamespace()) + result = await _run_query_kb(kb_name="FAQ", query_text="auth", runtime=runtime) + + assert result[0]["metadata"]["filepath"] == "/home/gem/kbs/FAQ/API/auth-guide.pdf" + assert result[0]["metadata"]["parsed_path"] == "/home/gem/kbs/FAQ/parsed/API/auth-guide.pdf.md" + + +@pytest.mark.asyncio +async def test_query_kb_rejects_non_milvus_knowledge_base(monkeypatch) -> None: + async def _fake_retriever(query_text: str, **kwargs): + assert query_text == "auth" + return [] + + monkeypatch.setattr( + tools.knowledge_base, + "get_retrievers", + lambda: { + "db-1": { + "name": "FAQ", + "retriever": _fake_retriever, + "metadata": {"kb_type": "lightrag"}, + } + }, + ) + + async def _fake_visible_kbs(runtime): + return [{"db_id": "db-1", "name": "FAQ"}] + + monkeypatch.setattr(tools, "_resolve_visible_knowledge_bases_for_query", _fake_visible_kbs) + + runtime = SimpleNamespace(context=SimpleNamespace()) + result = await _run_query_kb(kb_name="FAQ", query_text="auth", runtime=runtime) + + assert result == "知识库 'FAQ' 不是 Milvus 类型，当前 query_kb 仅支持 Milvus" + + +@pytest.mark.asyncio +async def test_query_kb_uses_backend_filepath_injector(monkeypatch) -> None: + async def _fake_retriever(query_text: str, **kwargs): + assert query_text == "auth" + return [ + { + "content": "auth guide", + "metadata": { + "file_id": "file-1", + "source": "auth-guide.pdf", + }, + } + ] + + monkeypatch.setattr( + tools.knowledge_base, + "get_retrievers", + lambda: { + "db-1": { + "name": "FAQ", + "retriever": _fake_retriever, + "metadata": {"kb_type": "milvus"}, + } + }, + ) + + async def _fake_visible_kbs(runtime): + return [{"db_id": "db-1", "name": "FAQ"}] + + async def _fake_inject(*, retrieval_chunks, visible_kbs, target_db_id, target_kb_name=None): + assert visible_kbs == [{"db_id": "db-1", "name": "FAQ"}] + assert target_db_id == "db-1" + retrieval_chunks[0]["metadata"]["filepath"] = "/home/gem/kbs/FAQ/auth-guide.pdf" + retrieval_chunks[0]["metadata"]["parsed_path"] = "/home/gem/kbs/FAQ/parsed/auth-guide.pdf.md" + return retrieval_chunks + + monkeypatch.setattr(tools, "_resolve_visible_knowledge_bases_for_query", _fake_visible_kbs) + monkeypatch.setattr( + "yuxi.agents.backends.knowledge_base_backend.inject_filepaths_into_retrieval_result", + _fake_inject, + ) + + runtime = SimpleNamespace(context=SimpleNamespace()) + result = await _run_query_kb(kb_name="FAQ", query_text="auth", runtime=runtime) + + assert result[0]["metadata"]["filepath"] == "/home/gem/kbs/FAQ/auth-guide.pdf" + assert result[0]["metadata"]["parsed_path"] == "/home/gem/kbs/FAQ/parsed/auth-guide.pdf.md" diff --git a/docs/develop-guides/roadmap.md b/docs/develop-guides/roadmap.md index e1d821a6..1023bcff 100644 --- a/docs/develop-guides/roadmap.md +++ b/docs/develop-guides/roadmap.md @@ -40,6 +40,7 @@ - 新增 Skills 远程安装能力：Skills 管理页支持填写 `owner/repo` 或 GitHub URL，后端通过隔离的临时 `HOME` 调用 `npx skills add` 下载指定 skill，再复用现有导入链路写入 `saves/skills` 和数据库，避免将 `~/.agents/skills` 直接作为系统主存储；前端远程安装弹窗补充多选串行安装与批量进度展示，复用现有单 skill 安装接口逐个提交请求 - 调整部门删除语义：删除部门时不再要求用户数为 0，而是将部门下用户迁移到默认部门，同时清理部门级配置和部门 API Key，保证测试部门、撤换部门等场景可直接删除，并补充对应集成测试覆盖该链路 - 重构 MCP 运行时配置加载模型：移除 `MCP_SERVERS` 作为运行正确性前提的设计，改为每次直接从数据库读取最新 MCP 配置，并用 `server_name:config_hash` 作为本地工具缓存 key；同时将内置 MCP 初始化职责收敛为仅同步数据库默认项，前端 MCP 选项改为直接使用实时资源列表，解决 `api`/`worker` 分进程下的配置不一致与缓存失效问题 +- 为知识库检索工具补充 `metadata.filepath` 注入：在 `query_kb` 统一出口基于会话可见知识库构建 `file_id -> /home/gem/kbs/...` 映射并回填检索结果，注入逻辑复用知识库只读后端命名规则；并将工具调用范围收敛为 Milvus（仅支持 Milvus chunks 列表且要求显式 `file_id`），不再兼容无显式 `file_id` 的推断注入，新增单测覆盖该约束 ---