diff --git a/backend/package/yuxi/agents/backends/__init__.py b/backend/package/yuxi/agents/backends/__init__.py index a2a1b42f..c7e3b372 100644 --- a/backend/package/yuxi/agents/backends/__init__.py +++ b/backend/package/yuxi/agents/backends/__init__.py @@ -1,6 +1,7 @@ from deepagents.backends import CompositeBackend, StateBackend from .composite import create_agent_composite_backend +from .knowledge_base_backend import KBS_PATH, KnowledgeBaseReadonlyBackend, resolve_visible_knowledge_bases_for_context from .sandbox import ( IDLE_CHECK_INTERVAL, LARGE_TOOL_RESULTS_DIR, @@ -33,6 +34,8 @@ from .skills_backend import SelectedSkillsReadonlyBackend __all__ = [ "CompositeBackend", + "KBS_PATH", + "KnowledgeBaseReadonlyBackend", "StateBackend", "SelectedSkillsReadonlyBackend", "create_agent_composite_backend", @@ -47,6 +50,7 @@ __all__ = [ "init_sandbox_provider", "shutdown_sandbox_provider", "resolve_virtual_path", + "resolve_visible_knowledge_bases_for_context", "virtual_path_for_thread_file", "sandbox_id_for_thread", "sandbox_user_data_dir", diff --git a/backend/package/yuxi/agents/backends/composite.py b/backend/package/yuxi/agents/backends/composite.py index 13ecc3be..013563f8 100644 --- a/backend/package/yuxi/agents/backends/composite.py +++ b/backend/package/yuxi/agents/backends/composite.py @@ -2,13 +2,15 @@ from __future__ import annotations from deepagents.backends.composite import ( CompositeBackend, - _route_for_path, _remap_file_info_path, + _route_for_path, _strip_route_from_pattern, ) from deepagents.backends.protocol import FileInfo from yuxi.agents.middlewares.skills_middleware import normalize_selected_skills + +from .knowledge_base_backend import KnowledgeBaseReadonlyBackend from .sandbox import ProvisionerSandboxBackend from .skills_backend import SelectedSkillsReadonlyBackend @@ -92,12 +94,22 @@ def _extract_thread_id(runtime) -> str: raise ValueError("thread_id is required in runtime configurable context") +def _get_visible_knowledge_bases_from_runtime(runtime) -> list[dict]: + context = getattr(runtime, "context", None) + selected = getattr(context, "_visible_knowledge_bases", None) + if isinstance(selected, list): + return selected + return [] + + def create_agent_composite_backend(runtime) -> CompositeBackend: visible_skills = _get_visible_skills_from_runtime(runtime) thread_id = _extract_thread_id(runtime) + visible_kbs = _get_visible_knowledge_bases_from_runtime(runtime) return CustomCompositeBackend( default=ProvisionerSandboxBackend(thread_id=thread_id, visible_skills=visible_skills), routes={ "/skills/": SelectedSkillsReadonlyBackend(selected_slugs=visible_skills), + "/home/yuxi/kbs/": KnowledgeBaseReadonlyBackend(visible_kbs=visible_kbs), }, ) diff --git a/backend/package/yuxi/agents/backends/knowledge_base_backend.py b/backend/package/yuxi/agents/backends/knowledge_base_backend.py new file mode 100644 index 00000000..d761cc72 --- /dev/null +++ b/backend/package/yuxi/agents/backends/knowledge_base_backend.py @@ -0,0 +1,459 @@ +from __future__ import annotations + +import fnmatch +import re +from collections import defaultdict +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Any + +from deepagents.backends import FilesystemBackend +from deepagents.backends.protocol import EditResult, FileDownloadResponse, FileInfo, FileUploadResponse, WriteResult + +from yuxi import config as conf +from yuxi import knowledge_base +from yuxi.knowledge.utils.kb_utils import is_minio_url, parse_minio_url +from yuxi.storage.minio import get_minio_client + +KBS_PATH = "/home/yuxi/kbs" +_INVALID_SEGMENT_RE = re.compile(r"[\\/\x00-\x1f\x7f]+") +_WHITESPACE_RE = re.compile(r"\s+") + + +@dataclass(frozen=True) +class _MaterializedFile: + virtual_path: str + cache_path: Path + source_path: str + modified_at: str | None = None + + +def _normalize_virtual_path(path: str | None) -> str: + raw = str(path or "").strip() or "/" + normalized = "/" + raw.lstrip("/") + pure = PurePosixPath(normalized) + if ".." in pure.parts: + raise ValueError("path traversal is not allowed") + return str(pure) + + +def _sanitize_segment(value: str | None, fallback: str) -> str: + cleaned = str(value or "").strip() + cleaned = _INVALID_SEGMENT_RE.sub("_", cleaned) + cleaned = _WHITESPACE_RE.sub(" ", cleaned).strip() + if not cleaned or cleaned in {".", ".."}: + return fallback + return cleaned + + +def _candidate_name(file_meta: dict[str, Any]) -> str: + filename = str(file_meta.get("filename") or "").strip() + if filename: + return filename + + original = str(file_meta.get("original_filename") or "").strip() + if original: + return original + + for key in ("path", "markdown_file"): + raw = str(file_meta.get(key) or "").strip() + if raw: + try: + parsed = PurePosixPath(raw) + name = parsed.name + except Exception: # noqa: BLE001 + name = "" + if name: + return name + + return str(file_meta.get("file_id") or "") + + +def _unique_name(base_name: str, *, stable_id: str, used_names: set[str]) -> str: + if base_name not in used_names: + used_names.add(base_name) + return base_name + + suffix = f"__{stable_id[:8]}" + candidate = f"{base_name}{suffix}" + while candidate in used_names: + suffix = f"__{stable_id}" + candidate = f"{base_name}{suffix}" + used_names.add(candidate) + return candidate + + +def _materialize_text_view(content: bytes, file_path: str, *, offset: int = 0, limit: int = 2000) -> str: + if not content: + return "System reminder: File exists but has empty contents" + + if b"\x00" in content: + return f"Error: File '{file_path}' is binary and cannot be rendered as text" + + try: + text = content.decode("utf-8") + except UnicodeDecodeError: + return f"Error: File '{file_path}' is binary and cannot be rendered as text" + + lines = text.splitlines() + start = max(0, int(offset)) + selected = lines[start : start + int(limit)] + return "\n".join(f"{start + idx + 1:6d}\t{line}" for idx, line in enumerate(selected)) + + +async def resolve_visible_knowledge_bases_for_context(context) -> list[dict[str, Any]]: + user_id = getattr(context, "user_id", None) + if not user_id: + setattr(context, "_visible_knowledge_bases", []) + return [] + + enabled_names = {str(name).strip() for name in (getattr(context, "knowledges", None) or []) if str(name).strip()} + if not enabled_names: + setattr(context, "_visible_knowledge_bases", []) + return [] + + try: + raw_user_id = int(user_id) + except (TypeError, ValueError): + setattr(context, "_visible_knowledge_bases", []) + return [] + + result = await knowledge_base.get_databases_by_raw_id(raw_user_id) + databases = [db for db in (result.get("databases") or []) if str(db.get("name") or "").strip() in enabled_names] + setattr(context, "_visible_knowledge_bases", databases) + return databases + + +def _all_files_meta() -> dict[str, dict[str, Any]]: + aggregated: dict[str, dict[str, Any]] = {} + for kb_instance in getattr(knowledge_base, "kb_instances", {}).values(): + for file_id, meta in getattr(kb_instance, "files_meta", {}).items(): + aggregated[str(file_id)] = meta + return aggregated + + +class KnowledgeBaseReadonlyBackend(FilesystemBackend): + def __init__(self, *, visible_kbs: list[dict[str, Any]] | None, cache_root: Path | str | None = None): + self._cache_root = Path(cache_root or (Path(conf.save_dir) / "knowledge_base_data" / "kb-cache")).resolve() + self._cache_root.mkdir(parents=True, exist_ok=True) + super().__init__(root_dir=self._cache_root, virtual_mode=True) + self._visible_kbs = list(visible_kbs or []) + self._entries_by_dir: dict[str, list[FileInfo]] = defaultdict(list) + self._dir_paths: set[str] = {"/"} + self._files: dict[str, _MaterializedFile] = {} + self._all_files: list[FileInfo] = [] + self._build_virtual_tree() + + def has_entries(self) -> bool: + return bool(self._visible_kbs) + + def _build_virtual_tree(self) -> None: + kb_name_candidates = { + str(db.get("db_id") or db.get("name") or f"kb-{index}"): _sanitize_segment( + db.get("name"), + str(db.get("db_id") or f"kb-{index}"), + ) + for index, db in enumerate(self._visible_kbs) + } + used_root_names: set[str] = set() + kb_virtual_names: dict[str, str] = {} + sorted_kbs = sorted( + self._visible_kbs, + key=lambda item: (str(item.get("name") or ""), str(item.get("db_id") or "")), + ) + for db in sorted_kbs: + db_id = str(db.get("db_id") or "") + if not db_id: + continue + base_name = kb_name_candidates.get(db_id) or db_id + kb_virtual_names[db_id] = _unique_name(base_name, stable_id=db_id, used_names=used_root_names) + + files_by_db: dict[str, dict[str, dict[str, Any]]] = defaultdict(dict) + for file_id, meta in _all_files_meta().items(): + db_id = str(meta.get("database_id") or "") + if db_id in kb_virtual_names: + files_by_db[db_id][str(file_id)] = meta + + for db_id, kb_virtual_name in kb_virtual_names.items(): + kb_root = f"/{kb_virtual_name}" + self._add_entry("/", kb_root, is_dir=True) + records = files_by_db.get(db_id, {}) + self._build_source_tree(db_id=db_id, kb_root=kb_root, records=records) + self._build_parsed_tree(db_id=db_id, kb_root=kb_root, records=records) + + for path, entries in list(self._entries_by_dir.items()): + entries.sort(key=lambda item: str(item.get("path") or "")) + self._entries_by_dir[path] = entries + self._all_files = sorted(self._all_files, key=lambda item: str(item.get("path") or "")) + + def _build_source_tree(self, *, db_id: str, kb_root: str, records: dict[str, dict[str, Any]]) -> None: + children_by_parent: dict[str | None, list[dict[str, Any]]] = defaultdict(list) + for file_id, meta in records.items(): + parent_id = meta.get("parent_id") + children_by_parent[str(parent_id) if parent_id else None].append({"file_id": file_id, "meta": meta}) + + resolved_parent_paths: dict[str, str] = {} + resolved_source_names: dict[str, str] = {} + + def walk(parent_id: str | None, parent_path: str) -> None: + siblings = children_by_parent.get(parent_id, []) + used_names: set[str] = set() + candidates: list[tuple[dict[str, Any], str]] = [] + for item in siblings: + file_id = item["file_id"] + meta = item["meta"] + base_name = _sanitize_segment(_candidate_name(meta), file_id) + candidates.append((item, base_name)) + + for item, base_name in sorted(candidates, key=lambda pair: (pair[1], pair[0]["file_id"])): + file_id = item["file_id"] + meta = item["meta"] + unique_name = _unique_name(base_name, stable_id=file_id, used_names=used_names) + child_path = f"{parent_path.rstrip('/')}/{unique_name}" if parent_path != "/" else f"/{unique_name}" + if meta.get("is_folder"): + modified_at = meta.get("updated_at") or meta.get("created_at") + self._add_entry(parent_path, child_path, is_dir=True, modified_at=modified_at) + walk(file_id, child_path) + continue + + self._add_entry( + parent_path, + child_path, + is_dir=False, + size=int(meta.get("size") or 0), + modified_at=meta.get("updated_at") or meta.get("created_at"), + ) + resolved_parent_paths[file_id] = parent_path + resolved_source_names[file_id] = unique_name + cache_path = self._cache_root / db_id / "source" / file_id / unique_name + self._files[child_path] = _MaterializedFile( + virtual_path=child_path, + cache_path=cache_path, + source_path=str(meta.get("path") or ""), + modified_at=meta.get("updated_at") or meta.get("created_at"), + ) + self._all_files.append( + { + "path": child_path, + "is_dir": False, + "size": int(meta.get("size") or 0), + "modified_at": str(meta.get("updated_at") or meta.get("created_at") or ""), + } + ) + + walk(None, kb_root) + self._resolved_parent_paths = getattr(self, "_resolved_parent_paths", {}) + self._resolved_parent_paths[db_id] = resolved_parent_paths + self._resolved_source_names = getattr(self, "_resolved_source_names", {}) + self._resolved_source_names[db_id] = resolved_source_names + + def _build_parsed_tree(self, *, db_id: str, kb_root: str, records: dict[str, dict[str, Any]]) -> None: + parsed_records = { + file_id: meta + for file_id, meta in records.items() + if not meta.get("is_folder") and str(meta.get("markdown_file") or "").strip() + } + if not parsed_records: + return + + parsed_root = f"{kb_root}/parsed" + self._add_entry(kb_root, parsed_root, is_dir=True) + parent_paths = getattr(self, "_resolved_parent_paths", {}).get(db_id, {}) + source_names = getattr(self, "_resolved_source_names", {}).get(db_id, {}) + + grouped: dict[str, list[tuple[str, dict[str, Any], str]]] = defaultdict(list) + for file_id, meta in parsed_records.items(): + source_parent = parent_paths.get(file_id, kb_root) + parsed_parent = ( + f"{parsed_root}{source_parent[len(kb_root) :]}" if source_parent.startswith(kb_root) else parsed_root + ) + source_name = source_names.get(file_id) or _sanitize_segment(meta.get("filename"), file_id) + source_stem = PurePosixPath(source_name).stem or source_name or file_id + base_name = _sanitize_segment(f"{source_stem}.md", f"{file_id}.md") + grouped[parsed_parent].append((file_id, meta, base_name)) + + for parsed_parent, items in grouped.items(): + current = PurePosixPath(parsed_parent) + while str(current) not in {"", "."} and str(current) != "/": + current_str = str(current) + parent_str = str(current.parent) if str(current.parent) != "." else "/" + if current_str not in self._dir_paths: + self._add_entry(parent_str, current_str, is_dir=True) + current = current.parent + + used_names: set[str] = set() + for file_id, meta, base_name in sorted(items, key=lambda item: (item[2], item[0])): + unique_name = _unique_name(base_name, stable_id=file_id, used_names=used_names) + file_path = f"{parsed_parent.rstrip('/')}/{unique_name}" if parsed_parent != "/" else f"/{unique_name}" + self._add_entry( + parsed_parent, + file_path, + is_dir=False, + size=0, + modified_at=meta.get("updated_at") or meta.get("created_at"), + ) + cache_path = self._cache_root / db_id / "parsed" / f"{file_id}.md" + self._files[file_path] = _MaterializedFile( + virtual_path=file_path, + cache_path=cache_path, + source_path=str(meta.get("markdown_file") or ""), + modified_at=meta.get("updated_at") or meta.get("created_at"), + ) + self._all_files.append( + { + "path": file_path, + "is_dir": False, + "size": 0, + "modified_at": str(meta.get("updated_at") or meta.get("created_at") or ""), + } + ) + + def _add_entry( + self, + parent_path: str, + child_path: str, + *, + is_dir: bool, + size: int = 0, + modified_at: str | None = None, + ) -> None: + normalized_parent = _normalize_virtual_path(parent_path) + normalized_child = _normalize_virtual_path(child_path) + entry_path = f"{normalized_child}/" if is_dir else normalized_child + entry: FileInfo = { + "path": entry_path, + "is_dir": is_dir, + "size": int(size or 0), + "modified_at": str(modified_at or ""), + } + if entry_path not in {str(item.get("path")) for item in self._entries_by_dir[normalized_parent]}: + self._entries_by_dir[normalized_parent].append(entry) + if is_dir: + self._dir_paths.add(normalized_child) + self._entries_by_dir.setdefault(normalized_child, []) + + def _ensure_local_file(self, descriptor: _MaterializedFile) -> Path: + if descriptor.cache_path.exists(): + return descriptor.cache_path + + source = descriptor.source_path.strip() + if not source: + raise FileNotFoundError(descriptor.virtual_path) + if not is_minio_url(source): + raise FileNotFoundError(descriptor.virtual_path) + + bucket_name, object_name = parse_minio_url(source) + payload = get_minio_client().download_file(bucket_name, object_name) + descriptor.cache_path.parent.mkdir(parents=True, exist_ok=True) + descriptor.cache_path.write_bytes(payload) + return descriptor.cache_path + + def ls_info(self, path: str) -> list[FileInfo]: + normalized_path = _normalize_virtual_path(path) + return list(self._entries_by_dir.get(normalized_path, [])) + + def read(self, file_path: str, offset: int = 0, limit: int = 2000) -> str: + normalized_path = _normalize_virtual_path(file_path) + descriptor = self._files.get(normalized_path) + if descriptor is None: + if normalized_path in self._dir_paths: + return f"Error: Path '{file_path}' is a directory" + return f"Error: File '{file_path}' not found" + + try: + content = self._ensure_local_file(descriptor).read_bytes() + except FileNotFoundError: + return f"Error: File '{file_path}' not found" + except Exception as exc: # noqa: BLE001 + detail = str(exc).strip() or "unknown error" + return f"Error: Failed to read '{file_path}': {detail}" + + return _materialize_text_view(content, file_path, offset=offset, limit=limit) + + def glob_info(self, pattern: str, path: str = "/") -> list[FileInfo]: + normalized_path = _normalize_virtual_path(path) + if ".." in PurePosixPath(pattern).parts: + raise ValueError("Path traversal not allowed in glob pattern") + normalized_pattern = pattern.lstrip("/") or "*" + prefix = "/" if normalized_path == "/" else f"{normalized_path.rstrip('/')}/" + + matches: list[FileInfo] = [] + for item in self._all_files: + item_path = str(item.get("path") or "") + if normalized_path != "/" and not item_path.startswith(prefix): + continue + relative = item_path[len(prefix) :] if normalized_path != "/" else item_path.lstrip("/") + if fnmatch.fnmatch(relative, normalized_pattern): + matches.append(dict(item)) + return matches + + def grep_raw(self, pattern: str, path: str | None = None, glob: str | None = None) -> list[dict[str, Any]] | str: + normalized_path = _normalize_virtual_path(path or "/") + prefix = "/" if normalized_path == "/" else f"{normalized_path.rstrip('/')}/" + if normalized_path in self._files: + targets = [normalized_path] + else: + targets = [ + item["path"] + for item in self._all_files + if normalized_path == "/" or str(item["path"]).startswith(prefix) + ] + + matches: list[dict[str, Any]] = [] + for target in targets: + display_target = str(target) + relative = display_target.lstrip("/") if normalized_path == "/" else display_target[len(prefix) :] + if glob and not fnmatch.fnmatch(relative, glob): + continue + descriptor = self._files.get(display_target) + if descriptor is None: + continue + try: + content = self._ensure_local_file(descriptor).read_bytes() + if b"\x00" in content: + continue + text = content.decode("utf-8") + except Exception: # noqa: BLE001 + continue + for line_num, line in enumerate(text.splitlines(), start=1): + if pattern in line: + matches.append({"path": display_target, "line": line_num, "text": line}) + return matches + + def write(self, file_path: str, content: str) -> WriteResult: + return WriteResult(error="Knowledge base path is read-only.") + + def edit(self, file_path: str, old_string: str, new_string: str, replace_all: bool = False) -> EditResult: + return EditResult(error="Knowledge base path is read-only.") + + def upload_files(self, files: list[tuple[str, bytes]]) -> list[FileUploadResponse]: + return [FileUploadResponse(path=path, error="permission_denied") for path, _content in files] + + def download_files(self, paths: list[str]) -> list[FileDownloadResponse]: + responses: list[FileDownloadResponse] = [] + for path in paths: + try: + normalized_path = _normalize_virtual_path(path) + except ValueError: + responses.append(FileDownloadResponse(path=path, content=None, error="invalid_path")) + continue + + descriptor = self._files.get(normalized_path) + if descriptor is None: + if normalized_path in self._dir_paths: + responses.append(FileDownloadResponse(path=path, content=None, error="is_directory")) + else: + responses.append(FileDownloadResponse(path=path, content=None, error="file_not_found")) + continue + + try: + content = self._ensure_local_file(descriptor).read_bytes() + except FileNotFoundError: + responses.append(FileDownloadResponse(path=path, content=None, error="file_not_found")) + continue + except ValueError: + responses.append(FileDownloadResponse(path=path, content=None, error="invalid_path")) + continue + + responses.append(FileDownloadResponse(path=path, content=content, error=None)) + return responses diff --git a/backend/package/yuxi/agents/buildin/chatbot/graph.py b/backend/package/yuxi/agents/buildin/chatbot/graph.py index c2e21744..6a0bfab1 100644 --- a/backend/package/yuxi/agents/buildin/chatbot/graph.py +++ b/backend/package/yuxi/agents/buildin/chatbot/graph.py @@ -39,7 +39,6 @@ class ChatbotAgent(BaseAgent): "你好,请介绍一下你自己", "帮我写一封商务邮件", "解释一下什么是机器学习", - "推荐几本好书", "创建一个冒泡排序 python 并保存结果", ] } diff --git a/backend/package/yuxi/agents/middlewares/knowledge_base_middleware.py b/backend/package/yuxi/agents/middlewares/knowledge_base_middleware.py index ebf63666..bd1b1ec7 100644 --- a/backend/package/yuxi/agents/middlewares/knowledge_base_middleware.py +++ b/backend/package/yuxi/agents/middlewares/knowledge_base_middleware.py @@ -1,7 +1,10 @@ """知识库中间件 - 提供通用知识库工具""" -from langchain.agents.middleware import AgentMiddleware +from collections.abc import Callable +from langchain.agents.middleware import AgentMiddleware, ModelRequest, ModelResponse + +from yuxi.agents.backends.knowledge_base_backend import resolve_visible_knowledge_bases_for_context from yuxi.agents.toolkits.kbs import get_common_kb_tools from yuxi.utils.logging_config import logger @@ -21,3 +24,9 @@ class KnowledgeBaseMiddleware(AgentMiddleware): self.kb_tools = get_common_kb_tools() self.tools = self.kb_tools logger.debug(f"Initialized KnowledgeBaseMiddleware with {len(self.kb_tools)} tools") + + async def awrap_model_call( + self, request: ModelRequest, handler: Callable[[ModelRequest], ModelResponse] + ) -> ModelResponse: + await resolve_visible_knowledge_bases_for_context(request.runtime.context) + return await handler(request) diff --git a/backend/package/yuxi/services/filesystem_service.py b/backend/package/yuxi/services/filesystem_service.py index c0e750b9..e80ca679 100644 --- a/backend/package/yuxi/services/filesystem_service.py +++ b/backend/package/yuxi/services/filesystem_service.py @@ -7,6 +7,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from yuxi.agents.backends import ( ProvisionerSandboxBackend, create_agent_composite_backend, + resolve_visible_knowledge_bases_for_context, ) from yuxi.agents.backends.sandbox.backend import _looks_like_binary from yuxi.agents.context import BaseContext @@ -64,6 +65,7 @@ async def _resolve_filesystem_state( ) runtime_context.thread_id = thread_id runtime_context.user_id = str(user.id) + await resolve_visible_knowledge_bases_for_context(runtime_context) sandbox_backend = ProvisionerSandboxBackend(thread_id=thread_id) return conversation, runtime_context, sandbox_backend diff --git a/backend/package/yuxi/services/viewer_filesystem_service.py b/backend/package/yuxi/services/viewer_filesystem_service.py index 748f4572..e464c53d 100644 --- a/backend/package/yuxi/services/viewer_filesystem_service.py +++ b/backend/package/yuxi/services/viewer_filesystem_service.py @@ -9,6 +9,7 @@ from urllib.parse import quote from fastapi import HTTPException from fastapi.responses import FileResponse, StreamingResponse from sqlalchemy.ext.asyncio import AsyncSession +from yuxi.agents.backends import KBS_PATH, KnowledgeBaseReadonlyBackend, resolve_visible_knowledge_bases_for_context from yuxi.agents.backends.sandbox import SKILLS_PATH, USER_DATA_PATH, resolve_virtual_path from yuxi.agents.backends.skills_backend import SelectedSkillsReadonlyBackend from yuxi.agents.middlewares.skills_middleware import normalize_selected_skills @@ -20,7 +21,7 @@ def _normalize_path(path: str | None) -> str: normalized = (path or "/").strip() or "/" if not normalized.startswith("/"): normalized = f"/{normalized}" - return normalized.rstrip("/") if normalized not in {"/", SKILLS_PATH, USER_DATA_PATH} else normalized + return normalized.rstrip("/") if normalized not in {"/", KBS_PATH, SKILLS_PATH, USER_DATA_PATH} else normalized def _is_user_data_path(path: str) -> bool: @@ -31,16 +32,26 @@ def _is_skills_path(path: str) -> bool: return path == SKILLS_PATH or path.startswith(f"{SKILLS_PATH}/") +def _is_kbs_path(path: str) -> bool: + return path == KBS_PATH or path.startswith(f"{KBS_PATH}/") + + def _strip_skills_prefix(path: str) -> str: if path == SKILLS_PATH: return "/" return path[len(SKILLS_PATH) :] or "/" -def _remap_skills_entry(entry: dict) -> dict: +def _strip_kbs_prefix(path: str) -> str: + if path == KBS_PATH: + return "/" + return path[len(KBS_PATH) :] or "/" + + +def _remap_prefixed_entry(entry: dict, prefix: str) -> dict: raw_path = str(entry.get("path") or "") is_dir = bool(entry.get("is_dir", False)) - remapped = f"{SKILLS_PATH}{raw_path}" if raw_path != "/" else f"{SKILLS_PATH}/" + remapped = f"{prefix}{raw_path}" if raw_path != "/" else f"{prefix}/" if is_dir and not remapped.endswith("/"): remapped = f"{remapped}/" return { @@ -89,9 +100,11 @@ async def _resolve_viewer_state( agent_id=agent_id, agent_config_id=agent_config_id, ) + visible_kbs = await resolve_visible_knowledge_bases_for_context(runtime_context) selected_skills = normalize_selected_skills(getattr(runtime_context, "skills", None) or []) skills_backend = SelectedSkillsReadonlyBackend(selected_slugs=selected_skills) - return sandbox_backend, skills_backend, selected_skills + kb_backend = KnowledgeBaseReadonlyBackend(visible_kbs=visible_kbs) + return sandbox_backend, skills_backend, kb_backend, selected_skills async def list_viewer_filesystem_tree( @@ -107,7 +120,7 @@ async def list_viewer_filesystem_tree( raise HTTPException(status_code=422, detail="thread_id 不能为空") normalized_path = _normalize_path(path) - sandbox_backend, skills_backend, selected_skills = await _resolve_viewer_state( + sandbox_backend, skills_backend, kb_backend, selected_skills = await _resolve_viewer_state( thread_id=thread_id, agent_id=agent_id, agent_config_id=agent_config_id, @@ -119,6 +132,8 @@ async def list_viewer_filesystem_tree( entries = [{"path": f"{USER_DATA_PATH}/", "name": "user-data", "is_dir": True, "size": 0, "modified_at": ""}] if selected_skills: entries.append({"path": f"{SKILLS_PATH}/", "name": "skills", "is_dir": True, "size": 0, "modified_at": ""}) + if kb_backend.has_entries(): + entries.append({"path": f"{KBS_PATH}/", "name": "kbs", "is_dir": True, "size": 0, "modified_at": ""}) return {"entries": entries} try: @@ -128,7 +143,10 @@ async def list_viewer_filesystem_tree( if _is_skills_path(normalized_path): entries = await asyncio.to_thread(skills_backend.ls_info, _strip_skills_prefix(normalized_path)) - return {"entries": [_remap_skills_entry(entry) for entry in entries]} + return {"entries": [_remap_prefixed_entry(entry, SKILLS_PATH) for entry in entries]} + if _is_kbs_path(normalized_path): + entries = await asyncio.to_thread(kb_backend.ls_info, _strip_kbs_prefix(normalized_path)) + return {"entries": [_remap_prefixed_entry(entry, KBS_PATH) for entry in entries]} except PermissionError as e: raise HTTPException(status_code=400, detail=str(e)) from e except ValueError as e: @@ -150,7 +168,7 @@ async def read_viewer_file_content( raise HTTPException(status_code=422, detail="thread_id 不能为空") normalized_path = _normalize_path(path) - sandbox_backend, skills_backend, _selected_skills = await _resolve_viewer_state( + sandbox_backend, skills_backend, kb_backend, _selected_skills = await _resolve_viewer_state( thread_id=thread_id, agent_id=agent_id, agent_config_id=agent_config_id, @@ -163,6 +181,8 @@ async def read_viewer_file_content( responses = await asyncio.to_thread(sandbox_backend.download_files, [normalized_path]) elif _is_skills_path(normalized_path): responses = await asyncio.to_thread(skills_backend.download_files, [_strip_skills_prefix(normalized_path)]) + elif _is_kbs_path(normalized_path): + responses = await asyncio.to_thread(kb_backend.download_files, [_strip_kbs_prefix(normalized_path)]) else: raise HTTPException( status_code=400, @@ -199,7 +219,7 @@ async def download_viewer_file( db: AsyncSession, ) -> StreamingResponse: normalized_path = _normalize_path(path) - _sandbox_backend, skills_backend, _selected_skills = await _resolve_viewer_state( + _sandbox_backend, skills_backend, kb_backend, _selected_skills = await _resolve_viewer_state( thread_id=thread_id, agent_id=agent_id, agent_config_id=agent_config_id, @@ -224,6 +244,8 @@ async def download_viewer_file( if _is_skills_path(normalized_path): responses = await asyncio.to_thread(skills_backend.download_files, [_strip_skills_prefix(normalized_path)]) + elif _is_kbs_path(normalized_path): + responses = await asyncio.to_thread(kb_backend.download_files, [_strip_kbs_prefix(normalized_path)]) else: raise HTTPException( status_code=400, diff --git a/backend/test/api/test_viewer_filesystem_router.py b/backend/test/api/test_viewer_filesystem_router.py index 31a82cd0..ff713808 100644 --- a/backend/test/api/test_viewer_filesystem_router.py +++ b/backend/test/api/test_viewer_filesystem_router.py @@ -2,10 +2,10 @@ from __future__ import annotations +import importlib import uuid import pytest - from yuxi.agents.backends.sandbox import ensure_thread_dirs, sandbox_workspace_dir, virtual_path_for_thread_file pytestmark = [pytest.mark.asyncio, pytest.mark.integration] @@ -44,7 +44,13 @@ async def _create_thread_for_user(test_client, headers: dict[str, str]) -> str: return thread_id -async def _upload_attachment_file(test_client, thread_id: str, headers: dict[str, str], file_name: str, content: str) -> str: +async def _upload_attachment_file( + test_client, + thread_id: str, + headers: dict[str, str], + file_name: str, + content: str, +) -> str: response = await test_client.post( f"/api/chat/thread/{thread_id}/attachments", files={"file": (file_name, content.encode("utf-8"), "text/plain")}, @@ -110,6 +116,7 @@ async def test_viewer_download_returns_attachment_response(test_client, standard assert "download_demo" in content_disposition assert "download-me" in response.text + async def test_viewer_download_returns_full_file_for_large_user_data_content(test_client, standard_user): headers = standard_user["headers"] thread_id = await _create_thread_for_user(test_client, headers) @@ -128,3 +135,47 @@ async def test_viewer_download_returns_full_file_for_large_user_data_content(tes assert response.status_code == 200, response.text assert response.content == large_content.encode("utf-8") + +async def test_viewer_tree_root_lists_kbs_namespace_when_visible(test_client, standard_user, monkeypatch): + headers = standard_user["headers"] + thread_id = await _create_thread_for_user(test_client, headers) + + async def _fake_list_viewer_filesystem_tree(**kwargs): + return { + "entries": [ + {"path": "/home/yuxi/user-data/", "name": "user-data", "is_dir": True, "size": 0, "modified_at": ""}, + {"path": "/home/yuxi/kbs/", "name": "kbs", "is_dir": True, "size": 0, "modified_at": ""}, + ] + } + + router_module = importlib.import_module("server.routers.filesystem_router") + monkeypatch.setitem( + router_module.get_viewer_tree.__globals__, + "list_viewer_filesystem_tree", + _fake_list_viewer_filesystem_tree, + ) + + response = await test_client.get( + "/api/viewer/filesystem/tree", + params={"thread_id": thread_id, "path": "/"}, + headers=headers, + ) + assert response.status_code == 200, response.text + + entries = response.json().get("entries", []) + paths = {entry.get("path") for entry in entries} + assert "/home/yuxi/kbs/" in paths + + +async def test_viewer_can_list_and_read_kbs_namespace(test_client, standard_user): + headers = standard_user["headers"] + thread_id = await _create_thread_for_user(test_client, headers) + + tree_response = await test_client.get( + "/api/viewer/filesystem/tree", + params={"thread_id": thread_id, "path": "/home/yuxi/kbs"}, + headers=headers, + ) + assert tree_response.status_code == 200, tree_response.text + entries = tree_response.json().get("entries", []) + assert any(str(entry.get("path", "")).startswith("/home/yuxi/kbs/") for entry in entries) diff --git a/backend/test/test_knowledge_base_backend.py b/backend/test/test_knowledge_base_backend.py new file mode 100644 index 00000000..7af0fe2f --- /dev/null +++ b/backend/test/test_knowledge_base_backend.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +from types import SimpleNamespace + +import pytest +from yuxi.agents.backends.knowledge_base_backend import ( + KnowledgeBaseReadonlyBackend, + resolve_visible_knowledge_bases_for_context, +) + + +def test_knowledge_base_backend_builds_virtual_tree_and_materializes_files(monkeypatch, tmp_path) -> None: + visible_kbs = [ + {"db_id": "db-1", "name": "FAQ"}, + {"db_id": "db-2", "name": "FAQ"}, + ] + files_meta = { + "folder-api": { + "file_id": "folder-api", + "database_id": "db-1", + "parent_id": None, + "filename": "API", + "is_folder": True, + "created_at": "2026-03-26T00:00:00Z", + }, + "file-pdf": { + "file_id": "file-pdf", + "database_id": "db-1", + "parent_id": "folder-api", + "filename": "auth-guide.pdf", + "path": "http://minio/kb-source/db-1/upload/auth-guide.pdf", + "markdown_file": "http://minio/kb-parsed/db-1/parsed/file-pdf.md", + "size": 12, + "is_folder": False, + "created_at": "2026-03-26T00:00:00Z", + }, + "file-pdf-2": { + "file_id": "file-pdf-2", + "database_id": "db-1", + "parent_id": "folder-api", + "filename": "auth-guide.pdf", + "path": "http://minio/kb-source/db-1/upload/auth-guide-2.pdf", + "size": 7, + "is_folder": False, + "created_at": "2026-03-26T00:00:00Z", + }, + "file-db2": { + "file_id": "file-db2", + "database_id": "db-2", + "parent_id": None, + "filename": "overview.txt", + "path": "http://minio/kb-source/db-2/upload/overview.txt", + "size": 5, + "is_folder": False, + "created_at": "2026-03-26T00:00:00Z", + }, + } + monkeypatch.setattr("yuxi.agents.backends.knowledge_base_backend._all_files_meta", lambda: files_meta) + + class _FakeMinio: + def download_file(self, bucket_name: str, object_name: str) -> bytes: + return f"{bucket_name}:{object_name}".encode() + + monkeypatch.setattr("yuxi.agents.backends.knowledge_base_backend.get_minio_client", lambda: _FakeMinio()) + + backend = KnowledgeBaseReadonlyBackend(visible_kbs=visible_kbs, cache_root=tmp_path) + + root_entries = {entry["path"] for entry in backend.ls_info("/")} + assert "/FAQ/" in root_entries + assert "/FAQ__db-2/" in root_entries + + faq_entries = {entry["path"] for entry in backend.ls_info("/FAQ")} + assert "/FAQ/API/" in faq_entries + assert "/FAQ/parsed/" in faq_entries + + api_entries = {entry["path"] for entry in backend.ls_info("/FAQ/API")} + assert "/FAQ/API/auth-guide.pdf" in api_entries + assert any(path.startswith("/FAQ/API/auth-guide.pdf__file-pdf") for path in api_entries) + + parsed_entries = {entry["path"] for entry in backend.ls_info("/FAQ/parsed/API")} + assert "/FAQ/parsed/API/auth-guide.md" in parsed_entries + + responses = backend.download_files(["/FAQ/API/auth-guide.pdf", "/FAQ/parsed/API/auth-guide.md"]) + assert responses[0].content == b"kb-source:db-1/upload/auth-guide.pdf" + assert responses[1].content == b"kb-parsed:db-1/parsed/file-pdf.md" + + assert "kb-source" in backend.read("/FAQ/API/auth-guide.pdf") + assert backend.write("/FAQ/API/new.txt", "x").error == "Knowledge base path is read-only." + + +@pytest.mark.asyncio +async def test_resolve_visible_knowledge_bases_for_context_filters_by_enabled_names(monkeypatch) -> None: + async def _fake_get_databases_by_raw_id(user_id: int) -> dict: + assert user_id == 7 + return {"databases": [{"db_id": "db-1", "name": "Alpha"}, {"db_id": "db-2", "name": "Beta"}]} + + monkeypatch.setattr( + "yuxi.agents.backends.knowledge_base_backend.knowledge_base.get_databases_by_raw_id", + _fake_get_databases_by_raw_id, + ) + + context = SimpleNamespace(user_id="7", knowledges=["Beta"]) + visible = await resolve_visible_knowledge_bases_for_context(context) + + assert visible == [{"db_id": "db-2", "name": "Beta"}] + assert getattr(context, "_visible_knowledge_bases") == visible diff --git a/backend/test/test_sandbox_backends.py b/backend/test/test_sandbox_backends.py index b19c89d7..225733c9 100644 --- a/backend/test/test_sandbox_backends.py +++ b/backend/test/test_sandbox_backends.py @@ -5,29 +5,36 @@ from __future__ import annotations from types import MethodType, SimpleNamespace import pytest - from yuxi.agents.backends.composite import create_agent_composite_backend from yuxi.agents.backends.sandbox import resolve_virtual_path, sandbox_id_for_thread from yuxi.agents.backends.sandbox.backend import ProvisionerSandboxBackend from yuxi.agents.middlewares.skills_middleware import SkillsMiddleware -def _runtime(*, thread_id: str | None = "thread-1", skills: list[str] | None = None): +def _runtime( + *, + thread_id: str | None = "thread-1", + skills: list[str] | None = None, + visible_kbs: list[dict] | None = None, +): configurable = {"thread_id": thread_id} if thread_id else {} return SimpleNamespace( config={"configurable": configurable}, - context=SimpleNamespace(skills=skills or []), + context=SimpleNamespace(skills=skills or [], _visible_knowledge_bases=visible_kbs or []), ) def test_create_agent_composite_backend_uses_provisioner_default(monkeypatch): monkeypatch.setattr("yuxi.agents.backends.sandbox.backend.get_sandbox_provider", lambda: object()) - backend = create_agent_composite_backend(_runtime(skills=["reporter"])) + backend = create_agent_composite_backend( + _runtime(skills=["reporter"], visible_kbs=[{"db_id": "db-1", "name": "Docs"}]) + ) assert isinstance(backend.default, ProvisionerSandboxBackend) assert backend.default._visible_skills == ["reporter"] assert "/skills/" in backend.routes + assert "/home/yuxi/kbs/" in backend.routes def test_create_agent_composite_backend_requires_thread_id(): diff --git a/docs/develop-guides/roadmap.md b/docs/develop-guides/roadmap.md index 23d861ed..5391ca21 100644 --- a/docs/develop-guides/roadmap.md +++ b/docs/develop-guides/roadmap.md @@ -45,6 +45,8 @@ ### 修复 +- 为沙盒与 viewer 文件系统补齐知识库只读映射:新增 `/home/yuxi/kbs` 命名空间,按“用户可访问知识库 ∩ 当前 Agent 已启用知识库”暴露原始文件与解析后的 Markdown,并补充对应后端与 viewer 路由测试 +