feat: 添加知识库文件在沙盒以及文件系统中的映射 #576
This commit is contained in:
parent
a2f35cbeb2
commit
2be5aedaf2
@ -1,6 +1,7 @@
|
|||||||
from deepagents.backends import CompositeBackend, StateBackend
|
from deepagents.backends import CompositeBackend, StateBackend
|
||||||
|
|
||||||
from .composite import create_agent_composite_backend
|
from .composite import create_agent_composite_backend
|
||||||
|
from .knowledge_base_backend import KBS_PATH, KnowledgeBaseReadonlyBackend, resolve_visible_knowledge_bases_for_context
|
||||||
from .sandbox import (
|
from .sandbox import (
|
||||||
IDLE_CHECK_INTERVAL,
|
IDLE_CHECK_INTERVAL,
|
||||||
LARGE_TOOL_RESULTS_DIR,
|
LARGE_TOOL_RESULTS_DIR,
|
||||||
@ -33,6 +34,8 @@ from .skills_backend import SelectedSkillsReadonlyBackend
|
|||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"CompositeBackend",
|
"CompositeBackend",
|
||||||
|
"KBS_PATH",
|
||||||
|
"KnowledgeBaseReadonlyBackend",
|
||||||
"StateBackend",
|
"StateBackend",
|
||||||
"SelectedSkillsReadonlyBackend",
|
"SelectedSkillsReadonlyBackend",
|
||||||
"create_agent_composite_backend",
|
"create_agent_composite_backend",
|
||||||
@ -47,6 +50,7 @@ __all__ = [
|
|||||||
"init_sandbox_provider",
|
"init_sandbox_provider",
|
||||||
"shutdown_sandbox_provider",
|
"shutdown_sandbox_provider",
|
||||||
"resolve_virtual_path",
|
"resolve_virtual_path",
|
||||||
|
"resolve_visible_knowledge_bases_for_context",
|
||||||
"virtual_path_for_thread_file",
|
"virtual_path_for_thread_file",
|
||||||
"sandbox_id_for_thread",
|
"sandbox_id_for_thread",
|
||||||
"sandbox_user_data_dir",
|
"sandbox_user_data_dir",
|
||||||
|
|||||||
@ -2,13 +2,15 @@ from __future__ import annotations
|
|||||||
|
|
||||||
from deepagents.backends.composite import (
|
from deepagents.backends.composite import (
|
||||||
CompositeBackend,
|
CompositeBackend,
|
||||||
_route_for_path,
|
|
||||||
_remap_file_info_path,
|
_remap_file_info_path,
|
||||||
|
_route_for_path,
|
||||||
_strip_route_from_pattern,
|
_strip_route_from_pattern,
|
||||||
)
|
)
|
||||||
from deepagents.backends.protocol import FileInfo
|
from deepagents.backends.protocol import FileInfo
|
||||||
|
|
||||||
from yuxi.agents.middlewares.skills_middleware import normalize_selected_skills
|
from yuxi.agents.middlewares.skills_middleware import normalize_selected_skills
|
||||||
|
|
||||||
|
from .knowledge_base_backend import KnowledgeBaseReadonlyBackend
|
||||||
from .sandbox import ProvisionerSandboxBackend
|
from .sandbox import ProvisionerSandboxBackend
|
||||||
from .skills_backend import SelectedSkillsReadonlyBackend
|
from .skills_backend import SelectedSkillsReadonlyBackend
|
||||||
|
|
||||||
@ -92,12 +94,22 @@ def _extract_thread_id(runtime) -> str:
|
|||||||
raise ValueError("thread_id is required in runtime configurable context")
|
raise ValueError("thread_id is required in runtime configurable context")
|
||||||
|
|
||||||
|
|
||||||
|
def _get_visible_knowledge_bases_from_runtime(runtime) -> list[dict]:
|
||||||
|
context = getattr(runtime, "context", None)
|
||||||
|
selected = getattr(context, "_visible_knowledge_bases", None)
|
||||||
|
if isinstance(selected, list):
|
||||||
|
return selected
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
def create_agent_composite_backend(runtime) -> CompositeBackend:
|
def create_agent_composite_backend(runtime) -> CompositeBackend:
|
||||||
visible_skills = _get_visible_skills_from_runtime(runtime)
|
visible_skills = _get_visible_skills_from_runtime(runtime)
|
||||||
thread_id = _extract_thread_id(runtime)
|
thread_id = _extract_thread_id(runtime)
|
||||||
|
visible_kbs = _get_visible_knowledge_bases_from_runtime(runtime)
|
||||||
return CustomCompositeBackend(
|
return CustomCompositeBackend(
|
||||||
default=ProvisionerSandboxBackend(thread_id=thread_id, visible_skills=visible_skills),
|
default=ProvisionerSandboxBackend(thread_id=thread_id, visible_skills=visible_skills),
|
||||||
routes={
|
routes={
|
||||||
"/skills/": SelectedSkillsReadonlyBackend(selected_slugs=visible_skills),
|
"/skills/": SelectedSkillsReadonlyBackend(selected_slugs=visible_skills),
|
||||||
|
"/home/yuxi/kbs/": KnowledgeBaseReadonlyBackend(visible_kbs=visible_kbs),
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|||||||
459
backend/package/yuxi/agents/backends/knowledge_base_backend.py
Normal file
459
backend/package/yuxi/agents/backends/knowledge_base_backend.py
Normal file
@ -0,0 +1,459 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import fnmatch
|
||||||
|
import re
|
||||||
|
from collections import defaultdict
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path, PurePosixPath
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from deepagents.backends import FilesystemBackend
|
||||||
|
from deepagents.backends.protocol import EditResult, FileDownloadResponse, FileInfo, FileUploadResponse, WriteResult
|
||||||
|
|
||||||
|
from yuxi import config as conf
|
||||||
|
from yuxi import knowledge_base
|
||||||
|
from yuxi.knowledge.utils.kb_utils import is_minio_url, parse_minio_url
|
||||||
|
from yuxi.storage.minio import get_minio_client
|
||||||
|
|
||||||
|
KBS_PATH = "/home/yuxi/kbs"
|
||||||
|
_INVALID_SEGMENT_RE = re.compile(r"[\\/\x00-\x1f\x7f]+")
|
||||||
|
_WHITESPACE_RE = re.compile(r"\s+")
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class _MaterializedFile:
|
||||||
|
virtual_path: str
|
||||||
|
cache_path: Path
|
||||||
|
source_path: str
|
||||||
|
modified_at: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_virtual_path(path: str | None) -> str:
|
||||||
|
raw = str(path or "").strip() or "/"
|
||||||
|
normalized = "/" + raw.lstrip("/")
|
||||||
|
pure = PurePosixPath(normalized)
|
||||||
|
if ".." in pure.parts:
|
||||||
|
raise ValueError("path traversal is not allowed")
|
||||||
|
return str(pure)
|
||||||
|
|
||||||
|
|
||||||
|
def _sanitize_segment(value: str | None, fallback: str) -> str:
|
||||||
|
cleaned = str(value or "").strip()
|
||||||
|
cleaned = _INVALID_SEGMENT_RE.sub("_", cleaned)
|
||||||
|
cleaned = _WHITESPACE_RE.sub(" ", cleaned).strip()
|
||||||
|
if not cleaned or cleaned in {".", ".."}:
|
||||||
|
return fallback
|
||||||
|
return cleaned
|
||||||
|
|
||||||
|
|
||||||
|
def _candidate_name(file_meta: dict[str, Any]) -> str:
|
||||||
|
filename = str(file_meta.get("filename") or "").strip()
|
||||||
|
if filename:
|
||||||
|
return filename
|
||||||
|
|
||||||
|
original = str(file_meta.get("original_filename") or "").strip()
|
||||||
|
if original:
|
||||||
|
return original
|
||||||
|
|
||||||
|
for key in ("path", "markdown_file"):
|
||||||
|
raw = str(file_meta.get(key) or "").strip()
|
||||||
|
if raw:
|
||||||
|
try:
|
||||||
|
parsed = PurePosixPath(raw)
|
||||||
|
name = parsed.name
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
name = ""
|
||||||
|
if name:
|
||||||
|
return name
|
||||||
|
|
||||||
|
return str(file_meta.get("file_id") or "")
|
||||||
|
|
||||||
|
|
||||||
|
def _unique_name(base_name: str, *, stable_id: str, used_names: set[str]) -> str:
|
||||||
|
if base_name not in used_names:
|
||||||
|
used_names.add(base_name)
|
||||||
|
return base_name
|
||||||
|
|
||||||
|
suffix = f"__{stable_id[:8]}"
|
||||||
|
candidate = f"{base_name}{suffix}"
|
||||||
|
while candidate in used_names:
|
||||||
|
suffix = f"__{stable_id}"
|
||||||
|
candidate = f"{base_name}{suffix}"
|
||||||
|
used_names.add(candidate)
|
||||||
|
return candidate
|
||||||
|
|
||||||
|
|
||||||
|
def _materialize_text_view(content: bytes, file_path: str, *, offset: int = 0, limit: int = 2000) -> str:
|
||||||
|
if not content:
|
||||||
|
return "System reminder: File exists but has empty contents"
|
||||||
|
|
||||||
|
if b"\x00" in content:
|
||||||
|
return f"Error: File '{file_path}' is binary and cannot be rendered as text"
|
||||||
|
|
||||||
|
try:
|
||||||
|
text = content.decode("utf-8")
|
||||||
|
except UnicodeDecodeError:
|
||||||
|
return f"Error: File '{file_path}' is binary and cannot be rendered as text"
|
||||||
|
|
||||||
|
lines = text.splitlines()
|
||||||
|
start = max(0, int(offset))
|
||||||
|
selected = lines[start : start + int(limit)]
|
||||||
|
return "\n".join(f"{start + idx + 1:6d}\t{line}" for idx, line in enumerate(selected))
|
||||||
|
|
||||||
|
|
||||||
|
async def resolve_visible_knowledge_bases_for_context(context) -> list[dict[str, Any]]:
|
||||||
|
user_id = getattr(context, "user_id", None)
|
||||||
|
if not user_id:
|
||||||
|
setattr(context, "_visible_knowledge_bases", [])
|
||||||
|
return []
|
||||||
|
|
||||||
|
enabled_names = {str(name).strip() for name in (getattr(context, "knowledges", None) or []) if str(name).strip()}
|
||||||
|
if not enabled_names:
|
||||||
|
setattr(context, "_visible_knowledge_bases", [])
|
||||||
|
return []
|
||||||
|
|
||||||
|
try:
|
||||||
|
raw_user_id = int(user_id)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
setattr(context, "_visible_knowledge_bases", [])
|
||||||
|
return []
|
||||||
|
|
||||||
|
result = await knowledge_base.get_databases_by_raw_id(raw_user_id)
|
||||||
|
databases = [db for db in (result.get("databases") or []) if str(db.get("name") or "").strip() in enabled_names]
|
||||||
|
setattr(context, "_visible_knowledge_bases", databases)
|
||||||
|
return databases
|
||||||
|
|
||||||
|
|
||||||
|
def _all_files_meta() -> dict[str, dict[str, Any]]:
|
||||||
|
aggregated: dict[str, dict[str, Any]] = {}
|
||||||
|
for kb_instance in getattr(knowledge_base, "kb_instances", {}).values():
|
||||||
|
for file_id, meta in getattr(kb_instance, "files_meta", {}).items():
|
||||||
|
aggregated[str(file_id)] = meta
|
||||||
|
return aggregated
|
||||||
|
|
||||||
|
|
||||||
|
class KnowledgeBaseReadonlyBackend(FilesystemBackend):
|
||||||
|
def __init__(self, *, visible_kbs: list[dict[str, Any]] | None, cache_root: Path | str | None = None):
|
||||||
|
self._cache_root = Path(cache_root or (Path(conf.save_dir) / "knowledge_base_data" / "kb-cache")).resolve()
|
||||||
|
self._cache_root.mkdir(parents=True, exist_ok=True)
|
||||||
|
super().__init__(root_dir=self._cache_root, virtual_mode=True)
|
||||||
|
self._visible_kbs = list(visible_kbs or [])
|
||||||
|
self._entries_by_dir: dict[str, list[FileInfo]] = defaultdict(list)
|
||||||
|
self._dir_paths: set[str] = {"/"}
|
||||||
|
self._files: dict[str, _MaterializedFile] = {}
|
||||||
|
self._all_files: list[FileInfo] = []
|
||||||
|
self._build_virtual_tree()
|
||||||
|
|
||||||
|
def has_entries(self) -> bool:
|
||||||
|
return bool(self._visible_kbs)
|
||||||
|
|
||||||
|
def _build_virtual_tree(self) -> None:
|
||||||
|
kb_name_candidates = {
|
||||||
|
str(db.get("db_id") or db.get("name") or f"kb-{index}"): _sanitize_segment(
|
||||||
|
db.get("name"),
|
||||||
|
str(db.get("db_id") or f"kb-{index}"),
|
||||||
|
)
|
||||||
|
for index, db in enumerate(self._visible_kbs)
|
||||||
|
}
|
||||||
|
used_root_names: set[str] = set()
|
||||||
|
kb_virtual_names: dict[str, str] = {}
|
||||||
|
sorted_kbs = sorted(
|
||||||
|
self._visible_kbs,
|
||||||
|
key=lambda item: (str(item.get("name") or ""), str(item.get("db_id") or "")),
|
||||||
|
)
|
||||||
|
for db in sorted_kbs:
|
||||||
|
db_id = str(db.get("db_id") or "")
|
||||||
|
if not db_id:
|
||||||
|
continue
|
||||||
|
base_name = kb_name_candidates.get(db_id) or db_id
|
||||||
|
kb_virtual_names[db_id] = _unique_name(base_name, stable_id=db_id, used_names=used_root_names)
|
||||||
|
|
||||||
|
files_by_db: dict[str, dict[str, dict[str, Any]]] = defaultdict(dict)
|
||||||
|
for file_id, meta in _all_files_meta().items():
|
||||||
|
db_id = str(meta.get("database_id") or "")
|
||||||
|
if db_id in kb_virtual_names:
|
||||||
|
files_by_db[db_id][str(file_id)] = meta
|
||||||
|
|
||||||
|
for db_id, kb_virtual_name in kb_virtual_names.items():
|
||||||
|
kb_root = f"/{kb_virtual_name}"
|
||||||
|
self._add_entry("/", kb_root, is_dir=True)
|
||||||
|
records = files_by_db.get(db_id, {})
|
||||||
|
self._build_source_tree(db_id=db_id, kb_root=kb_root, records=records)
|
||||||
|
self._build_parsed_tree(db_id=db_id, kb_root=kb_root, records=records)
|
||||||
|
|
||||||
|
for path, entries in list(self._entries_by_dir.items()):
|
||||||
|
entries.sort(key=lambda item: str(item.get("path") or ""))
|
||||||
|
self._entries_by_dir[path] = entries
|
||||||
|
self._all_files = sorted(self._all_files, key=lambda item: str(item.get("path") or ""))
|
||||||
|
|
||||||
|
def _build_source_tree(self, *, db_id: str, kb_root: str, records: dict[str, dict[str, Any]]) -> None:
|
||||||
|
children_by_parent: dict[str | None, list[dict[str, Any]]] = defaultdict(list)
|
||||||
|
for file_id, meta in records.items():
|
||||||
|
parent_id = meta.get("parent_id")
|
||||||
|
children_by_parent[str(parent_id) if parent_id else None].append({"file_id": file_id, "meta": meta})
|
||||||
|
|
||||||
|
resolved_parent_paths: dict[str, str] = {}
|
||||||
|
resolved_source_names: dict[str, str] = {}
|
||||||
|
|
||||||
|
def walk(parent_id: str | None, parent_path: str) -> None:
|
||||||
|
siblings = children_by_parent.get(parent_id, [])
|
||||||
|
used_names: set[str] = set()
|
||||||
|
candidates: list[tuple[dict[str, Any], str]] = []
|
||||||
|
for item in siblings:
|
||||||
|
file_id = item["file_id"]
|
||||||
|
meta = item["meta"]
|
||||||
|
base_name = _sanitize_segment(_candidate_name(meta), file_id)
|
||||||
|
candidates.append((item, base_name))
|
||||||
|
|
||||||
|
for item, base_name in sorted(candidates, key=lambda pair: (pair[1], pair[0]["file_id"])):
|
||||||
|
file_id = item["file_id"]
|
||||||
|
meta = item["meta"]
|
||||||
|
unique_name = _unique_name(base_name, stable_id=file_id, used_names=used_names)
|
||||||
|
child_path = f"{parent_path.rstrip('/')}/{unique_name}" if parent_path != "/" else f"/{unique_name}"
|
||||||
|
if meta.get("is_folder"):
|
||||||
|
modified_at = meta.get("updated_at") or meta.get("created_at")
|
||||||
|
self._add_entry(parent_path, child_path, is_dir=True, modified_at=modified_at)
|
||||||
|
walk(file_id, child_path)
|
||||||
|
continue
|
||||||
|
|
||||||
|
self._add_entry(
|
||||||
|
parent_path,
|
||||||
|
child_path,
|
||||||
|
is_dir=False,
|
||||||
|
size=int(meta.get("size") or 0),
|
||||||
|
modified_at=meta.get("updated_at") or meta.get("created_at"),
|
||||||
|
)
|
||||||
|
resolved_parent_paths[file_id] = parent_path
|
||||||
|
resolved_source_names[file_id] = unique_name
|
||||||
|
cache_path = self._cache_root / db_id / "source" / file_id / unique_name
|
||||||
|
self._files[child_path] = _MaterializedFile(
|
||||||
|
virtual_path=child_path,
|
||||||
|
cache_path=cache_path,
|
||||||
|
source_path=str(meta.get("path") or ""),
|
||||||
|
modified_at=meta.get("updated_at") or meta.get("created_at"),
|
||||||
|
)
|
||||||
|
self._all_files.append(
|
||||||
|
{
|
||||||
|
"path": child_path,
|
||||||
|
"is_dir": False,
|
||||||
|
"size": int(meta.get("size") or 0),
|
||||||
|
"modified_at": str(meta.get("updated_at") or meta.get("created_at") or ""),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
walk(None, kb_root)
|
||||||
|
self._resolved_parent_paths = getattr(self, "_resolved_parent_paths", {})
|
||||||
|
self._resolved_parent_paths[db_id] = resolved_parent_paths
|
||||||
|
self._resolved_source_names = getattr(self, "_resolved_source_names", {})
|
||||||
|
self._resolved_source_names[db_id] = resolved_source_names
|
||||||
|
|
||||||
|
def _build_parsed_tree(self, *, db_id: str, kb_root: str, records: dict[str, dict[str, Any]]) -> None:
|
||||||
|
parsed_records = {
|
||||||
|
file_id: meta
|
||||||
|
for file_id, meta in records.items()
|
||||||
|
if not meta.get("is_folder") and str(meta.get("markdown_file") or "").strip()
|
||||||
|
}
|
||||||
|
if not parsed_records:
|
||||||
|
return
|
||||||
|
|
||||||
|
parsed_root = f"{kb_root}/parsed"
|
||||||
|
self._add_entry(kb_root, parsed_root, is_dir=True)
|
||||||
|
parent_paths = getattr(self, "_resolved_parent_paths", {}).get(db_id, {})
|
||||||
|
source_names = getattr(self, "_resolved_source_names", {}).get(db_id, {})
|
||||||
|
|
||||||
|
grouped: dict[str, list[tuple[str, dict[str, Any], str]]] = defaultdict(list)
|
||||||
|
for file_id, meta in parsed_records.items():
|
||||||
|
source_parent = parent_paths.get(file_id, kb_root)
|
||||||
|
parsed_parent = (
|
||||||
|
f"{parsed_root}{source_parent[len(kb_root) :]}" if source_parent.startswith(kb_root) else parsed_root
|
||||||
|
)
|
||||||
|
source_name = source_names.get(file_id) or _sanitize_segment(meta.get("filename"), file_id)
|
||||||
|
source_stem = PurePosixPath(source_name).stem or source_name or file_id
|
||||||
|
base_name = _sanitize_segment(f"{source_stem}.md", f"{file_id}.md")
|
||||||
|
grouped[parsed_parent].append((file_id, meta, base_name))
|
||||||
|
|
||||||
|
for parsed_parent, items in grouped.items():
|
||||||
|
current = PurePosixPath(parsed_parent)
|
||||||
|
while str(current) not in {"", "."} and str(current) != "/":
|
||||||
|
current_str = str(current)
|
||||||
|
parent_str = str(current.parent) if str(current.parent) != "." else "/"
|
||||||
|
if current_str not in self._dir_paths:
|
||||||
|
self._add_entry(parent_str, current_str, is_dir=True)
|
||||||
|
current = current.parent
|
||||||
|
|
||||||
|
used_names: set[str] = set()
|
||||||
|
for file_id, meta, base_name in sorted(items, key=lambda item: (item[2], item[0])):
|
||||||
|
unique_name = _unique_name(base_name, stable_id=file_id, used_names=used_names)
|
||||||
|
file_path = f"{parsed_parent.rstrip('/')}/{unique_name}" if parsed_parent != "/" else f"/{unique_name}"
|
||||||
|
self._add_entry(
|
||||||
|
parsed_parent,
|
||||||
|
file_path,
|
||||||
|
is_dir=False,
|
||||||
|
size=0,
|
||||||
|
modified_at=meta.get("updated_at") or meta.get("created_at"),
|
||||||
|
)
|
||||||
|
cache_path = self._cache_root / db_id / "parsed" / f"{file_id}.md"
|
||||||
|
self._files[file_path] = _MaterializedFile(
|
||||||
|
virtual_path=file_path,
|
||||||
|
cache_path=cache_path,
|
||||||
|
source_path=str(meta.get("markdown_file") or ""),
|
||||||
|
modified_at=meta.get("updated_at") or meta.get("created_at"),
|
||||||
|
)
|
||||||
|
self._all_files.append(
|
||||||
|
{
|
||||||
|
"path": file_path,
|
||||||
|
"is_dir": False,
|
||||||
|
"size": 0,
|
||||||
|
"modified_at": str(meta.get("updated_at") or meta.get("created_at") or ""),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
def _add_entry(
|
||||||
|
self,
|
||||||
|
parent_path: str,
|
||||||
|
child_path: str,
|
||||||
|
*,
|
||||||
|
is_dir: bool,
|
||||||
|
size: int = 0,
|
||||||
|
modified_at: str | None = None,
|
||||||
|
) -> None:
|
||||||
|
normalized_parent = _normalize_virtual_path(parent_path)
|
||||||
|
normalized_child = _normalize_virtual_path(child_path)
|
||||||
|
entry_path = f"{normalized_child}/" if is_dir else normalized_child
|
||||||
|
entry: FileInfo = {
|
||||||
|
"path": entry_path,
|
||||||
|
"is_dir": is_dir,
|
||||||
|
"size": int(size or 0),
|
||||||
|
"modified_at": str(modified_at or ""),
|
||||||
|
}
|
||||||
|
if entry_path not in {str(item.get("path")) for item in self._entries_by_dir[normalized_parent]}:
|
||||||
|
self._entries_by_dir[normalized_parent].append(entry)
|
||||||
|
if is_dir:
|
||||||
|
self._dir_paths.add(normalized_child)
|
||||||
|
self._entries_by_dir.setdefault(normalized_child, [])
|
||||||
|
|
||||||
|
def _ensure_local_file(self, descriptor: _MaterializedFile) -> Path:
|
||||||
|
if descriptor.cache_path.exists():
|
||||||
|
return descriptor.cache_path
|
||||||
|
|
||||||
|
source = descriptor.source_path.strip()
|
||||||
|
if not source:
|
||||||
|
raise FileNotFoundError(descriptor.virtual_path)
|
||||||
|
if not is_minio_url(source):
|
||||||
|
raise FileNotFoundError(descriptor.virtual_path)
|
||||||
|
|
||||||
|
bucket_name, object_name = parse_minio_url(source)
|
||||||
|
payload = get_minio_client().download_file(bucket_name, object_name)
|
||||||
|
descriptor.cache_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
descriptor.cache_path.write_bytes(payload)
|
||||||
|
return descriptor.cache_path
|
||||||
|
|
||||||
|
def ls_info(self, path: str) -> list[FileInfo]:
|
||||||
|
normalized_path = _normalize_virtual_path(path)
|
||||||
|
return list(self._entries_by_dir.get(normalized_path, []))
|
||||||
|
|
||||||
|
def read(self, file_path: str, offset: int = 0, limit: int = 2000) -> str:
|
||||||
|
normalized_path = _normalize_virtual_path(file_path)
|
||||||
|
descriptor = self._files.get(normalized_path)
|
||||||
|
if descriptor is None:
|
||||||
|
if normalized_path in self._dir_paths:
|
||||||
|
return f"Error: Path '{file_path}' is a directory"
|
||||||
|
return f"Error: File '{file_path}' not found"
|
||||||
|
|
||||||
|
try:
|
||||||
|
content = self._ensure_local_file(descriptor).read_bytes()
|
||||||
|
except FileNotFoundError:
|
||||||
|
return f"Error: File '{file_path}' not found"
|
||||||
|
except Exception as exc: # noqa: BLE001
|
||||||
|
detail = str(exc).strip() or "unknown error"
|
||||||
|
return f"Error: Failed to read '{file_path}': {detail}"
|
||||||
|
|
||||||
|
return _materialize_text_view(content, file_path, offset=offset, limit=limit)
|
||||||
|
|
||||||
|
def glob_info(self, pattern: str, path: str = "/") -> list[FileInfo]:
|
||||||
|
normalized_path = _normalize_virtual_path(path)
|
||||||
|
if ".." in PurePosixPath(pattern).parts:
|
||||||
|
raise ValueError("Path traversal not allowed in glob pattern")
|
||||||
|
normalized_pattern = pattern.lstrip("/") or "*"
|
||||||
|
prefix = "/" if normalized_path == "/" else f"{normalized_path.rstrip('/')}/"
|
||||||
|
|
||||||
|
matches: list[FileInfo] = []
|
||||||
|
for item in self._all_files:
|
||||||
|
item_path = str(item.get("path") or "")
|
||||||
|
if normalized_path != "/" and not item_path.startswith(prefix):
|
||||||
|
continue
|
||||||
|
relative = item_path[len(prefix) :] if normalized_path != "/" else item_path.lstrip("/")
|
||||||
|
if fnmatch.fnmatch(relative, normalized_pattern):
|
||||||
|
matches.append(dict(item))
|
||||||
|
return matches
|
||||||
|
|
||||||
|
def grep_raw(self, pattern: str, path: str | None = None, glob: str | None = None) -> list[dict[str, Any]] | str:
|
||||||
|
normalized_path = _normalize_virtual_path(path or "/")
|
||||||
|
prefix = "/" if normalized_path == "/" else f"{normalized_path.rstrip('/')}/"
|
||||||
|
if normalized_path in self._files:
|
||||||
|
targets = [normalized_path]
|
||||||
|
else:
|
||||||
|
targets = [
|
||||||
|
item["path"]
|
||||||
|
for item in self._all_files
|
||||||
|
if normalized_path == "/" or str(item["path"]).startswith(prefix)
|
||||||
|
]
|
||||||
|
|
||||||
|
matches: list[dict[str, Any]] = []
|
||||||
|
for target in targets:
|
||||||
|
display_target = str(target)
|
||||||
|
relative = display_target.lstrip("/") if normalized_path == "/" else display_target[len(prefix) :]
|
||||||
|
if glob and not fnmatch.fnmatch(relative, glob):
|
||||||
|
continue
|
||||||
|
descriptor = self._files.get(display_target)
|
||||||
|
if descriptor is None:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
content = self._ensure_local_file(descriptor).read_bytes()
|
||||||
|
if b"\x00" in content:
|
||||||
|
continue
|
||||||
|
text = content.decode("utf-8")
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
continue
|
||||||
|
for line_num, line in enumerate(text.splitlines(), start=1):
|
||||||
|
if pattern in line:
|
||||||
|
matches.append({"path": display_target, "line": line_num, "text": line})
|
||||||
|
return matches
|
||||||
|
|
||||||
|
def write(self, file_path: str, content: str) -> WriteResult:
|
||||||
|
return WriteResult(error="Knowledge base path is read-only.")
|
||||||
|
|
||||||
|
def edit(self, file_path: str, old_string: str, new_string: str, replace_all: bool = False) -> EditResult:
|
||||||
|
return EditResult(error="Knowledge base path is read-only.")
|
||||||
|
|
||||||
|
def upload_files(self, files: list[tuple[str, bytes]]) -> list[FileUploadResponse]:
|
||||||
|
return [FileUploadResponse(path=path, error="permission_denied") for path, _content in files]
|
||||||
|
|
||||||
|
def download_files(self, paths: list[str]) -> list[FileDownloadResponse]:
|
||||||
|
responses: list[FileDownloadResponse] = []
|
||||||
|
for path in paths:
|
||||||
|
try:
|
||||||
|
normalized_path = _normalize_virtual_path(path)
|
||||||
|
except ValueError:
|
||||||
|
responses.append(FileDownloadResponse(path=path, content=None, error="invalid_path"))
|
||||||
|
continue
|
||||||
|
|
||||||
|
descriptor = self._files.get(normalized_path)
|
||||||
|
if descriptor is None:
|
||||||
|
if normalized_path in self._dir_paths:
|
||||||
|
responses.append(FileDownloadResponse(path=path, content=None, error="is_directory"))
|
||||||
|
else:
|
||||||
|
responses.append(FileDownloadResponse(path=path, content=None, error="file_not_found"))
|
||||||
|
continue
|
||||||
|
|
||||||
|
try:
|
||||||
|
content = self._ensure_local_file(descriptor).read_bytes()
|
||||||
|
except FileNotFoundError:
|
||||||
|
responses.append(FileDownloadResponse(path=path, content=None, error="file_not_found"))
|
||||||
|
continue
|
||||||
|
except ValueError:
|
||||||
|
responses.append(FileDownloadResponse(path=path, content=None, error="invalid_path"))
|
||||||
|
continue
|
||||||
|
|
||||||
|
responses.append(FileDownloadResponse(path=path, content=content, error=None))
|
||||||
|
return responses
|
||||||
@ -39,7 +39,6 @@ class ChatbotAgent(BaseAgent):
|
|||||||
"你好,请介绍一下你自己",
|
"你好,请介绍一下你自己",
|
||||||
"帮我写一封商务邮件",
|
"帮我写一封商务邮件",
|
||||||
"解释一下什么是机器学习",
|
"解释一下什么是机器学习",
|
||||||
"推荐几本好书",
|
|
||||||
"创建一个冒泡排序 python 并保存结果",
|
"创建一个冒泡排序 python 并保存结果",
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,7 +1,10 @@
|
|||||||
"""知识库中间件 - 提供通用知识库工具"""
|
"""知识库中间件 - 提供通用知识库工具"""
|
||||||
|
|
||||||
from langchain.agents.middleware import AgentMiddleware
|
from collections.abc import Callable
|
||||||
|
|
||||||
|
from langchain.agents.middleware import AgentMiddleware, ModelRequest, ModelResponse
|
||||||
|
|
||||||
|
from yuxi.agents.backends.knowledge_base_backend import resolve_visible_knowledge_bases_for_context
|
||||||
from yuxi.agents.toolkits.kbs import get_common_kb_tools
|
from yuxi.agents.toolkits.kbs import get_common_kb_tools
|
||||||
from yuxi.utils.logging_config import logger
|
from yuxi.utils.logging_config import logger
|
||||||
|
|
||||||
@ -21,3 +24,9 @@ class KnowledgeBaseMiddleware(AgentMiddleware):
|
|||||||
self.kb_tools = get_common_kb_tools()
|
self.kb_tools = get_common_kb_tools()
|
||||||
self.tools = self.kb_tools
|
self.tools = self.kb_tools
|
||||||
logger.debug(f"Initialized KnowledgeBaseMiddleware with {len(self.kb_tools)} tools")
|
logger.debug(f"Initialized KnowledgeBaseMiddleware with {len(self.kb_tools)} tools")
|
||||||
|
|
||||||
|
async def awrap_model_call(
|
||||||
|
self, request: ModelRequest, handler: Callable[[ModelRequest], ModelResponse]
|
||||||
|
) -> ModelResponse:
|
||||||
|
await resolve_visible_knowledge_bases_for_context(request.runtime.context)
|
||||||
|
return await handler(request)
|
||||||
|
|||||||
@ -7,6 +7,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
|||||||
from yuxi.agents.backends import (
|
from yuxi.agents.backends import (
|
||||||
ProvisionerSandboxBackend,
|
ProvisionerSandboxBackend,
|
||||||
create_agent_composite_backend,
|
create_agent_composite_backend,
|
||||||
|
resolve_visible_knowledge_bases_for_context,
|
||||||
)
|
)
|
||||||
from yuxi.agents.backends.sandbox.backend import _looks_like_binary
|
from yuxi.agents.backends.sandbox.backend import _looks_like_binary
|
||||||
from yuxi.agents.context import BaseContext
|
from yuxi.agents.context import BaseContext
|
||||||
@ -64,6 +65,7 @@ async def _resolve_filesystem_state(
|
|||||||
)
|
)
|
||||||
runtime_context.thread_id = thread_id
|
runtime_context.thread_id = thread_id
|
||||||
runtime_context.user_id = str(user.id)
|
runtime_context.user_id = str(user.id)
|
||||||
|
await resolve_visible_knowledge_bases_for_context(runtime_context)
|
||||||
|
|
||||||
sandbox_backend = ProvisionerSandboxBackend(thread_id=thread_id)
|
sandbox_backend = ProvisionerSandboxBackend(thread_id=thread_id)
|
||||||
return conversation, runtime_context, sandbox_backend
|
return conversation, runtime_context, sandbox_backend
|
||||||
|
|||||||
@ -9,6 +9,7 @@ from urllib.parse import quote
|
|||||||
from fastapi import HTTPException
|
from fastapi import HTTPException
|
||||||
from fastapi.responses import FileResponse, StreamingResponse
|
from fastapi.responses import FileResponse, StreamingResponse
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
from yuxi.agents.backends import KBS_PATH, KnowledgeBaseReadonlyBackend, resolve_visible_knowledge_bases_for_context
|
||||||
from yuxi.agents.backends.sandbox import SKILLS_PATH, USER_DATA_PATH, resolve_virtual_path
|
from yuxi.agents.backends.sandbox import SKILLS_PATH, USER_DATA_PATH, resolve_virtual_path
|
||||||
from yuxi.agents.backends.skills_backend import SelectedSkillsReadonlyBackend
|
from yuxi.agents.backends.skills_backend import SelectedSkillsReadonlyBackend
|
||||||
from yuxi.agents.middlewares.skills_middleware import normalize_selected_skills
|
from yuxi.agents.middlewares.skills_middleware import normalize_selected_skills
|
||||||
@ -20,7 +21,7 @@ def _normalize_path(path: str | None) -> str:
|
|||||||
normalized = (path or "/").strip() or "/"
|
normalized = (path or "/").strip() or "/"
|
||||||
if not normalized.startswith("/"):
|
if not normalized.startswith("/"):
|
||||||
normalized = f"/{normalized}"
|
normalized = f"/{normalized}"
|
||||||
return normalized.rstrip("/") if normalized not in {"/", SKILLS_PATH, USER_DATA_PATH} else normalized
|
return normalized.rstrip("/") if normalized not in {"/", KBS_PATH, SKILLS_PATH, USER_DATA_PATH} else normalized
|
||||||
|
|
||||||
|
|
||||||
def _is_user_data_path(path: str) -> bool:
|
def _is_user_data_path(path: str) -> bool:
|
||||||
@ -31,16 +32,26 @@ def _is_skills_path(path: str) -> bool:
|
|||||||
return path == SKILLS_PATH or path.startswith(f"{SKILLS_PATH}/")
|
return path == SKILLS_PATH or path.startswith(f"{SKILLS_PATH}/")
|
||||||
|
|
||||||
|
|
||||||
|
def _is_kbs_path(path: str) -> bool:
|
||||||
|
return path == KBS_PATH or path.startswith(f"{KBS_PATH}/")
|
||||||
|
|
||||||
|
|
||||||
def _strip_skills_prefix(path: str) -> str:
|
def _strip_skills_prefix(path: str) -> str:
|
||||||
if path == SKILLS_PATH:
|
if path == SKILLS_PATH:
|
||||||
return "/"
|
return "/"
|
||||||
return path[len(SKILLS_PATH) :] or "/"
|
return path[len(SKILLS_PATH) :] or "/"
|
||||||
|
|
||||||
|
|
||||||
def _remap_skills_entry(entry: dict) -> dict:
|
def _strip_kbs_prefix(path: str) -> str:
|
||||||
|
if path == KBS_PATH:
|
||||||
|
return "/"
|
||||||
|
return path[len(KBS_PATH) :] or "/"
|
||||||
|
|
||||||
|
|
||||||
|
def _remap_prefixed_entry(entry: dict, prefix: str) -> dict:
|
||||||
raw_path = str(entry.get("path") or "")
|
raw_path = str(entry.get("path") or "")
|
||||||
is_dir = bool(entry.get("is_dir", False))
|
is_dir = bool(entry.get("is_dir", False))
|
||||||
remapped = f"{SKILLS_PATH}{raw_path}" if raw_path != "/" else f"{SKILLS_PATH}/"
|
remapped = f"{prefix}{raw_path}" if raw_path != "/" else f"{prefix}/"
|
||||||
if is_dir and not remapped.endswith("/"):
|
if is_dir and not remapped.endswith("/"):
|
||||||
remapped = f"{remapped}/"
|
remapped = f"{remapped}/"
|
||||||
return {
|
return {
|
||||||
@ -89,9 +100,11 @@ async def _resolve_viewer_state(
|
|||||||
agent_id=agent_id,
|
agent_id=agent_id,
|
||||||
agent_config_id=agent_config_id,
|
agent_config_id=agent_config_id,
|
||||||
)
|
)
|
||||||
|
visible_kbs = await resolve_visible_knowledge_bases_for_context(runtime_context)
|
||||||
selected_skills = normalize_selected_skills(getattr(runtime_context, "skills", None) or [])
|
selected_skills = normalize_selected_skills(getattr(runtime_context, "skills", None) or [])
|
||||||
skills_backend = SelectedSkillsReadonlyBackend(selected_slugs=selected_skills)
|
skills_backend = SelectedSkillsReadonlyBackend(selected_slugs=selected_skills)
|
||||||
return sandbox_backend, skills_backend, selected_skills
|
kb_backend = KnowledgeBaseReadonlyBackend(visible_kbs=visible_kbs)
|
||||||
|
return sandbox_backend, skills_backend, kb_backend, selected_skills
|
||||||
|
|
||||||
|
|
||||||
async def list_viewer_filesystem_tree(
|
async def list_viewer_filesystem_tree(
|
||||||
@ -107,7 +120,7 @@ async def list_viewer_filesystem_tree(
|
|||||||
raise HTTPException(status_code=422, detail="thread_id 不能为空")
|
raise HTTPException(status_code=422, detail="thread_id 不能为空")
|
||||||
|
|
||||||
normalized_path = _normalize_path(path)
|
normalized_path = _normalize_path(path)
|
||||||
sandbox_backend, skills_backend, selected_skills = await _resolve_viewer_state(
|
sandbox_backend, skills_backend, kb_backend, selected_skills = await _resolve_viewer_state(
|
||||||
thread_id=thread_id,
|
thread_id=thread_id,
|
||||||
agent_id=agent_id,
|
agent_id=agent_id,
|
||||||
agent_config_id=agent_config_id,
|
agent_config_id=agent_config_id,
|
||||||
@ -119,6 +132,8 @@ async def list_viewer_filesystem_tree(
|
|||||||
entries = [{"path": f"{USER_DATA_PATH}/", "name": "user-data", "is_dir": True, "size": 0, "modified_at": ""}]
|
entries = [{"path": f"{USER_DATA_PATH}/", "name": "user-data", "is_dir": True, "size": 0, "modified_at": ""}]
|
||||||
if selected_skills:
|
if selected_skills:
|
||||||
entries.append({"path": f"{SKILLS_PATH}/", "name": "skills", "is_dir": True, "size": 0, "modified_at": ""})
|
entries.append({"path": f"{SKILLS_PATH}/", "name": "skills", "is_dir": True, "size": 0, "modified_at": ""})
|
||||||
|
if kb_backend.has_entries():
|
||||||
|
entries.append({"path": f"{KBS_PATH}/", "name": "kbs", "is_dir": True, "size": 0, "modified_at": ""})
|
||||||
return {"entries": entries}
|
return {"entries": entries}
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@ -128,7 +143,10 @@ async def list_viewer_filesystem_tree(
|
|||||||
|
|
||||||
if _is_skills_path(normalized_path):
|
if _is_skills_path(normalized_path):
|
||||||
entries = await asyncio.to_thread(skills_backend.ls_info, _strip_skills_prefix(normalized_path))
|
entries = await asyncio.to_thread(skills_backend.ls_info, _strip_skills_prefix(normalized_path))
|
||||||
return {"entries": [_remap_skills_entry(entry) for entry in entries]}
|
return {"entries": [_remap_prefixed_entry(entry, SKILLS_PATH) for entry in entries]}
|
||||||
|
if _is_kbs_path(normalized_path):
|
||||||
|
entries = await asyncio.to_thread(kb_backend.ls_info, _strip_kbs_prefix(normalized_path))
|
||||||
|
return {"entries": [_remap_prefixed_entry(entry, KBS_PATH) for entry in entries]}
|
||||||
except PermissionError as e:
|
except PermissionError as e:
|
||||||
raise HTTPException(status_code=400, detail=str(e)) from e
|
raise HTTPException(status_code=400, detail=str(e)) from e
|
||||||
except ValueError as e:
|
except ValueError as e:
|
||||||
@ -150,7 +168,7 @@ async def read_viewer_file_content(
|
|||||||
raise HTTPException(status_code=422, detail="thread_id 不能为空")
|
raise HTTPException(status_code=422, detail="thread_id 不能为空")
|
||||||
normalized_path = _normalize_path(path)
|
normalized_path = _normalize_path(path)
|
||||||
|
|
||||||
sandbox_backend, skills_backend, _selected_skills = await _resolve_viewer_state(
|
sandbox_backend, skills_backend, kb_backend, _selected_skills = await _resolve_viewer_state(
|
||||||
thread_id=thread_id,
|
thread_id=thread_id,
|
||||||
agent_id=agent_id,
|
agent_id=agent_id,
|
||||||
agent_config_id=agent_config_id,
|
agent_config_id=agent_config_id,
|
||||||
@ -163,6 +181,8 @@ async def read_viewer_file_content(
|
|||||||
responses = await asyncio.to_thread(sandbox_backend.download_files, [normalized_path])
|
responses = await asyncio.to_thread(sandbox_backend.download_files, [normalized_path])
|
||||||
elif _is_skills_path(normalized_path):
|
elif _is_skills_path(normalized_path):
|
||||||
responses = await asyncio.to_thread(skills_backend.download_files, [_strip_skills_prefix(normalized_path)])
|
responses = await asyncio.to_thread(skills_backend.download_files, [_strip_skills_prefix(normalized_path)])
|
||||||
|
elif _is_kbs_path(normalized_path):
|
||||||
|
responses = await asyncio.to_thread(kb_backend.download_files, [_strip_kbs_prefix(normalized_path)])
|
||||||
else:
|
else:
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=400,
|
status_code=400,
|
||||||
@ -199,7 +219,7 @@ async def download_viewer_file(
|
|||||||
db: AsyncSession,
|
db: AsyncSession,
|
||||||
) -> StreamingResponse:
|
) -> StreamingResponse:
|
||||||
normalized_path = _normalize_path(path)
|
normalized_path = _normalize_path(path)
|
||||||
_sandbox_backend, skills_backend, _selected_skills = await _resolve_viewer_state(
|
_sandbox_backend, skills_backend, kb_backend, _selected_skills = await _resolve_viewer_state(
|
||||||
thread_id=thread_id,
|
thread_id=thread_id,
|
||||||
agent_id=agent_id,
|
agent_id=agent_id,
|
||||||
agent_config_id=agent_config_id,
|
agent_config_id=agent_config_id,
|
||||||
@ -224,6 +244,8 @@ async def download_viewer_file(
|
|||||||
|
|
||||||
if _is_skills_path(normalized_path):
|
if _is_skills_path(normalized_path):
|
||||||
responses = await asyncio.to_thread(skills_backend.download_files, [_strip_skills_prefix(normalized_path)])
|
responses = await asyncio.to_thread(skills_backend.download_files, [_strip_skills_prefix(normalized_path)])
|
||||||
|
elif _is_kbs_path(normalized_path):
|
||||||
|
responses = await asyncio.to_thread(kb_backend.download_files, [_strip_kbs_prefix(normalized_path)])
|
||||||
else:
|
else:
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=400,
|
status_code=400,
|
||||||
|
|||||||
@ -2,10 +2,10 @@
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import importlib
|
||||||
import uuid
|
import uuid
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from yuxi.agents.backends.sandbox import ensure_thread_dirs, sandbox_workspace_dir, virtual_path_for_thread_file
|
from yuxi.agents.backends.sandbox import ensure_thread_dirs, sandbox_workspace_dir, virtual_path_for_thread_file
|
||||||
|
|
||||||
pytestmark = [pytest.mark.asyncio, pytest.mark.integration]
|
pytestmark = [pytest.mark.asyncio, pytest.mark.integration]
|
||||||
@ -44,7 +44,13 @@ async def _create_thread_for_user(test_client, headers: dict[str, str]) -> str:
|
|||||||
return thread_id
|
return thread_id
|
||||||
|
|
||||||
|
|
||||||
async def _upload_attachment_file(test_client, thread_id: str, headers: dict[str, str], file_name: str, content: str) -> str:
|
async def _upload_attachment_file(
|
||||||
|
test_client,
|
||||||
|
thread_id: str,
|
||||||
|
headers: dict[str, str],
|
||||||
|
file_name: str,
|
||||||
|
content: str,
|
||||||
|
) -> str:
|
||||||
response = await test_client.post(
|
response = await test_client.post(
|
||||||
f"/api/chat/thread/{thread_id}/attachments",
|
f"/api/chat/thread/{thread_id}/attachments",
|
||||||
files={"file": (file_name, content.encode("utf-8"), "text/plain")},
|
files={"file": (file_name, content.encode("utf-8"), "text/plain")},
|
||||||
@ -110,6 +116,7 @@ async def test_viewer_download_returns_attachment_response(test_client, standard
|
|||||||
assert "download_demo" in content_disposition
|
assert "download_demo" in content_disposition
|
||||||
assert "download-me" in response.text
|
assert "download-me" in response.text
|
||||||
|
|
||||||
|
|
||||||
async def test_viewer_download_returns_full_file_for_large_user_data_content(test_client, standard_user):
|
async def test_viewer_download_returns_full_file_for_large_user_data_content(test_client, standard_user):
|
||||||
headers = standard_user["headers"]
|
headers = standard_user["headers"]
|
||||||
thread_id = await _create_thread_for_user(test_client, headers)
|
thread_id = await _create_thread_for_user(test_client, headers)
|
||||||
@ -128,3 +135,47 @@ async def test_viewer_download_returns_full_file_for_large_user_data_content(tes
|
|||||||
assert response.status_code == 200, response.text
|
assert response.status_code == 200, response.text
|
||||||
assert response.content == large_content.encode("utf-8")
|
assert response.content == large_content.encode("utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
async def test_viewer_tree_root_lists_kbs_namespace_when_visible(test_client, standard_user, monkeypatch):
|
||||||
|
headers = standard_user["headers"]
|
||||||
|
thread_id = await _create_thread_for_user(test_client, headers)
|
||||||
|
|
||||||
|
async def _fake_list_viewer_filesystem_tree(**kwargs):
|
||||||
|
return {
|
||||||
|
"entries": [
|
||||||
|
{"path": "/home/yuxi/user-data/", "name": "user-data", "is_dir": True, "size": 0, "modified_at": ""},
|
||||||
|
{"path": "/home/yuxi/kbs/", "name": "kbs", "is_dir": True, "size": 0, "modified_at": ""},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
router_module = importlib.import_module("server.routers.filesystem_router")
|
||||||
|
monkeypatch.setitem(
|
||||||
|
router_module.get_viewer_tree.__globals__,
|
||||||
|
"list_viewer_filesystem_tree",
|
||||||
|
_fake_list_viewer_filesystem_tree,
|
||||||
|
)
|
||||||
|
|
||||||
|
response = await test_client.get(
|
||||||
|
"/api/viewer/filesystem/tree",
|
||||||
|
params={"thread_id": thread_id, "path": "/"},
|
||||||
|
headers=headers,
|
||||||
|
)
|
||||||
|
assert response.status_code == 200, response.text
|
||||||
|
|
||||||
|
entries = response.json().get("entries", [])
|
||||||
|
paths = {entry.get("path") for entry in entries}
|
||||||
|
assert "/home/yuxi/kbs/" in paths
|
||||||
|
|
||||||
|
|
||||||
|
async def test_viewer_can_list_and_read_kbs_namespace(test_client, standard_user):
|
||||||
|
headers = standard_user["headers"]
|
||||||
|
thread_id = await _create_thread_for_user(test_client, headers)
|
||||||
|
|
||||||
|
tree_response = await test_client.get(
|
||||||
|
"/api/viewer/filesystem/tree",
|
||||||
|
params={"thread_id": thread_id, "path": "/home/yuxi/kbs"},
|
||||||
|
headers=headers,
|
||||||
|
)
|
||||||
|
assert tree_response.status_code == 200, tree_response.text
|
||||||
|
entries = tree_response.json().get("entries", [])
|
||||||
|
assert any(str(entry.get("path", "")).startswith("/home/yuxi/kbs/") for entry in entries)
|
||||||
|
|||||||
106
backend/test/test_knowledge_base_backend.py
Normal file
106
backend/test/test_knowledge_base_backend.py
Normal file
@ -0,0 +1,106 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from yuxi.agents.backends.knowledge_base_backend import (
|
||||||
|
KnowledgeBaseReadonlyBackend,
|
||||||
|
resolve_visible_knowledge_bases_for_context,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_knowledge_base_backend_builds_virtual_tree_and_materializes_files(monkeypatch, tmp_path) -> None:
|
||||||
|
visible_kbs = [
|
||||||
|
{"db_id": "db-1", "name": "FAQ"},
|
||||||
|
{"db_id": "db-2", "name": "FAQ"},
|
||||||
|
]
|
||||||
|
files_meta = {
|
||||||
|
"folder-api": {
|
||||||
|
"file_id": "folder-api",
|
||||||
|
"database_id": "db-1",
|
||||||
|
"parent_id": None,
|
||||||
|
"filename": "API",
|
||||||
|
"is_folder": True,
|
||||||
|
"created_at": "2026-03-26T00:00:00Z",
|
||||||
|
},
|
||||||
|
"file-pdf": {
|
||||||
|
"file_id": "file-pdf",
|
||||||
|
"database_id": "db-1",
|
||||||
|
"parent_id": "folder-api",
|
||||||
|
"filename": "auth-guide.pdf",
|
||||||
|
"path": "http://minio/kb-source/db-1/upload/auth-guide.pdf",
|
||||||
|
"markdown_file": "http://minio/kb-parsed/db-1/parsed/file-pdf.md",
|
||||||
|
"size": 12,
|
||||||
|
"is_folder": False,
|
||||||
|
"created_at": "2026-03-26T00:00:00Z",
|
||||||
|
},
|
||||||
|
"file-pdf-2": {
|
||||||
|
"file_id": "file-pdf-2",
|
||||||
|
"database_id": "db-1",
|
||||||
|
"parent_id": "folder-api",
|
||||||
|
"filename": "auth-guide.pdf",
|
||||||
|
"path": "http://minio/kb-source/db-1/upload/auth-guide-2.pdf",
|
||||||
|
"size": 7,
|
||||||
|
"is_folder": False,
|
||||||
|
"created_at": "2026-03-26T00:00:00Z",
|
||||||
|
},
|
||||||
|
"file-db2": {
|
||||||
|
"file_id": "file-db2",
|
||||||
|
"database_id": "db-2",
|
||||||
|
"parent_id": None,
|
||||||
|
"filename": "overview.txt",
|
||||||
|
"path": "http://minio/kb-source/db-2/upload/overview.txt",
|
||||||
|
"size": 5,
|
||||||
|
"is_folder": False,
|
||||||
|
"created_at": "2026-03-26T00:00:00Z",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
monkeypatch.setattr("yuxi.agents.backends.knowledge_base_backend._all_files_meta", lambda: files_meta)
|
||||||
|
|
||||||
|
class _FakeMinio:
|
||||||
|
def download_file(self, bucket_name: str, object_name: str) -> bytes:
|
||||||
|
return f"{bucket_name}:{object_name}".encode()
|
||||||
|
|
||||||
|
monkeypatch.setattr("yuxi.agents.backends.knowledge_base_backend.get_minio_client", lambda: _FakeMinio())
|
||||||
|
|
||||||
|
backend = KnowledgeBaseReadonlyBackend(visible_kbs=visible_kbs, cache_root=tmp_path)
|
||||||
|
|
||||||
|
root_entries = {entry["path"] for entry in backend.ls_info("/")}
|
||||||
|
assert "/FAQ/" in root_entries
|
||||||
|
assert "/FAQ__db-2/" in root_entries
|
||||||
|
|
||||||
|
faq_entries = {entry["path"] for entry in backend.ls_info("/FAQ")}
|
||||||
|
assert "/FAQ/API/" in faq_entries
|
||||||
|
assert "/FAQ/parsed/" in faq_entries
|
||||||
|
|
||||||
|
api_entries = {entry["path"] for entry in backend.ls_info("/FAQ/API")}
|
||||||
|
assert "/FAQ/API/auth-guide.pdf" in api_entries
|
||||||
|
assert any(path.startswith("/FAQ/API/auth-guide.pdf__file-pdf") for path in api_entries)
|
||||||
|
|
||||||
|
parsed_entries = {entry["path"] for entry in backend.ls_info("/FAQ/parsed/API")}
|
||||||
|
assert "/FAQ/parsed/API/auth-guide.md" in parsed_entries
|
||||||
|
|
||||||
|
responses = backend.download_files(["/FAQ/API/auth-guide.pdf", "/FAQ/parsed/API/auth-guide.md"])
|
||||||
|
assert responses[0].content == b"kb-source:db-1/upload/auth-guide.pdf"
|
||||||
|
assert responses[1].content == b"kb-parsed:db-1/parsed/file-pdf.md"
|
||||||
|
|
||||||
|
assert "kb-source" in backend.read("/FAQ/API/auth-guide.pdf")
|
||||||
|
assert backend.write("/FAQ/API/new.txt", "x").error == "Knowledge base path is read-only."
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_resolve_visible_knowledge_bases_for_context_filters_by_enabled_names(monkeypatch) -> None:
|
||||||
|
async def _fake_get_databases_by_raw_id(user_id: int) -> dict:
|
||||||
|
assert user_id == 7
|
||||||
|
return {"databases": [{"db_id": "db-1", "name": "Alpha"}, {"db_id": "db-2", "name": "Beta"}]}
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"yuxi.agents.backends.knowledge_base_backend.knowledge_base.get_databases_by_raw_id",
|
||||||
|
_fake_get_databases_by_raw_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
context = SimpleNamespace(user_id="7", knowledges=["Beta"])
|
||||||
|
visible = await resolve_visible_knowledge_bases_for_context(context)
|
||||||
|
|
||||||
|
assert visible == [{"db_id": "db-2", "name": "Beta"}]
|
||||||
|
assert getattr(context, "_visible_knowledge_bases") == visible
|
||||||
@ -5,29 +5,36 @@ from __future__ import annotations
|
|||||||
from types import MethodType, SimpleNamespace
|
from types import MethodType, SimpleNamespace
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from yuxi.agents.backends.composite import create_agent_composite_backend
|
from yuxi.agents.backends.composite import create_agent_composite_backend
|
||||||
from yuxi.agents.backends.sandbox import resolve_virtual_path, sandbox_id_for_thread
|
from yuxi.agents.backends.sandbox import resolve_virtual_path, sandbox_id_for_thread
|
||||||
from yuxi.agents.backends.sandbox.backend import ProvisionerSandboxBackend
|
from yuxi.agents.backends.sandbox.backend import ProvisionerSandboxBackend
|
||||||
from yuxi.agents.middlewares.skills_middleware import SkillsMiddleware
|
from yuxi.agents.middlewares.skills_middleware import SkillsMiddleware
|
||||||
|
|
||||||
|
|
||||||
def _runtime(*, thread_id: str | None = "thread-1", skills: list[str] | None = None):
|
def _runtime(
|
||||||
|
*,
|
||||||
|
thread_id: str | None = "thread-1",
|
||||||
|
skills: list[str] | None = None,
|
||||||
|
visible_kbs: list[dict] | None = None,
|
||||||
|
):
|
||||||
configurable = {"thread_id": thread_id} if thread_id else {}
|
configurable = {"thread_id": thread_id} if thread_id else {}
|
||||||
return SimpleNamespace(
|
return SimpleNamespace(
|
||||||
config={"configurable": configurable},
|
config={"configurable": configurable},
|
||||||
context=SimpleNamespace(skills=skills or []),
|
context=SimpleNamespace(skills=skills or [], _visible_knowledge_bases=visible_kbs or []),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_create_agent_composite_backend_uses_provisioner_default(monkeypatch):
|
def test_create_agent_composite_backend_uses_provisioner_default(monkeypatch):
|
||||||
monkeypatch.setattr("yuxi.agents.backends.sandbox.backend.get_sandbox_provider", lambda: object())
|
monkeypatch.setattr("yuxi.agents.backends.sandbox.backend.get_sandbox_provider", lambda: object())
|
||||||
|
|
||||||
backend = create_agent_composite_backend(_runtime(skills=["reporter"]))
|
backend = create_agent_composite_backend(
|
||||||
|
_runtime(skills=["reporter"], visible_kbs=[{"db_id": "db-1", "name": "Docs"}])
|
||||||
|
)
|
||||||
|
|
||||||
assert isinstance(backend.default, ProvisionerSandboxBackend)
|
assert isinstance(backend.default, ProvisionerSandboxBackend)
|
||||||
assert backend.default._visible_skills == ["reporter"]
|
assert backend.default._visible_skills == ["reporter"]
|
||||||
assert "/skills/" in backend.routes
|
assert "/skills/" in backend.routes
|
||||||
|
assert "/home/yuxi/kbs/" in backend.routes
|
||||||
|
|
||||||
|
|
||||||
def test_create_agent_composite_backend_requires_thread_id():
|
def test_create_agent_composite_backend_requires_thread_id():
|
||||||
|
|||||||
@ -45,6 +45,8 @@
|
|||||||
|
|
||||||
### 修复
|
### 修复
|
||||||
|
|
||||||
|
- 为沙盒与 viewer 文件系统补齐知识库只读映射:新增 `/home/yuxi/kbs` 命名空间,按“用户可访问知识库 ∩ 当前 Agent 已启用知识库”暴露原始文件与解析后的 Markdown,并补充对应后端与 viewer 路由测试
|
||||||
|
|
||||||
<!-- 添加到这里 -->
|
<!-- 添加到这里 -->
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user