feat: 添加知识库文件在沙盒以及文件系统中的映射 #576
This commit is contained in:
parent
a2f35cbeb2
commit
2be5aedaf2
@ -1,6 +1,7 @@
|
||||
from deepagents.backends import CompositeBackend, StateBackend
|
||||
|
||||
from .composite import create_agent_composite_backend
|
||||
from .knowledge_base_backend import KBS_PATH, KnowledgeBaseReadonlyBackend, resolve_visible_knowledge_bases_for_context
|
||||
from .sandbox import (
|
||||
IDLE_CHECK_INTERVAL,
|
||||
LARGE_TOOL_RESULTS_DIR,
|
||||
@ -33,6 +34,8 @@ from .skills_backend import SelectedSkillsReadonlyBackend
|
||||
|
||||
__all__ = [
|
||||
"CompositeBackend",
|
||||
"KBS_PATH",
|
||||
"KnowledgeBaseReadonlyBackend",
|
||||
"StateBackend",
|
||||
"SelectedSkillsReadonlyBackend",
|
||||
"create_agent_composite_backend",
|
||||
@ -47,6 +50,7 @@ __all__ = [
|
||||
"init_sandbox_provider",
|
||||
"shutdown_sandbox_provider",
|
||||
"resolve_virtual_path",
|
||||
"resolve_visible_knowledge_bases_for_context",
|
||||
"virtual_path_for_thread_file",
|
||||
"sandbox_id_for_thread",
|
||||
"sandbox_user_data_dir",
|
||||
|
||||
@ -2,13 +2,15 @@ from __future__ import annotations
|
||||
|
||||
from deepagents.backends.composite import (
|
||||
CompositeBackend,
|
||||
_route_for_path,
|
||||
_remap_file_info_path,
|
||||
_route_for_path,
|
||||
_strip_route_from_pattern,
|
||||
)
|
||||
from deepagents.backends.protocol import FileInfo
|
||||
|
||||
from yuxi.agents.middlewares.skills_middleware import normalize_selected_skills
|
||||
|
||||
from .knowledge_base_backend import KnowledgeBaseReadonlyBackend
|
||||
from .sandbox import ProvisionerSandboxBackend
|
||||
from .skills_backend import SelectedSkillsReadonlyBackend
|
||||
|
||||
@ -92,12 +94,22 @@ def _extract_thread_id(runtime) -> str:
|
||||
raise ValueError("thread_id is required in runtime configurable context")
|
||||
|
||||
|
||||
def _get_visible_knowledge_bases_from_runtime(runtime) -> list[dict]:
|
||||
context = getattr(runtime, "context", None)
|
||||
selected = getattr(context, "_visible_knowledge_bases", None)
|
||||
if isinstance(selected, list):
|
||||
return selected
|
||||
return []
|
||||
|
||||
|
||||
def create_agent_composite_backend(runtime) -> CompositeBackend:
|
||||
visible_skills = _get_visible_skills_from_runtime(runtime)
|
||||
thread_id = _extract_thread_id(runtime)
|
||||
visible_kbs = _get_visible_knowledge_bases_from_runtime(runtime)
|
||||
return CustomCompositeBackend(
|
||||
default=ProvisionerSandboxBackend(thread_id=thread_id, visible_skills=visible_skills),
|
||||
routes={
|
||||
"/skills/": SelectedSkillsReadonlyBackend(selected_slugs=visible_skills),
|
||||
"/home/yuxi/kbs/": KnowledgeBaseReadonlyBackend(visible_kbs=visible_kbs),
|
||||
},
|
||||
)
|
||||
|
||||
459
backend/package/yuxi/agents/backends/knowledge_base_backend.py
Normal file
459
backend/package/yuxi/agents/backends/knowledge_base_backend.py
Normal file
@ -0,0 +1,459 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import fnmatch
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any
|
||||
|
||||
from deepagents.backends import FilesystemBackend
|
||||
from deepagents.backends.protocol import EditResult, FileDownloadResponse, FileInfo, FileUploadResponse, WriteResult
|
||||
|
||||
from yuxi import config as conf
|
||||
from yuxi import knowledge_base
|
||||
from yuxi.knowledge.utils.kb_utils import is_minio_url, parse_minio_url
|
||||
from yuxi.storage.minio import get_minio_client
|
||||
|
||||
KBS_PATH = "/home/yuxi/kbs"
|
||||
_INVALID_SEGMENT_RE = re.compile(r"[\\/\x00-\x1f\x7f]+")
|
||||
_WHITESPACE_RE = re.compile(r"\s+")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _MaterializedFile:
|
||||
virtual_path: str
|
||||
cache_path: Path
|
||||
source_path: str
|
||||
modified_at: str | None = None
|
||||
|
||||
|
||||
def _normalize_virtual_path(path: str | None) -> str:
|
||||
raw = str(path or "").strip() or "/"
|
||||
normalized = "/" + raw.lstrip("/")
|
||||
pure = PurePosixPath(normalized)
|
||||
if ".." in pure.parts:
|
||||
raise ValueError("path traversal is not allowed")
|
||||
return str(pure)
|
||||
|
||||
|
||||
def _sanitize_segment(value: str | None, fallback: str) -> str:
|
||||
cleaned = str(value or "").strip()
|
||||
cleaned = _INVALID_SEGMENT_RE.sub("_", cleaned)
|
||||
cleaned = _WHITESPACE_RE.sub(" ", cleaned).strip()
|
||||
if not cleaned or cleaned in {".", ".."}:
|
||||
return fallback
|
||||
return cleaned
|
||||
|
||||
|
||||
def _candidate_name(file_meta: dict[str, Any]) -> str:
|
||||
filename = str(file_meta.get("filename") or "").strip()
|
||||
if filename:
|
||||
return filename
|
||||
|
||||
original = str(file_meta.get("original_filename") or "").strip()
|
||||
if original:
|
||||
return original
|
||||
|
||||
for key in ("path", "markdown_file"):
|
||||
raw = str(file_meta.get(key) or "").strip()
|
||||
if raw:
|
||||
try:
|
||||
parsed = PurePosixPath(raw)
|
||||
name = parsed.name
|
||||
except Exception: # noqa: BLE001
|
||||
name = ""
|
||||
if name:
|
||||
return name
|
||||
|
||||
return str(file_meta.get("file_id") or "")
|
||||
|
||||
|
||||
def _unique_name(base_name: str, *, stable_id: str, used_names: set[str]) -> str:
|
||||
if base_name not in used_names:
|
||||
used_names.add(base_name)
|
||||
return base_name
|
||||
|
||||
suffix = f"__{stable_id[:8]}"
|
||||
candidate = f"{base_name}{suffix}"
|
||||
while candidate in used_names:
|
||||
suffix = f"__{stable_id}"
|
||||
candidate = f"{base_name}{suffix}"
|
||||
used_names.add(candidate)
|
||||
return candidate
|
||||
|
||||
|
||||
def _materialize_text_view(content: bytes, file_path: str, *, offset: int = 0, limit: int = 2000) -> str:
|
||||
if not content:
|
||||
return "System reminder: File exists but has empty contents"
|
||||
|
||||
if b"\x00" in content:
|
||||
return f"Error: File '{file_path}' is binary and cannot be rendered as text"
|
||||
|
||||
try:
|
||||
text = content.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
return f"Error: File '{file_path}' is binary and cannot be rendered as text"
|
||||
|
||||
lines = text.splitlines()
|
||||
start = max(0, int(offset))
|
||||
selected = lines[start : start + int(limit)]
|
||||
return "\n".join(f"{start + idx + 1:6d}\t{line}" for idx, line in enumerate(selected))
|
||||
|
||||
|
||||
async def resolve_visible_knowledge_bases_for_context(context) -> list[dict[str, Any]]:
|
||||
user_id = getattr(context, "user_id", None)
|
||||
if not user_id:
|
||||
setattr(context, "_visible_knowledge_bases", [])
|
||||
return []
|
||||
|
||||
enabled_names = {str(name).strip() for name in (getattr(context, "knowledges", None) or []) if str(name).strip()}
|
||||
if not enabled_names:
|
||||
setattr(context, "_visible_knowledge_bases", [])
|
||||
return []
|
||||
|
||||
try:
|
||||
raw_user_id = int(user_id)
|
||||
except (TypeError, ValueError):
|
||||
setattr(context, "_visible_knowledge_bases", [])
|
||||
return []
|
||||
|
||||
result = await knowledge_base.get_databases_by_raw_id(raw_user_id)
|
||||
databases = [db for db in (result.get("databases") or []) if str(db.get("name") or "").strip() in enabled_names]
|
||||
setattr(context, "_visible_knowledge_bases", databases)
|
||||
return databases
|
||||
|
||||
|
||||
def _all_files_meta() -> dict[str, dict[str, Any]]:
|
||||
aggregated: dict[str, dict[str, Any]] = {}
|
||||
for kb_instance in getattr(knowledge_base, "kb_instances", {}).values():
|
||||
for file_id, meta in getattr(kb_instance, "files_meta", {}).items():
|
||||
aggregated[str(file_id)] = meta
|
||||
return aggregated
|
||||
|
||||
|
||||
class KnowledgeBaseReadonlyBackend(FilesystemBackend):
|
||||
def __init__(self, *, visible_kbs: list[dict[str, Any]] | None, cache_root: Path | str | None = None):
|
||||
self._cache_root = Path(cache_root or (Path(conf.save_dir) / "knowledge_base_data" / "kb-cache")).resolve()
|
||||
self._cache_root.mkdir(parents=True, exist_ok=True)
|
||||
super().__init__(root_dir=self._cache_root, virtual_mode=True)
|
||||
self._visible_kbs = list(visible_kbs or [])
|
||||
self._entries_by_dir: dict[str, list[FileInfo]] = defaultdict(list)
|
||||
self._dir_paths: set[str] = {"/"}
|
||||
self._files: dict[str, _MaterializedFile] = {}
|
||||
self._all_files: list[FileInfo] = []
|
||||
self._build_virtual_tree()
|
||||
|
||||
def has_entries(self) -> bool:
|
||||
return bool(self._visible_kbs)
|
||||
|
||||
def _build_virtual_tree(self) -> None:
|
||||
kb_name_candidates = {
|
||||
str(db.get("db_id") or db.get("name") or f"kb-{index}"): _sanitize_segment(
|
||||
db.get("name"),
|
||||
str(db.get("db_id") or f"kb-{index}"),
|
||||
)
|
||||
for index, db in enumerate(self._visible_kbs)
|
||||
}
|
||||
used_root_names: set[str] = set()
|
||||
kb_virtual_names: dict[str, str] = {}
|
||||
sorted_kbs = sorted(
|
||||
self._visible_kbs,
|
||||
key=lambda item: (str(item.get("name") or ""), str(item.get("db_id") or "")),
|
||||
)
|
||||
for db in sorted_kbs:
|
||||
db_id = str(db.get("db_id") or "")
|
||||
if not db_id:
|
||||
continue
|
||||
base_name = kb_name_candidates.get(db_id) or db_id
|
||||
kb_virtual_names[db_id] = _unique_name(base_name, stable_id=db_id, used_names=used_root_names)
|
||||
|
||||
files_by_db: dict[str, dict[str, dict[str, Any]]] = defaultdict(dict)
|
||||
for file_id, meta in _all_files_meta().items():
|
||||
db_id = str(meta.get("database_id") or "")
|
||||
if db_id in kb_virtual_names:
|
||||
files_by_db[db_id][str(file_id)] = meta
|
||||
|
||||
for db_id, kb_virtual_name in kb_virtual_names.items():
|
||||
kb_root = f"/{kb_virtual_name}"
|
||||
self._add_entry("/", kb_root, is_dir=True)
|
||||
records = files_by_db.get(db_id, {})
|
||||
self._build_source_tree(db_id=db_id, kb_root=kb_root, records=records)
|
||||
self._build_parsed_tree(db_id=db_id, kb_root=kb_root, records=records)
|
||||
|
||||
for path, entries in list(self._entries_by_dir.items()):
|
||||
entries.sort(key=lambda item: str(item.get("path") or ""))
|
||||
self._entries_by_dir[path] = entries
|
||||
self._all_files = sorted(self._all_files, key=lambda item: str(item.get("path") or ""))
|
||||
|
||||
def _build_source_tree(self, *, db_id: str, kb_root: str, records: dict[str, dict[str, Any]]) -> None:
|
||||
children_by_parent: dict[str | None, list[dict[str, Any]]] = defaultdict(list)
|
||||
for file_id, meta in records.items():
|
||||
parent_id = meta.get("parent_id")
|
||||
children_by_parent[str(parent_id) if parent_id else None].append({"file_id": file_id, "meta": meta})
|
||||
|
||||
resolved_parent_paths: dict[str, str] = {}
|
||||
resolved_source_names: dict[str, str] = {}
|
||||
|
||||
def walk(parent_id: str | None, parent_path: str) -> None:
|
||||
siblings = children_by_parent.get(parent_id, [])
|
||||
used_names: set[str] = set()
|
||||
candidates: list[tuple[dict[str, Any], str]] = []
|
||||
for item in siblings:
|
||||
file_id = item["file_id"]
|
||||
meta = item["meta"]
|
||||
base_name = _sanitize_segment(_candidate_name(meta), file_id)
|
||||
candidates.append((item, base_name))
|
||||
|
||||
for item, base_name in sorted(candidates, key=lambda pair: (pair[1], pair[0]["file_id"])):
|
||||
file_id = item["file_id"]
|
||||
meta = item["meta"]
|
||||
unique_name = _unique_name(base_name, stable_id=file_id, used_names=used_names)
|
||||
child_path = f"{parent_path.rstrip('/')}/{unique_name}" if parent_path != "/" else f"/{unique_name}"
|
||||
if meta.get("is_folder"):
|
||||
modified_at = meta.get("updated_at") or meta.get("created_at")
|
||||
self._add_entry(parent_path, child_path, is_dir=True, modified_at=modified_at)
|
||||
walk(file_id, child_path)
|
||||
continue
|
||||
|
||||
self._add_entry(
|
||||
parent_path,
|
||||
child_path,
|
||||
is_dir=False,
|
||||
size=int(meta.get("size") or 0),
|
||||
modified_at=meta.get("updated_at") or meta.get("created_at"),
|
||||
)
|
||||
resolved_parent_paths[file_id] = parent_path
|
||||
resolved_source_names[file_id] = unique_name
|
||||
cache_path = self._cache_root / db_id / "source" / file_id / unique_name
|
||||
self._files[child_path] = _MaterializedFile(
|
||||
virtual_path=child_path,
|
||||
cache_path=cache_path,
|
||||
source_path=str(meta.get("path") or ""),
|
||||
modified_at=meta.get("updated_at") or meta.get("created_at"),
|
||||
)
|
||||
self._all_files.append(
|
||||
{
|
||||
"path": child_path,
|
||||
"is_dir": False,
|
||||
"size": int(meta.get("size") or 0),
|
||||
"modified_at": str(meta.get("updated_at") or meta.get("created_at") or ""),
|
||||
}
|
||||
)
|
||||
|
||||
walk(None, kb_root)
|
||||
self._resolved_parent_paths = getattr(self, "_resolved_parent_paths", {})
|
||||
self._resolved_parent_paths[db_id] = resolved_parent_paths
|
||||
self._resolved_source_names = getattr(self, "_resolved_source_names", {})
|
||||
self._resolved_source_names[db_id] = resolved_source_names
|
||||
|
||||
def _build_parsed_tree(self, *, db_id: str, kb_root: str, records: dict[str, dict[str, Any]]) -> None:
|
||||
parsed_records = {
|
||||
file_id: meta
|
||||
for file_id, meta in records.items()
|
||||
if not meta.get("is_folder") and str(meta.get("markdown_file") or "").strip()
|
||||
}
|
||||
if not parsed_records:
|
||||
return
|
||||
|
||||
parsed_root = f"{kb_root}/parsed"
|
||||
self._add_entry(kb_root, parsed_root, is_dir=True)
|
||||
parent_paths = getattr(self, "_resolved_parent_paths", {}).get(db_id, {})
|
||||
source_names = getattr(self, "_resolved_source_names", {}).get(db_id, {})
|
||||
|
||||
grouped: dict[str, list[tuple[str, dict[str, Any], str]]] = defaultdict(list)
|
||||
for file_id, meta in parsed_records.items():
|
||||
source_parent = parent_paths.get(file_id, kb_root)
|
||||
parsed_parent = (
|
||||
f"{parsed_root}{source_parent[len(kb_root) :]}" if source_parent.startswith(kb_root) else parsed_root
|
||||
)
|
||||
source_name = source_names.get(file_id) or _sanitize_segment(meta.get("filename"), file_id)
|
||||
source_stem = PurePosixPath(source_name).stem or source_name or file_id
|
||||
base_name = _sanitize_segment(f"{source_stem}.md", f"{file_id}.md")
|
||||
grouped[parsed_parent].append((file_id, meta, base_name))
|
||||
|
||||
for parsed_parent, items in grouped.items():
|
||||
current = PurePosixPath(parsed_parent)
|
||||
while str(current) not in {"", "."} and str(current) != "/":
|
||||
current_str = str(current)
|
||||
parent_str = str(current.parent) if str(current.parent) != "." else "/"
|
||||
if current_str not in self._dir_paths:
|
||||
self._add_entry(parent_str, current_str, is_dir=True)
|
||||
current = current.parent
|
||||
|
||||
used_names: set[str] = set()
|
||||
for file_id, meta, base_name in sorted(items, key=lambda item: (item[2], item[0])):
|
||||
unique_name = _unique_name(base_name, stable_id=file_id, used_names=used_names)
|
||||
file_path = f"{parsed_parent.rstrip('/')}/{unique_name}" if parsed_parent != "/" else f"/{unique_name}"
|
||||
self._add_entry(
|
||||
parsed_parent,
|
||||
file_path,
|
||||
is_dir=False,
|
||||
size=0,
|
||||
modified_at=meta.get("updated_at") or meta.get("created_at"),
|
||||
)
|
||||
cache_path = self._cache_root / db_id / "parsed" / f"{file_id}.md"
|
||||
self._files[file_path] = _MaterializedFile(
|
||||
virtual_path=file_path,
|
||||
cache_path=cache_path,
|
||||
source_path=str(meta.get("markdown_file") or ""),
|
||||
modified_at=meta.get("updated_at") or meta.get("created_at"),
|
||||
)
|
||||
self._all_files.append(
|
||||
{
|
||||
"path": file_path,
|
||||
"is_dir": False,
|
||||
"size": 0,
|
||||
"modified_at": str(meta.get("updated_at") or meta.get("created_at") or ""),
|
||||
}
|
||||
)
|
||||
|
||||
def _add_entry(
|
||||
self,
|
||||
parent_path: str,
|
||||
child_path: str,
|
||||
*,
|
||||
is_dir: bool,
|
||||
size: int = 0,
|
||||
modified_at: str | None = None,
|
||||
) -> None:
|
||||
normalized_parent = _normalize_virtual_path(parent_path)
|
||||
normalized_child = _normalize_virtual_path(child_path)
|
||||
entry_path = f"{normalized_child}/" if is_dir else normalized_child
|
||||
entry: FileInfo = {
|
||||
"path": entry_path,
|
||||
"is_dir": is_dir,
|
||||
"size": int(size or 0),
|
||||
"modified_at": str(modified_at or ""),
|
||||
}
|
||||
if entry_path not in {str(item.get("path")) for item in self._entries_by_dir[normalized_parent]}:
|
||||
self._entries_by_dir[normalized_parent].append(entry)
|
||||
if is_dir:
|
||||
self._dir_paths.add(normalized_child)
|
||||
self._entries_by_dir.setdefault(normalized_child, [])
|
||||
|
||||
def _ensure_local_file(self, descriptor: _MaterializedFile) -> Path:
|
||||
if descriptor.cache_path.exists():
|
||||
return descriptor.cache_path
|
||||
|
||||
source = descriptor.source_path.strip()
|
||||
if not source:
|
||||
raise FileNotFoundError(descriptor.virtual_path)
|
||||
if not is_minio_url(source):
|
||||
raise FileNotFoundError(descriptor.virtual_path)
|
||||
|
||||
bucket_name, object_name = parse_minio_url(source)
|
||||
payload = get_minio_client().download_file(bucket_name, object_name)
|
||||
descriptor.cache_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
descriptor.cache_path.write_bytes(payload)
|
||||
return descriptor.cache_path
|
||||
|
||||
def ls_info(self, path: str) -> list[FileInfo]:
|
||||
normalized_path = _normalize_virtual_path(path)
|
||||
return list(self._entries_by_dir.get(normalized_path, []))
|
||||
|
||||
def read(self, file_path: str, offset: int = 0, limit: int = 2000) -> str:
|
||||
normalized_path = _normalize_virtual_path(file_path)
|
||||
descriptor = self._files.get(normalized_path)
|
||||
if descriptor is None:
|
||||
if normalized_path in self._dir_paths:
|
||||
return f"Error: Path '{file_path}' is a directory"
|
||||
return f"Error: File '{file_path}' not found"
|
||||
|
||||
try:
|
||||
content = self._ensure_local_file(descriptor).read_bytes()
|
||||
except FileNotFoundError:
|
||||
return f"Error: File '{file_path}' not found"
|
||||
except Exception as exc: # noqa: BLE001
|
||||
detail = str(exc).strip() or "unknown error"
|
||||
return f"Error: Failed to read '{file_path}': {detail}"
|
||||
|
||||
return _materialize_text_view(content, file_path, offset=offset, limit=limit)
|
||||
|
||||
def glob_info(self, pattern: str, path: str = "/") -> list[FileInfo]:
|
||||
normalized_path = _normalize_virtual_path(path)
|
||||
if ".." in PurePosixPath(pattern).parts:
|
||||
raise ValueError("Path traversal not allowed in glob pattern")
|
||||
normalized_pattern = pattern.lstrip("/") or "*"
|
||||
prefix = "/" if normalized_path == "/" else f"{normalized_path.rstrip('/')}/"
|
||||
|
||||
matches: list[FileInfo] = []
|
||||
for item in self._all_files:
|
||||
item_path = str(item.get("path") or "")
|
||||
if normalized_path != "/" and not item_path.startswith(prefix):
|
||||
continue
|
||||
relative = item_path[len(prefix) :] if normalized_path != "/" else item_path.lstrip("/")
|
||||
if fnmatch.fnmatch(relative, normalized_pattern):
|
||||
matches.append(dict(item))
|
||||
return matches
|
||||
|
||||
def grep_raw(self, pattern: str, path: str | None = None, glob: str | None = None) -> list[dict[str, Any]] | str:
|
||||
normalized_path = _normalize_virtual_path(path or "/")
|
||||
prefix = "/" if normalized_path == "/" else f"{normalized_path.rstrip('/')}/"
|
||||
if normalized_path in self._files:
|
||||
targets = [normalized_path]
|
||||
else:
|
||||
targets = [
|
||||
item["path"]
|
||||
for item in self._all_files
|
||||
if normalized_path == "/" or str(item["path"]).startswith(prefix)
|
||||
]
|
||||
|
||||
matches: list[dict[str, Any]] = []
|
||||
for target in targets:
|
||||
display_target = str(target)
|
||||
relative = display_target.lstrip("/") if normalized_path == "/" else display_target[len(prefix) :]
|
||||
if glob and not fnmatch.fnmatch(relative, glob):
|
||||
continue
|
||||
descriptor = self._files.get(display_target)
|
||||
if descriptor is None:
|
||||
continue
|
||||
try:
|
||||
content = self._ensure_local_file(descriptor).read_bytes()
|
||||
if b"\x00" in content:
|
||||
continue
|
||||
text = content.decode("utf-8")
|
||||
except Exception: # noqa: BLE001
|
||||
continue
|
||||
for line_num, line in enumerate(text.splitlines(), start=1):
|
||||
if pattern in line:
|
||||
matches.append({"path": display_target, "line": line_num, "text": line})
|
||||
return matches
|
||||
|
||||
def write(self, file_path: str, content: str) -> WriteResult:
|
||||
return WriteResult(error="Knowledge base path is read-only.")
|
||||
|
||||
def edit(self, file_path: str, old_string: str, new_string: str, replace_all: bool = False) -> EditResult:
|
||||
return EditResult(error="Knowledge base path is read-only.")
|
||||
|
||||
def upload_files(self, files: list[tuple[str, bytes]]) -> list[FileUploadResponse]:
|
||||
return [FileUploadResponse(path=path, error="permission_denied") for path, _content in files]
|
||||
|
||||
def download_files(self, paths: list[str]) -> list[FileDownloadResponse]:
|
||||
responses: list[FileDownloadResponse] = []
|
||||
for path in paths:
|
||||
try:
|
||||
normalized_path = _normalize_virtual_path(path)
|
||||
except ValueError:
|
||||
responses.append(FileDownloadResponse(path=path, content=None, error="invalid_path"))
|
||||
continue
|
||||
|
||||
descriptor = self._files.get(normalized_path)
|
||||
if descriptor is None:
|
||||
if normalized_path in self._dir_paths:
|
||||
responses.append(FileDownloadResponse(path=path, content=None, error="is_directory"))
|
||||
else:
|
||||
responses.append(FileDownloadResponse(path=path, content=None, error="file_not_found"))
|
||||
continue
|
||||
|
||||
try:
|
||||
content = self._ensure_local_file(descriptor).read_bytes()
|
||||
except FileNotFoundError:
|
||||
responses.append(FileDownloadResponse(path=path, content=None, error="file_not_found"))
|
||||
continue
|
||||
except ValueError:
|
||||
responses.append(FileDownloadResponse(path=path, content=None, error="invalid_path"))
|
||||
continue
|
||||
|
||||
responses.append(FileDownloadResponse(path=path, content=content, error=None))
|
||||
return responses
|
||||
@ -39,7 +39,6 @@ class ChatbotAgent(BaseAgent):
|
||||
"你好,请介绍一下你自己",
|
||||
"帮我写一封商务邮件",
|
||||
"解释一下什么是机器学习",
|
||||
"推荐几本好书",
|
||||
"创建一个冒泡排序 python 并保存结果",
|
||||
]
|
||||
}
|
||||
|
||||
@ -1,7 +1,10 @@
|
||||
"""知识库中间件 - 提供通用知识库工具"""
|
||||
|
||||
from langchain.agents.middleware import AgentMiddleware
|
||||
from collections.abc import Callable
|
||||
|
||||
from langchain.agents.middleware import AgentMiddleware, ModelRequest, ModelResponse
|
||||
|
||||
from yuxi.agents.backends.knowledge_base_backend import resolve_visible_knowledge_bases_for_context
|
||||
from yuxi.agents.toolkits.kbs import get_common_kb_tools
|
||||
from yuxi.utils.logging_config import logger
|
||||
|
||||
@ -21,3 +24,9 @@ class KnowledgeBaseMiddleware(AgentMiddleware):
|
||||
self.kb_tools = get_common_kb_tools()
|
||||
self.tools = self.kb_tools
|
||||
logger.debug(f"Initialized KnowledgeBaseMiddleware with {len(self.kb_tools)} tools")
|
||||
|
||||
async def awrap_model_call(
|
||||
self, request: ModelRequest, handler: Callable[[ModelRequest], ModelResponse]
|
||||
) -> ModelResponse:
|
||||
await resolve_visible_knowledge_bases_for_context(request.runtime.context)
|
||||
return await handler(request)
|
||||
|
||||
@ -7,6 +7,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from yuxi.agents.backends import (
|
||||
ProvisionerSandboxBackend,
|
||||
create_agent_composite_backend,
|
||||
resolve_visible_knowledge_bases_for_context,
|
||||
)
|
||||
from yuxi.agents.backends.sandbox.backend import _looks_like_binary
|
||||
from yuxi.agents.context import BaseContext
|
||||
@ -64,6 +65,7 @@ async def _resolve_filesystem_state(
|
||||
)
|
||||
runtime_context.thread_id = thread_id
|
||||
runtime_context.user_id = str(user.id)
|
||||
await resolve_visible_knowledge_bases_for_context(runtime_context)
|
||||
|
||||
sandbox_backend = ProvisionerSandboxBackend(thread_id=thread_id)
|
||||
return conversation, runtime_context, sandbox_backend
|
||||
|
||||
@ -9,6 +9,7 @@ from urllib.parse import quote
|
||||
from fastapi import HTTPException
|
||||
from fastapi.responses import FileResponse, StreamingResponse
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from yuxi.agents.backends import KBS_PATH, KnowledgeBaseReadonlyBackend, resolve_visible_knowledge_bases_for_context
|
||||
from yuxi.agents.backends.sandbox import SKILLS_PATH, USER_DATA_PATH, resolve_virtual_path
|
||||
from yuxi.agents.backends.skills_backend import SelectedSkillsReadonlyBackend
|
||||
from yuxi.agents.middlewares.skills_middleware import normalize_selected_skills
|
||||
@ -20,7 +21,7 @@ def _normalize_path(path: str | None) -> str:
|
||||
normalized = (path or "/").strip() or "/"
|
||||
if not normalized.startswith("/"):
|
||||
normalized = f"/{normalized}"
|
||||
return normalized.rstrip("/") if normalized not in {"/", SKILLS_PATH, USER_DATA_PATH} else normalized
|
||||
return normalized.rstrip("/") if normalized not in {"/", KBS_PATH, SKILLS_PATH, USER_DATA_PATH} else normalized
|
||||
|
||||
|
||||
def _is_user_data_path(path: str) -> bool:
|
||||
@ -31,16 +32,26 @@ def _is_skills_path(path: str) -> bool:
|
||||
return path == SKILLS_PATH or path.startswith(f"{SKILLS_PATH}/")
|
||||
|
||||
|
||||
def _is_kbs_path(path: str) -> bool:
|
||||
return path == KBS_PATH or path.startswith(f"{KBS_PATH}/")
|
||||
|
||||
|
||||
def _strip_skills_prefix(path: str) -> str:
|
||||
if path == SKILLS_PATH:
|
||||
return "/"
|
||||
return path[len(SKILLS_PATH) :] or "/"
|
||||
|
||||
|
||||
def _remap_skills_entry(entry: dict) -> dict:
|
||||
def _strip_kbs_prefix(path: str) -> str:
|
||||
if path == KBS_PATH:
|
||||
return "/"
|
||||
return path[len(KBS_PATH) :] or "/"
|
||||
|
||||
|
||||
def _remap_prefixed_entry(entry: dict, prefix: str) -> dict:
|
||||
raw_path = str(entry.get("path") or "")
|
||||
is_dir = bool(entry.get("is_dir", False))
|
||||
remapped = f"{SKILLS_PATH}{raw_path}" if raw_path != "/" else f"{SKILLS_PATH}/"
|
||||
remapped = f"{prefix}{raw_path}" if raw_path != "/" else f"{prefix}/"
|
||||
if is_dir and not remapped.endswith("/"):
|
||||
remapped = f"{remapped}/"
|
||||
return {
|
||||
@ -89,9 +100,11 @@ async def _resolve_viewer_state(
|
||||
agent_id=agent_id,
|
||||
agent_config_id=agent_config_id,
|
||||
)
|
||||
visible_kbs = await resolve_visible_knowledge_bases_for_context(runtime_context)
|
||||
selected_skills = normalize_selected_skills(getattr(runtime_context, "skills", None) or [])
|
||||
skills_backend = SelectedSkillsReadonlyBackend(selected_slugs=selected_skills)
|
||||
return sandbox_backend, skills_backend, selected_skills
|
||||
kb_backend = KnowledgeBaseReadonlyBackend(visible_kbs=visible_kbs)
|
||||
return sandbox_backend, skills_backend, kb_backend, selected_skills
|
||||
|
||||
|
||||
async def list_viewer_filesystem_tree(
|
||||
@ -107,7 +120,7 @@ async def list_viewer_filesystem_tree(
|
||||
raise HTTPException(status_code=422, detail="thread_id 不能为空")
|
||||
|
||||
normalized_path = _normalize_path(path)
|
||||
sandbox_backend, skills_backend, selected_skills = await _resolve_viewer_state(
|
||||
sandbox_backend, skills_backend, kb_backend, selected_skills = await _resolve_viewer_state(
|
||||
thread_id=thread_id,
|
||||
agent_id=agent_id,
|
||||
agent_config_id=agent_config_id,
|
||||
@ -119,6 +132,8 @@ async def list_viewer_filesystem_tree(
|
||||
entries = [{"path": f"{USER_DATA_PATH}/", "name": "user-data", "is_dir": True, "size": 0, "modified_at": ""}]
|
||||
if selected_skills:
|
||||
entries.append({"path": f"{SKILLS_PATH}/", "name": "skills", "is_dir": True, "size": 0, "modified_at": ""})
|
||||
if kb_backend.has_entries():
|
||||
entries.append({"path": f"{KBS_PATH}/", "name": "kbs", "is_dir": True, "size": 0, "modified_at": ""})
|
||||
return {"entries": entries}
|
||||
|
||||
try:
|
||||
@ -128,7 +143,10 @@ async def list_viewer_filesystem_tree(
|
||||
|
||||
if _is_skills_path(normalized_path):
|
||||
entries = await asyncio.to_thread(skills_backend.ls_info, _strip_skills_prefix(normalized_path))
|
||||
return {"entries": [_remap_skills_entry(entry) for entry in entries]}
|
||||
return {"entries": [_remap_prefixed_entry(entry, SKILLS_PATH) for entry in entries]}
|
||||
if _is_kbs_path(normalized_path):
|
||||
entries = await asyncio.to_thread(kb_backend.ls_info, _strip_kbs_prefix(normalized_path))
|
||||
return {"entries": [_remap_prefixed_entry(entry, KBS_PATH) for entry in entries]}
|
||||
except PermissionError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e)) from e
|
||||
except ValueError as e:
|
||||
@ -150,7 +168,7 @@ async def read_viewer_file_content(
|
||||
raise HTTPException(status_code=422, detail="thread_id 不能为空")
|
||||
normalized_path = _normalize_path(path)
|
||||
|
||||
sandbox_backend, skills_backend, _selected_skills = await _resolve_viewer_state(
|
||||
sandbox_backend, skills_backend, kb_backend, _selected_skills = await _resolve_viewer_state(
|
||||
thread_id=thread_id,
|
||||
agent_id=agent_id,
|
||||
agent_config_id=agent_config_id,
|
||||
@ -163,6 +181,8 @@ async def read_viewer_file_content(
|
||||
responses = await asyncio.to_thread(sandbox_backend.download_files, [normalized_path])
|
||||
elif _is_skills_path(normalized_path):
|
||||
responses = await asyncio.to_thread(skills_backend.download_files, [_strip_skills_prefix(normalized_path)])
|
||||
elif _is_kbs_path(normalized_path):
|
||||
responses = await asyncio.to_thread(kb_backend.download_files, [_strip_kbs_prefix(normalized_path)])
|
||||
else:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
@ -199,7 +219,7 @@ async def download_viewer_file(
|
||||
db: AsyncSession,
|
||||
) -> StreamingResponse:
|
||||
normalized_path = _normalize_path(path)
|
||||
_sandbox_backend, skills_backend, _selected_skills = await _resolve_viewer_state(
|
||||
_sandbox_backend, skills_backend, kb_backend, _selected_skills = await _resolve_viewer_state(
|
||||
thread_id=thread_id,
|
||||
agent_id=agent_id,
|
||||
agent_config_id=agent_config_id,
|
||||
@ -224,6 +244,8 @@ async def download_viewer_file(
|
||||
|
||||
if _is_skills_path(normalized_path):
|
||||
responses = await asyncio.to_thread(skills_backend.download_files, [_strip_skills_prefix(normalized_path)])
|
||||
elif _is_kbs_path(normalized_path):
|
||||
responses = await asyncio.to_thread(kb_backend.download_files, [_strip_kbs_prefix(normalized_path)])
|
||||
else:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
|
||||
@ -2,10 +2,10 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
|
||||
from yuxi.agents.backends.sandbox import ensure_thread_dirs, sandbox_workspace_dir, virtual_path_for_thread_file
|
||||
|
||||
pytestmark = [pytest.mark.asyncio, pytest.mark.integration]
|
||||
@ -44,7 +44,13 @@ async def _create_thread_for_user(test_client, headers: dict[str, str]) -> str:
|
||||
return thread_id
|
||||
|
||||
|
||||
async def _upload_attachment_file(test_client, thread_id: str, headers: dict[str, str], file_name: str, content: str) -> str:
|
||||
async def _upload_attachment_file(
|
||||
test_client,
|
||||
thread_id: str,
|
||||
headers: dict[str, str],
|
||||
file_name: str,
|
||||
content: str,
|
||||
) -> str:
|
||||
response = await test_client.post(
|
||||
f"/api/chat/thread/{thread_id}/attachments",
|
||||
files={"file": (file_name, content.encode("utf-8"), "text/plain")},
|
||||
@ -110,6 +116,7 @@ async def test_viewer_download_returns_attachment_response(test_client, standard
|
||||
assert "download_demo" in content_disposition
|
||||
assert "download-me" in response.text
|
||||
|
||||
|
||||
async def test_viewer_download_returns_full_file_for_large_user_data_content(test_client, standard_user):
|
||||
headers = standard_user["headers"]
|
||||
thread_id = await _create_thread_for_user(test_client, headers)
|
||||
@ -128,3 +135,47 @@ async def test_viewer_download_returns_full_file_for_large_user_data_content(tes
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.content == large_content.encode("utf-8")
|
||||
|
||||
|
||||
async def test_viewer_tree_root_lists_kbs_namespace_when_visible(test_client, standard_user, monkeypatch):
|
||||
headers = standard_user["headers"]
|
||||
thread_id = await _create_thread_for_user(test_client, headers)
|
||||
|
||||
async def _fake_list_viewer_filesystem_tree(**kwargs):
|
||||
return {
|
||||
"entries": [
|
||||
{"path": "/home/yuxi/user-data/", "name": "user-data", "is_dir": True, "size": 0, "modified_at": ""},
|
||||
{"path": "/home/yuxi/kbs/", "name": "kbs", "is_dir": True, "size": 0, "modified_at": ""},
|
||||
]
|
||||
}
|
||||
|
||||
router_module = importlib.import_module("server.routers.filesystem_router")
|
||||
monkeypatch.setitem(
|
||||
router_module.get_viewer_tree.__globals__,
|
||||
"list_viewer_filesystem_tree",
|
||||
_fake_list_viewer_filesystem_tree,
|
||||
)
|
||||
|
||||
response = await test_client.get(
|
||||
"/api/viewer/filesystem/tree",
|
||||
params={"thread_id": thread_id, "path": "/"},
|
||||
headers=headers,
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
|
||||
entries = response.json().get("entries", [])
|
||||
paths = {entry.get("path") for entry in entries}
|
||||
assert "/home/yuxi/kbs/" in paths
|
||||
|
||||
|
||||
async def test_viewer_can_list_and_read_kbs_namespace(test_client, standard_user):
|
||||
headers = standard_user["headers"]
|
||||
thread_id = await _create_thread_for_user(test_client, headers)
|
||||
|
||||
tree_response = await test_client.get(
|
||||
"/api/viewer/filesystem/tree",
|
||||
params={"thread_id": thread_id, "path": "/home/yuxi/kbs"},
|
||||
headers=headers,
|
||||
)
|
||||
assert tree_response.status_code == 200, tree_response.text
|
||||
entries = tree_response.json().get("entries", [])
|
||||
assert any(str(entry.get("path", "")).startswith("/home/yuxi/kbs/") for entry in entries)
|
||||
|
||||
106
backend/test/test_knowledge_base_backend.py
Normal file
106
backend/test/test_knowledge_base_backend.py
Normal file
@ -0,0 +1,106 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from yuxi.agents.backends.knowledge_base_backend import (
|
||||
KnowledgeBaseReadonlyBackend,
|
||||
resolve_visible_knowledge_bases_for_context,
|
||||
)
|
||||
|
||||
|
||||
def test_knowledge_base_backend_builds_virtual_tree_and_materializes_files(monkeypatch, tmp_path) -> None:
|
||||
visible_kbs = [
|
||||
{"db_id": "db-1", "name": "FAQ"},
|
||||
{"db_id": "db-2", "name": "FAQ"},
|
||||
]
|
||||
files_meta = {
|
||||
"folder-api": {
|
||||
"file_id": "folder-api",
|
||||
"database_id": "db-1",
|
||||
"parent_id": None,
|
||||
"filename": "API",
|
||||
"is_folder": True,
|
||||
"created_at": "2026-03-26T00:00:00Z",
|
||||
},
|
||||
"file-pdf": {
|
||||
"file_id": "file-pdf",
|
||||
"database_id": "db-1",
|
||||
"parent_id": "folder-api",
|
||||
"filename": "auth-guide.pdf",
|
||||
"path": "http://minio/kb-source/db-1/upload/auth-guide.pdf",
|
||||
"markdown_file": "http://minio/kb-parsed/db-1/parsed/file-pdf.md",
|
||||
"size": 12,
|
||||
"is_folder": False,
|
||||
"created_at": "2026-03-26T00:00:00Z",
|
||||
},
|
||||
"file-pdf-2": {
|
||||
"file_id": "file-pdf-2",
|
||||
"database_id": "db-1",
|
||||
"parent_id": "folder-api",
|
||||
"filename": "auth-guide.pdf",
|
||||
"path": "http://minio/kb-source/db-1/upload/auth-guide-2.pdf",
|
||||
"size": 7,
|
||||
"is_folder": False,
|
||||
"created_at": "2026-03-26T00:00:00Z",
|
||||
},
|
||||
"file-db2": {
|
||||
"file_id": "file-db2",
|
||||
"database_id": "db-2",
|
||||
"parent_id": None,
|
||||
"filename": "overview.txt",
|
||||
"path": "http://minio/kb-source/db-2/upload/overview.txt",
|
||||
"size": 5,
|
||||
"is_folder": False,
|
||||
"created_at": "2026-03-26T00:00:00Z",
|
||||
},
|
||||
}
|
||||
monkeypatch.setattr("yuxi.agents.backends.knowledge_base_backend._all_files_meta", lambda: files_meta)
|
||||
|
||||
class _FakeMinio:
|
||||
def download_file(self, bucket_name: str, object_name: str) -> bytes:
|
||||
return f"{bucket_name}:{object_name}".encode()
|
||||
|
||||
monkeypatch.setattr("yuxi.agents.backends.knowledge_base_backend.get_minio_client", lambda: _FakeMinio())
|
||||
|
||||
backend = KnowledgeBaseReadonlyBackend(visible_kbs=visible_kbs, cache_root=tmp_path)
|
||||
|
||||
root_entries = {entry["path"] for entry in backend.ls_info("/")}
|
||||
assert "/FAQ/" in root_entries
|
||||
assert "/FAQ__db-2/" in root_entries
|
||||
|
||||
faq_entries = {entry["path"] for entry in backend.ls_info("/FAQ")}
|
||||
assert "/FAQ/API/" in faq_entries
|
||||
assert "/FAQ/parsed/" in faq_entries
|
||||
|
||||
api_entries = {entry["path"] for entry in backend.ls_info("/FAQ/API")}
|
||||
assert "/FAQ/API/auth-guide.pdf" in api_entries
|
||||
assert any(path.startswith("/FAQ/API/auth-guide.pdf__file-pdf") for path in api_entries)
|
||||
|
||||
parsed_entries = {entry["path"] for entry in backend.ls_info("/FAQ/parsed/API")}
|
||||
assert "/FAQ/parsed/API/auth-guide.md" in parsed_entries
|
||||
|
||||
responses = backend.download_files(["/FAQ/API/auth-guide.pdf", "/FAQ/parsed/API/auth-guide.md"])
|
||||
assert responses[0].content == b"kb-source:db-1/upload/auth-guide.pdf"
|
||||
assert responses[1].content == b"kb-parsed:db-1/parsed/file-pdf.md"
|
||||
|
||||
assert "kb-source" in backend.read("/FAQ/API/auth-guide.pdf")
|
||||
assert backend.write("/FAQ/API/new.txt", "x").error == "Knowledge base path is read-only."
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resolve_visible_knowledge_bases_for_context_filters_by_enabled_names(monkeypatch) -> None:
|
||||
async def _fake_get_databases_by_raw_id(user_id: int) -> dict:
|
||||
assert user_id == 7
|
||||
return {"databases": [{"db_id": "db-1", "name": "Alpha"}, {"db_id": "db-2", "name": "Beta"}]}
|
||||
|
||||
monkeypatch.setattr(
|
||||
"yuxi.agents.backends.knowledge_base_backend.knowledge_base.get_databases_by_raw_id",
|
||||
_fake_get_databases_by_raw_id,
|
||||
)
|
||||
|
||||
context = SimpleNamespace(user_id="7", knowledges=["Beta"])
|
||||
visible = await resolve_visible_knowledge_bases_for_context(context)
|
||||
|
||||
assert visible == [{"db_id": "db-2", "name": "Beta"}]
|
||||
assert getattr(context, "_visible_knowledge_bases") == visible
|
||||
@ -5,29 +5,36 @@ from __future__ import annotations
|
||||
from types import MethodType, SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from yuxi.agents.backends.composite import create_agent_composite_backend
|
||||
from yuxi.agents.backends.sandbox import resolve_virtual_path, sandbox_id_for_thread
|
||||
from yuxi.agents.backends.sandbox.backend import ProvisionerSandboxBackend
|
||||
from yuxi.agents.middlewares.skills_middleware import SkillsMiddleware
|
||||
|
||||
|
||||
def _runtime(*, thread_id: str | None = "thread-1", skills: list[str] | None = None):
|
||||
def _runtime(
|
||||
*,
|
||||
thread_id: str | None = "thread-1",
|
||||
skills: list[str] | None = None,
|
||||
visible_kbs: list[dict] | None = None,
|
||||
):
|
||||
configurable = {"thread_id": thread_id} if thread_id else {}
|
||||
return SimpleNamespace(
|
||||
config={"configurable": configurable},
|
||||
context=SimpleNamespace(skills=skills or []),
|
||||
context=SimpleNamespace(skills=skills or [], _visible_knowledge_bases=visible_kbs or []),
|
||||
)
|
||||
|
||||
|
||||
def test_create_agent_composite_backend_uses_provisioner_default(monkeypatch):
|
||||
monkeypatch.setattr("yuxi.agents.backends.sandbox.backend.get_sandbox_provider", lambda: object())
|
||||
|
||||
backend = create_agent_composite_backend(_runtime(skills=["reporter"]))
|
||||
backend = create_agent_composite_backend(
|
||||
_runtime(skills=["reporter"], visible_kbs=[{"db_id": "db-1", "name": "Docs"}])
|
||||
)
|
||||
|
||||
assert isinstance(backend.default, ProvisionerSandboxBackend)
|
||||
assert backend.default._visible_skills == ["reporter"]
|
||||
assert "/skills/" in backend.routes
|
||||
assert "/home/yuxi/kbs/" in backend.routes
|
||||
|
||||
|
||||
def test_create_agent_composite_backend_requires_thread_id():
|
||||
|
||||
@ -45,6 +45,8 @@
|
||||
|
||||
### 修复
|
||||
|
||||
- 为沙盒与 viewer 文件系统补齐知识库只读映射:新增 `/home/yuxi/kbs` 命名空间,按“用户可访问知识库 ∩ 当前 Agent 已启用知识库”暴露原始文件与解析后的 Markdown,并补充对应后端与 viewer 路由测试
|
||||
|
||||
<!-- 添加到这里 -->
|
||||
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user