feat: 将kb 从沙盒映射中移除，取而代之的是一个新的 Open 工具，可以提供自由且可控的文件访问
This commit is contained in:
Wenjie Zhang
parent
0627bc0db6
commit
8e17f44ec2
@ -26,7 +26,7 @@ Yuxi 是一个面向 RAG、知识图谱和多智能体工作流的知识库平
|
||||
|
||||
`backend/package/yuxi` 是后端主体：
|
||||
|
||||
- `agents` 定义 LangGraph 智能体体系。`BaseAgent` 是智能体基类，`BaseContext` 是运行配置上下文；`buildin` 放内置智能体；`middlewares` 负责把知识库、Skills、MCP、附件、运行配置等能力挂到运行时；`toolkits` 放工具注册与内置工具；`backends` 对接沙盒、知识库和 Skills 等外部执行/资源后端。
|
||||
- `agents` 定义 LangGraph 智能体体系。`BaseAgent` 是智能体基类，`BaseContext` 是运行配置上下文；`buildin` 放内置智能体；`middlewares` 负责把知识库、Skills、MCP、附件、运行配置等能力挂到运行时；`toolkits` 放工具注册与内置工具；`backends` 对接沙盒和 Skills 等外部执行/资源后端。
|
||||
- `services` 是用例层，负责串联 repositories、agents、knowledge、storage 和外部系统。聊天、运行队列、文件视图、Skills、MCP、SubAgents、评估等跨模块流程都从这里找入口。
|
||||
- `repositories` 是数据库访问边界，封装业务对象和知识库元数据的 SQLAlchemy 查询。不要让路由绕过 repository 直接操作模型，除非已有局部模式要求这样做。
|
||||
- `storage` 放持久化基础设施。`storage/postgres` 管理业务表、知识库表和 LangGraph checkpoint 所需连接池；`storage/minio` 管理对象存储。
|
||||
@ -56,8 +56,8 @@ Yuxi 是一个面向 RAG、知识图谱和多智能体工作流的知识库平
|
||||
2. `web/src/apis` 调用 `/api/chat` 相关接口。
|
||||
3. `server/routers/chat_router.py` 进入后端，委托 `yuxi.services.chat_service` 或 `agent_run_service`。
|
||||
4. 服务层读取 conversation、agent config、tools、skills、knowledge 等配置，必要时创建后台 run。
|
||||
5. `worker-dev` 执行 LangGraph 智能体；中间件按上下文挂载知识库、工具、Skills、MCP、附件与沙盒能力。
|
||||
6. 运行事件写入 Redis，最终状态和业务记录写入 Postgres；文件和产物落到 `saves`、MinIO 或沙盒映射目录。
|
||||
5. `worker-dev` 执行 LangGraph 智能体；中间件按上下文挂载知识库工具、Skills、MCP、附件与沙盒能力。
|
||||
6. 运行事件写入 Redis，最终状态和业务记录写入 Postgres；文件和产物落到 `saves`、MinIO 或沙盒用户数据目录。
|
||||
7. 前端通过 SSE/轮询消费运行事件，渲染消息、工具调用、引用来源、产物卡片和文件预览。
|
||||
|
||||
## 架构不变量
|
||||
@ -65,7 +65,7 @@ Yuxi 是一个面向 RAG、知识图谱和多智能体工作流的知识库平
|
||||
- Docker Compose 是开发环境的事实来源。开发时优先检查容器、日志和热重载，不要默认要求本地裸跑服务。
|
||||
- HTTP 路由层应保持薄；领域流程放在 `yuxi.services`，持久化查询放在 `yuxi.repositories`。
|
||||
- 前端 API 调用应集中在 `web/src/apis`，组件不要散落拼接后端 URL。
|
||||
- 智能体能力通过 context、middleware、toolkits、backends 组合，不要把知识库、MCP、Skills 或沙盒逻辑硬编码进单个页面或路由。
|
||||
- 智能体能力通过 context、middleware、toolkits、backends 组合；知识库通过工具访问，不要把知识库、MCP、Skills 或沙盒逻辑硬编码进单个页面或路由。
|
||||
- LITE 模式必须允许跳过知识库、图谱、评估等重依赖能力；新增相关接口或初始化逻辑时要尊重这个边界。
|
||||
- 沙盒虚拟路径以 `SANDBOX_VIRTUAL_PATH_PREFIX` 为边界，用户可见路径与宿主机真实路径不要混用。
|
||||
- 面向用户或外部系统的输入在边界校验；内部服务之间优先信任已有类型、仓储和框架约束，避免为了假设场景堆叠防御代码。
|
||||
|
||||
@ -1,13 +1,7 @@
|
||||
from deepagents.backends import CompositeBackend, StateBackend
|
||||
|
||||
from .composite import create_agent_composite_backend
|
||||
from .knowledge_base_backend import (
|
||||
KBS_PATH,
|
||||
KnowledgeBaseReadonlyBackend,
|
||||
build_knowledge_base_filepath_map,
|
||||
resolve_file_relative_virtual_path,
|
||||
resolve_visible_knowledge_bases_for_context,
|
||||
)
|
||||
from .knowledge_base_backend import resolve_visible_knowledge_bases_for_context
|
||||
from .sandbox import (
|
||||
IDLE_CHECK_INTERVAL,
|
||||
LARGE_TOOL_RESULTS_DIR,
|
||||
@ -37,10 +31,6 @@ from .skills_backend import SelectedSkillsReadonlyBackend
|
||||
|
||||
__all__ = [
|
||||
"CompositeBackend",
|
||||
"KBS_PATH",
|
||||
"KnowledgeBaseReadonlyBackend",
|
||||
"build_knowledge_base_filepath_map",
|
||||
"resolve_file_relative_virtual_path",
|
||||
"StateBackend",
|
||||
"SelectedSkillsReadonlyBackend",
|
||||
"create_agent_composite_backend",
|
||||
|
||||
@ -10,7 +10,6 @@ from deepagents.backends.protocol import FileInfo
|
||||
|
||||
from yuxi.agents.middlewares.skills_middleware import normalize_selected_skills
|
||||
|
||||
from .knowledge_base_backend import KnowledgeBaseReadonlyBackend
|
||||
from .sandbox import ProvisionerSandboxBackend
|
||||
from .skills_backend import SelectedSkillsReadonlyBackend
|
||||
|
||||
@ -111,23 +110,13 @@ def _extract_user_id(runtime) -> str:
|
||||
raise ValueError("user_id is required in runtime configurable context")
|
||||
|
||||
|
||||
def _get_visible_knowledge_bases_from_runtime(runtime) -> list[dict]:
|
||||
context = getattr(runtime, "context", None)
|
||||
selected = getattr(context, "_visible_knowledge_bases", None)
|
||||
if isinstance(selected, list):
|
||||
return selected
|
||||
return []
|
||||
|
||||
|
||||
def create_agent_composite_backend(runtime) -> CompositeBackend:
|
||||
visible_skills = _get_visible_skills_from_runtime(runtime)
|
||||
thread_id = _extract_thread_id(runtime)
|
||||
user_id = _extract_user_id(runtime)
|
||||
visible_kbs = _get_visible_knowledge_bases_from_runtime(runtime)
|
||||
return CustomCompositeBackend(
|
||||
default=ProvisionerSandboxBackend(thread_id=thread_id, user_id=user_id, visible_skills=visible_skills),
|
||||
routes={
|
||||
"/skills/": SelectedSkillsReadonlyBackend(selected_slugs=visible_skills),
|
||||
"/home/gem/kbs/": KnowledgeBaseReadonlyBackend(visible_kbs=visible_kbs),
|
||||
},
|
||||
)
|
||||
|
||||
@ -1,124 +1,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import fnmatch
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any
|
||||
|
||||
from deepagents.backends import FilesystemBackend
|
||||
from deepagents.backends.protocol import EditResult, FileDownloadResponse, FileInfo, FileUploadResponse, WriteResult
|
||||
|
||||
from yuxi import config as conf
|
||||
from yuxi import knowledge_base
|
||||
from yuxi.knowledge.utils.kb_utils import is_minio_url, parse_minio_url
|
||||
from yuxi.storage.minio import get_minio_client
|
||||
|
||||
KBS_PATH = "/home/gem/kbs"
|
||||
_INVALID_SEGMENT_RE = re.compile(r"[\\/\x00-\x1f\x7f]+")
|
||||
_WHITESPACE_RE = re.compile(r"\s+")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _MaterializedFile:
|
||||
virtual_path: str
|
||||
cache_path: Path
|
||||
source_path: str
|
||||
modified_at: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _ResolvedVirtualNode:
|
||||
db_id: str
|
||||
file_id: str
|
||||
path: str
|
||||
parent_path: str
|
||||
name: str
|
||||
is_folder: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _KnowledgeBaseVirtualLayout:
|
||||
kb_virtual_names: dict[str, str]
|
||||
files_by_db: dict[str, dict[str, dict[str, Any]]]
|
||||
nodes_by_db: dict[str, list[_ResolvedVirtualNode]]
|
||||
nodes_by_file_id: dict[str, _ResolvedVirtualNode]
|
||||
source_filepaths: dict[str, str]
|
||||
parsed_filepaths: dict[str, str]
|
||||
|
||||
|
||||
def _normalize_virtual_path(path: str | None) -> str:
|
||||
raw = str(path or "").strip() or "/"
|
||||
normalized = "/" + raw.lstrip("/")
|
||||
pure = PurePosixPath(normalized)
|
||||
if ".." in pure.parts:
|
||||
raise ValueError("path traversal is not allowed")
|
||||
return str(pure)
|
||||
|
||||
|
||||
def _sanitize_segment(value: str | None, fallback: str) -> str:
|
||||
cleaned = str(value or "").strip()
|
||||
cleaned = _INVALID_SEGMENT_RE.sub("_", cleaned)
|
||||
cleaned = _WHITESPACE_RE.sub(" ", cleaned).strip()
|
||||
if not cleaned or cleaned in {".", ".."}:
|
||||
return fallback
|
||||
return cleaned
|
||||
|
||||
|
||||
def _candidate_name(file_meta: dict[str, Any]) -> str:
|
||||
filename = str(file_meta.get("filename") or "").strip()
|
||||
if filename:
|
||||
return filename
|
||||
|
||||
original = str(file_meta.get("original_filename") or "").strip()
|
||||
if original:
|
||||
return original
|
||||
|
||||
for key in ("path", "markdown_file"):
|
||||
raw = str(file_meta.get(key) or "").strip()
|
||||
if raw:
|
||||
try:
|
||||
parsed = PurePosixPath(raw)
|
||||
name = parsed.name
|
||||
except Exception: # noqa: BLE001
|
||||
name = ""
|
||||
if name:
|
||||
return name
|
||||
|
||||
return str(file_meta.get("file_id") or "")
|
||||
|
||||
|
||||
def _unique_name(base_name: str, *, stable_id: str, used_names: set[str]) -> str:
|
||||
if base_name not in used_names:
|
||||
used_names.add(base_name)
|
||||
return base_name
|
||||
|
||||
suffix = f"__{stable_id[:8]}"
|
||||
candidate = f"{base_name}{suffix}"
|
||||
while candidate in used_names:
|
||||
suffix = f"__{stable_id}"
|
||||
candidate = f"{base_name}{suffix}"
|
||||
used_names.add(candidate)
|
||||
return candidate
|
||||
|
||||
|
||||
def _materialize_text_view(content: bytes, file_path: str, *, offset: int = 0, limit: int = 2000) -> str:
|
||||
if not content:
|
||||
return "System reminder: File exists but has empty contents"
|
||||
|
||||
if b"\x00" in content:
|
||||
return f"Error: File '{file_path}' is binary and cannot be rendered as text"
|
||||
|
||||
try:
|
||||
text = content.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
return f"Error: File '{file_path}' is binary and cannot be rendered as text"
|
||||
|
||||
lines = text.splitlines()
|
||||
start = max(0, int(offset))
|
||||
selected = lines[start : start + int(limit)]
|
||||
return "\n".join(f"{start + idx + 1:6d}\t{line}" for idx, line in enumerate(selected))
|
||||
|
||||
|
||||
async def resolve_visible_knowledge_bases_for_context(context) -> list[dict[str, Any]]:
|
||||
@ -142,533 +26,3 @@ async def resolve_visible_knowledge_bases_for_context(context) -> list[dict[str,
|
||||
|
||||
setattr(context, "_visible_knowledge_bases", databases)
|
||||
return databases
|
||||
|
||||
|
||||
def _all_files_meta() -> dict[str, dict[str, Any]]:
|
||||
aggregated: dict[str, dict[str, Any]] = {}
|
||||
for kb_instance in getattr(knowledge_base, "kb_instances", {}).values():
|
||||
for file_id, meta in getattr(kb_instance, "files_meta", {}).items():
|
||||
aggregated[str(file_id)] = meta
|
||||
return aggregated
|
||||
|
||||
|
||||
def _resolve_kb_virtual_names(visible_kbs: list[dict[str, Any]]) -> dict[str, str]:
|
||||
kb_name_candidates = {
|
||||
str(db.get("db_id") or db.get("name") or f"kb-{index}"): _sanitize_segment(
|
||||
db.get("name"),
|
||||
str(db.get("db_id") or f"kb-{index}"),
|
||||
)
|
||||
for index, db in enumerate(visible_kbs)
|
||||
}
|
||||
used_root_names: set[str] = set()
|
||||
kb_virtual_names: dict[str, str] = {}
|
||||
sorted_kbs = sorted(
|
||||
visible_kbs,
|
||||
key=lambda item: (str(item.get("name") or ""), str(item.get("db_id") or "")),
|
||||
)
|
||||
for db in sorted_kbs:
|
||||
db_id = str(db.get("db_id") or "")
|
||||
if not db_id:
|
||||
continue
|
||||
base_name = kb_name_candidates.get(db_id) or db_id
|
||||
kb_virtual_names[db_id] = _unique_name(base_name, stable_id=db_id, used_names=used_root_names)
|
||||
return kb_virtual_names
|
||||
|
||||
|
||||
def _resolve_db_virtual_nodes(
|
||||
*,
|
||||
db_id: str,
|
||||
kb_root: str,
|
||||
records: dict[str, dict[str, Any]],
|
||||
) -> list[_ResolvedVirtualNode]:
|
||||
# Keep sibling deduplication local to each parent directory so the generated tree is deterministic.
|
||||
children_by_parent: dict[str | None, list[tuple[str, dict[str, Any]]]] = defaultdict(list)
|
||||
for file_id, meta in records.items():
|
||||
parent_id = str(meta.get("parent_id") or "").strip() or None
|
||||
children_by_parent[parent_id].append((file_id, meta))
|
||||
|
||||
resolved_nodes: list[_ResolvedVirtualNode] = []
|
||||
|
||||
def walk(parent_id: str | None, parent_path: str) -> None:
|
||||
used_names: set[str] = set()
|
||||
siblings = children_by_parent.get(parent_id, [])
|
||||
sorted_siblings = sorted(
|
||||
siblings,
|
||||
key=lambda item: (_sanitize_segment(_candidate_name(item[1]), item[0]), item[0]),
|
||||
)
|
||||
for file_id, meta in sorted_siblings:
|
||||
base_name = _sanitize_segment(_candidate_name(meta), file_id)
|
||||
unique_name = _unique_name(base_name, stable_id=file_id, used_names=used_names)
|
||||
child_path = f"{parent_path.rstrip('/')}/{unique_name}" if parent_path != "/" else f"/{unique_name}"
|
||||
node = _ResolvedVirtualNode(
|
||||
db_id=db_id,
|
||||
file_id=file_id,
|
||||
path=child_path,
|
||||
parent_path=parent_path,
|
||||
name=unique_name,
|
||||
is_folder=bool(meta.get("is_folder")),
|
||||
)
|
||||
resolved_nodes.append(node)
|
||||
if node.is_folder:
|
||||
walk(file_id, child_path)
|
||||
|
||||
walk(None, kb_root)
|
||||
return resolved_nodes
|
||||
|
||||
|
||||
def _build_parsed_filepath_map(
|
||||
*,
|
||||
nodes_by_file_id: dict[str, _ResolvedVirtualNode],
|
||||
files_by_db: dict[str, dict[str, dict[str, Any]]],
|
||||
kb_virtual_names: dict[str, str],
|
||||
) -> dict[str, str]:
|
||||
# Parsed files mirror the source tree and only add a fixed `/parsed` segment plus `.md` suffix.
|
||||
parsed_paths: dict[str, str] = {}
|
||||
for file_id, node in nodes_by_file_id.items():
|
||||
if node.is_folder:
|
||||
continue
|
||||
|
||||
record = files_by_db.get(node.db_id, {}).get(file_id, {})
|
||||
if not str(record.get("markdown_file") or "").strip():
|
||||
continue
|
||||
|
||||
kb_root = f"/{kb_virtual_names[node.db_id]}"
|
||||
parsed_root = f"{kb_root}/parsed"
|
||||
parsed_parent = (
|
||||
f"{parsed_root}{node.parent_path[len(kb_root) :]}" if node.parent_path.startswith(kb_root) else parsed_root
|
||||
)
|
||||
parsed_name = _sanitize_segment(f"{node.name}.md", f"{file_id}.md")
|
||||
parsed_path = f"{parsed_parent.rstrip('/')}/{parsed_name}" if parsed_parent != "/" else f"/{parsed_name}"
|
||||
parsed_paths[file_id] = f"{KBS_PATH}{parsed_path}"
|
||||
|
||||
return parsed_paths
|
||||
|
||||
|
||||
def _resolve_virtual_layout(
|
||||
*,
|
||||
visible_kbs: list[dict[str, Any]] | None,
|
||||
files_meta: dict[str, dict[str, Any]] | None = None,
|
||||
) -> _KnowledgeBaseVirtualLayout:
|
||||
# Single source of truth for the virtual KB tree.
|
||||
# The readonly backend and query_kb filepath injection must stay fully aligned.
|
||||
kb_virtual_names = _resolve_kb_virtual_names(list(visible_kbs or []))
|
||||
if not kb_virtual_names:
|
||||
return _KnowledgeBaseVirtualLayout(
|
||||
kb_virtual_names={},
|
||||
files_by_db={},
|
||||
nodes_by_db={},
|
||||
nodes_by_file_id={},
|
||||
source_filepaths={},
|
||||
parsed_filepaths={},
|
||||
)
|
||||
|
||||
source_files = _all_files_meta() if files_meta is None else files_meta
|
||||
files_by_db: dict[str, dict[str, dict[str, Any]]] = defaultdict(dict)
|
||||
for file_id, meta in source_files.items():
|
||||
db_id = str(meta.get("database_id") or "")
|
||||
if db_id in kb_virtual_names:
|
||||
files_by_db[db_id][str(file_id)] = meta
|
||||
|
||||
nodes_by_db: dict[str, list[_ResolvedVirtualNode]] = {}
|
||||
nodes_by_file_id: dict[str, _ResolvedVirtualNode] = {}
|
||||
source_filepaths: dict[str, str] = {}
|
||||
|
||||
for db_id, kb_virtual_name in kb_virtual_names.items():
|
||||
kb_root = f"/{kb_virtual_name}"
|
||||
records = files_by_db.get(db_id, {})
|
||||
nodes = _resolve_db_virtual_nodes(db_id=db_id, kb_root=kb_root, records=records)
|
||||
nodes_by_db[db_id] = nodes
|
||||
for node in nodes:
|
||||
nodes_by_file_id[node.file_id] = node
|
||||
if not node.is_folder:
|
||||
source_filepaths[node.file_id] = f"{KBS_PATH}{node.path}"
|
||||
|
||||
return _KnowledgeBaseVirtualLayout(
|
||||
kb_virtual_names=kb_virtual_names,
|
||||
files_by_db=dict(files_by_db),
|
||||
nodes_by_db=nodes_by_db,
|
||||
nodes_by_file_id=nodes_by_file_id,
|
||||
source_filepaths=source_filepaths,
|
||||
parsed_filepaths=_build_parsed_filepath_map(
|
||||
nodes_by_file_id=nodes_by_file_id,
|
||||
files_by_db=files_by_db,
|
||||
kb_virtual_names=kb_virtual_names,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def resolve_file_relative_virtual_path(
|
||||
*,
|
||||
file_id: str,
|
||||
visible_kbs: list[dict[str, Any]] | None,
|
||||
files_meta: dict[str, dict[str, Any]] | None = None,
|
||||
) -> str | None:
|
||||
normalized_id = str(file_id or "").strip()
|
||||
if not normalized_id:
|
||||
return None
|
||||
|
||||
layout = _resolve_virtual_layout(
|
||||
visible_kbs=visible_kbs,
|
||||
files_meta=files_meta,
|
||||
)
|
||||
node = layout.nodes_by_file_id.get(normalized_id)
|
||||
if node is None or node.is_folder:
|
||||
return None
|
||||
return node.path
|
||||
|
||||
|
||||
def build_knowledge_base_filepath_map(
|
||||
*,
|
||||
visible_kbs: list[dict[str, Any]] | None,
|
||||
files_meta: dict[str, dict[str, Any]] | None = None,
|
||||
) -> dict[str, str]:
|
||||
layout = _resolve_virtual_layout(
|
||||
visible_kbs=visible_kbs,
|
||||
files_meta=files_meta,
|
||||
)
|
||||
return layout.source_filepaths.copy()
|
||||
|
||||
|
||||
def _inject_kb_filepaths(
|
||||
chunks: list[dict[str, Any]],
|
||||
filepath_map: dict[str, str],
|
||||
parsed_filepath_map: dict[str, str],
|
||||
) -> list[dict[str, Any]]:
|
||||
if not filepath_map and not parsed_filepath_map:
|
||||
return chunks
|
||||
|
||||
for chunk in chunks:
|
||||
if not isinstance(chunk, dict):
|
||||
continue
|
||||
|
||||
metadata = chunk.get("metadata")
|
||||
if metadata is None:
|
||||
metadata = {}
|
||||
if not isinstance(metadata, dict):
|
||||
continue
|
||||
|
||||
file_id = str(metadata.get("file_id") or chunk.get("file_id") or "").strip()
|
||||
if not file_id:
|
||||
continue
|
||||
|
||||
if not metadata.get("filepath"):
|
||||
filepath = filepath_map.get(file_id)
|
||||
if filepath:
|
||||
metadata["filepath"] = filepath
|
||||
|
||||
if not metadata.get("parsed_path"):
|
||||
parsed_path = parsed_filepath_map.get(file_id)
|
||||
if parsed_path:
|
||||
metadata["parsed_path"] = parsed_path
|
||||
|
||||
chunk["metadata"] = metadata
|
||||
|
||||
return chunks
|
||||
|
||||
|
||||
async def inject_filepaths_into_retrieval_result(
|
||||
*,
|
||||
retrieval_chunks: list[dict[str, Any]],
|
||||
visible_kbs: list[dict[str, Any]] | None,
|
||||
target_db_id: str | None,
|
||||
target_kb_name: str | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
scope_kbs = list(visible_kbs or [])
|
||||
if not scope_kbs and target_db_id:
|
||||
scope_kbs = [{"db_id": target_db_id, "name": target_kb_name or target_db_id}]
|
||||
|
||||
layout = _resolve_virtual_layout(visible_kbs=scope_kbs)
|
||||
return _inject_kb_filepaths(
|
||||
retrieval_chunks,
|
||||
layout.source_filepaths,
|
||||
layout.parsed_filepaths,
|
||||
)
|
||||
|
||||
|
||||
class KnowledgeBaseReadonlyBackend(FilesystemBackend):
|
||||
def __init__(self, *, visible_kbs: list[dict[str, Any]] | None, cache_root: Path | str | None = None):
|
||||
self._cache_root = Path(cache_root or (Path(conf.save_dir) / "knowledge_base_data" / "kb-cache")).resolve()
|
||||
self._cache_root.mkdir(parents=True, exist_ok=True)
|
||||
super().__init__(root_dir=self._cache_root, virtual_mode=True)
|
||||
self._visible_kbs = list(visible_kbs or [])
|
||||
self._entries_by_dir: dict[str, list[FileInfo]] = defaultdict(list)
|
||||
self._dir_paths: set[str] = {"/"}
|
||||
self._files: dict[str, _MaterializedFile] = {}
|
||||
self._all_files: list[FileInfo] = []
|
||||
self._build_virtual_tree()
|
||||
|
||||
def has_entries(self) -> bool:
|
||||
return bool(self._visible_kbs)
|
||||
|
||||
def _build_virtual_tree(self) -> None:
|
||||
layout = _resolve_virtual_layout(visible_kbs=self._visible_kbs)
|
||||
|
||||
for db_id, kb_virtual_name in layout.kb_virtual_names.items():
|
||||
kb_root = f"/{kb_virtual_name}"
|
||||
self._add_entry("/", kb_root, is_dir=True)
|
||||
records = layout.files_by_db.get(db_id, {})
|
||||
self._build_source_tree(db_id=db_id, records=records, nodes=layout.nodes_by_db.get(db_id, []))
|
||||
self._build_parsed_tree(db_id=db_id, kb_root=kb_root, records=records)
|
||||
|
||||
for path, entries in list(self._entries_by_dir.items()):
|
||||
entries.sort(key=lambda item: str(item.get("path") or ""))
|
||||
self._entries_by_dir[path] = entries
|
||||
self._all_files = sorted(self._all_files, key=lambda item: str(item.get("path") or ""))
|
||||
|
||||
def _build_source_tree(
|
||||
self,
|
||||
*,
|
||||
db_id: str,
|
||||
records: dict[str, dict[str, Any]],
|
||||
nodes: list[_ResolvedVirtualNode],
|
||||
) -> None:
|
||||
resolved_parent_paths: dict[str, str] = {}
|
||||
resolved_source_names: dict[str, str] = {}
|
||||
|
||||
for node in sorted(nodes, key=lambda item: item.path):
|
||||
meta = records.get(node.file_id, {})
|
||||
modified_at = meta.get("updated_at") or meta.get("created_at")
|
||||
if node.is_folder:
|
||||
self._add_entry(node.parent_path, node.path, is_dir=True, modified_at=modified_at)
|
||||
continue
|
||||
|
||||
self._add_entry(
|
||||
node.parent_path,
|
||||
node.path,
|
||||
is_dir=False,
|
||||
size=int(meta.get("size") or 0),
|
||||
modified_at=modified_at,
|
||||
)
|
||||
resolved_parent_paths[node.file_id] = node.parent_path
|
||||
resolved_source_names[node.file_id] = node.name
|
||||
cache_path = self._cache_root / db_id / "source" / node.file_id / node.name
|
||||
self._files[node.path] = _MaterializedFile(
|
||||
virtual_path=node.path,
|
||||
cache_path=cache_path,
|
||||
source_path=str(meta.get("path") or ""),
|
||||
modified_at=modified_at,
|
||||
)
|
||||
self._all_files.append(
|
||||
{
|
||||
"path": node.path,
|
||||
"is_dir": False,
|
||||
"size": int(meta.get("size") or 0),
|
||||
"modified_at": str(modified_at or ""),
|
||||
}
|
||||
)
|
||||
|
||||
self._resolved_parent_paths = getattr(self, "_resolved_parent_paths", {})
|
||||
self._resolved_parent_paths[db_id] = resolved_parent_paths
|
||||
self._resolved_source_names = getattr(self, "_resolved_source_names", {})
|
||||
self._resolved_source_names[db_id] = resolved_source_names
|
||||
|
||||
def _build_parsed_tree(self, *, db_id: str, kb_root: str, records: dict[str, dict[str, Any]]) -> None:
|
||||
parsed_records = {
|
||||
file_id: meta
|
||||
for file_id, meta in records.items()
|
||||
if not meta.get("is_folder") and str(meta.get("markdown_file") or "").strip()
|
||||
}
|
||||
if not parsed_records:
|
||||
return
|
||||
|
||||
parsed_root = f"{kb_root}/parsed"
|
||||
self._add_entry(kb_root, parsed_root, is_dir=True)
|
||||
parent_paths = getattr(self, "_resolved_parent_paths", {}).get(db_id, {})
|
||||
source_names = getattr(self, "_resolved_source_names", {}).get(db_id, {})
|
||||
|
||||
grouped: dict[str, list[tuple[str, dict[str, Any], str]]] = defaultdict(list)
|
||||
for file_id, meta in parsed_records.items():
|
||||
source_parent = parent_paths.get(file_id, kb_root)
|
||||
parsed_parent = (
|
||||
f"{parsed_root}{source_parent[len(kb_root) :]}" if source_parent.startswith(kb_root) else parsed_root
|
||||
)
|
||||
source_name = source_names.get(file_id) or _sanitize_segment(meta.get("filename"), file_id)
|
||||
safe_name = source_name or file_id
|
||||
base_name = _sanitize_segment(f"{safe_name}.md", f"{file_id}.md")
|
||||
grouped[parsed_parent].append((file_id, meta, base_name))
|
||||
|
||||
for parsed_parent, items in grouped.items():
|
||||
current = PurePosixPath(parsed_parent)
|
||||
while str(current) not in {"", "."} and str(current) != "/":
|
||||
current_str = str(current)
|
||||
parent_str = str(current.parent) if str(current.parent) != "." else "/"
|
||||
if current_str not in self._dir_paths:
|
||||
self._add_entry(parent_str, current_str, is_dir=True)
|
||||
current = current.parent
|
||||
|
||||
used_names: set[str] = set()
|
||||
for file_id, meta, base_name in sorted(items, key=lambda item: (item[2], item[0])):
|
||||
unique_name = _unique_name(base_name, stable_id=file_id, used_names=used_names)
|
||||
file_path = f"{parsed_parent.rstrip('/')}/{unique_name}" if parsed_parent != "/" else f"/{unique_name}"
|
||||
self._add_entry(
|
||||
parsed_parent,
|
||||
file_path,
|
||||
is_dir=False,
|
||||
size=0,
|
||||
modified_at=meta.get("updated_at") or meta.get("created_at"),
|
||||
)
|
||||
cache_path = self._cache_root / db_id / "parsed" / f"{file_id}.md"
|
||||
self._files[file_path] = _MaterializedFile(
|
||||
virtual_path=file_path,
|
||||
cache_path=cache_path,
|
||||
source_path=str(meta.get("markdown_file") or ""),
|
||||
modified_at=meta.get("updated_at") or meta.get("created_at"),
|
||||
)
|
||||
self._all_files.append(
|
||||
{
|
||||
"path": file_path,
|
||||
"is_dir": False,
|
||||
"size": 0,
|
||||
"modified_at": str(meta.get("updated_at") or meta.get("created_at") or ""),
|
||||
}
|
||||
)
|
||||
|
||||
def _add_entry(
|
||||
self,
|
||||
parent_path: str,
|
||||
child_path: str,
|
||||
*,
|
||||
is_dir: bool,
|
||||
size: int = 0,
|
||||
modified_at: str | None = None,
|
||||
) -> None:
|
||||
normalized_parent = _normalize_virtual_path(parent_path)
|
||||
normalized_child = _normalize_virtual_path(child_path)
|
||||
entry_path = f"{normalized_child}/" if is_dir else normalized_child
|
||||
entry: FileInfo = {
|
||||
"path": entry_path,
|
||||
"is_dir": is_dir,
|
||||
"size": int(size or 0),
|
||||
"modified_at": str(modified_at or ""),
|
||||
}
|
||||
if entry_path not in {str(item.get("path")) for item in self._entries_by_dir[normalized_parent]}:
|
||||
self._entries_by_dir[normalized_parent].append(entry)
|
||||
if is_dir:
|
||||
self._dir_paths.add(normalized_child)
|
||||
self._entries_by_dir.setdefault(normalized_child, [])
|
||||
|
||||
def _ensure_local_file(self, descriptor: _MaterializedFile) -> Path:
|
||||
if descriptor.cache_path.exists():
|
||||
return descriptor.cache_path
|
||||
|
||||
source = descriptor.source_path.strip()
|
||||
if not source:
|
||||
raise FileNotFoundError(descriptor.virtual_path)
|
||||
if not is_minio_url(source):
|
||||
raise FileNotFoundError(descriptor.virtual_path)
|
||||
|
||||
bucket_name, object_name = parse_minio_url(source)
|
||||
payload = get_minio_client().download_file(bucket_name, object_name)
|
||||
descriptor.cache_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
descriptor.cache_path.write_bytes(payload)
|
||||
return descriptor.cache_path
|
||||
|
||||
def ls_info(self, path: str) -> list[FileInfo]:
|
||||
normalized_path = _normalize_virtual_path(path)
|
||||
return list(self._entries_by_dir.get(normalized_path, []))
|
||||
|
||||
def read(self, file_path: str, offset: int = 0, limit: int = 2000) -> str:
|
||||
normalized_path = _normalize_virtual_path(file_path)
|
||||
descriptor = self._files.get(normalized_path)
|
||||
if descriptor is None:
|
||||
if normalized_path in self._dir_paths:
|
||||
return f"Error: Path '{file_path}' is a directory"
|
||||
return f"Error: File '{file_path}' not found"
|
||||
|
||||
try:
|
||||
content = self._ensure_local_file(descriptor).read_bytes()
|
||||
except FileNotFoundError:
|
||||
return f"Error: File '{file_path}' not found"
|
||||
except Exception as exc: # noqa: BLE001
|
||||
detail = str(exc).strip() or "unknown error"
|
||||
return f"Error: Failed to read '{file_path}': {detail}"
|
||||
|
||||
return _materialize_text_view(content, file_path, offset=offset, limit=limit)
|
||||
|
||||
def glob_info(self, pattern: str, path: str = "/") -> list[FileInfo]:
|
||||
normalized_path = _normalize_virtual_path(path)
|
||||
if ".." in PurePosixPath(pattern).parts:
|
||||
raise ValueError("Path traversal not allowed in glob pattern")
|
||||
normalized_pattern = pattern.lstrip("/") or "*"
|
||||
prefix = "/" if normalized_path == "/" else f"{normalized_path.rstrip('/')}/"
|
||||
|
||||
matches: list[FileInfo] = []
|
||||
for item in self._all_files:
|
||||
item_path = str(item.get("path") or "")
|
||||
if normalized_path != "/" and not item_path.startswith(prefix):
|
||||
continue
|
||||
relative = item_path[len(prefix) :] if normalized_path != "/" else item_path.lstrip("/")
|
||||
if fnmatch.fnmatch(relative, normalized_pattern):
|
||||
matches.append(dict(item))
|
||||
return matches
|
||||
|
||||
def grep_raw(self, pattern: str, path: str | None = None, glob: str | None = None) -> list[dict[str, Any]] | str:
|
||||
normalized_path = _normalize_virtual_path(path or "/")
|
||||
prefix = "/" if normalized_path == "/" else f"{normalized_path.rstrip('/')}/"
|
||||
if normalized_path in self._files:
|
||||
targets = [normalized_path]
|
||||
else:
|
||||
targets = [
|
||||
item["path"]
|
||||
for item in self._all_files
|
||||
if normalized_path == "/" or str(item["path"]).startswith(prefix)
|
||||
]
|
||||
|
||||
matches: list[dict[str, Any]] = []
|
||||
for target in targets:
|
||||
display_target = str(target)
|
||||
relative = display_target.lstrip("/") if normalized_path == "/" else display_target[len(prefix) :]
|
||||
if glob and not fnmatch.fnmatch(relative, glob):
|
||||
continue
|
||||
descriptor = self._files.get(display_target)
|
||||
if descriptor is None:
|
||||
continue
|
||||
try:
|
||||
content = self._ensure_local_file(descriptor).read_bytes()
|
||||
if b"\x00" in content:
|
||||
continue
|
||||
text = content.decode("utf-8")
|
||||
except Exception: # noqa: BLE001
|
||||
continue
|
||||
for line_num, line in enumerate(text.splitlines(), start=1):
|
||||
if pattern in line:
|
||||
matches.append({"path": display_target, "line": line_num, "text": line})
|
||||
return matches
|
||||
|
||||
def write(self, file_path: str, content: str) -> WriteResult:
|
||||
return WriteResult(error="Knowledge base path is read-only.")
|
||||
|
||||
def edit(self, file_path: str, old_string: str, new_string: str, replace_all: bool = False) -> EditResult:
|
||||
return EditResult(error="Knowledge base path is read-only.")
|
||||
|
||||
def upload_files(self, files: list[tuple[str, bytes]]) -> list[FileUploadResponse]:
|
||||
return [FileUploadResponse(path=path, error="permission_denied") for path, _content in files]
|
||||
|
||||
def download_files(self, paths: list[str]) -> list[FileDownloadResponse]:
|
||||
responses: list[FileDownloadResponse] = []
|
||||
for path in paths:
|
||||
try:
|
||||
normalized_path = _normalize_virtual_path(path)
|
||||
except ValueError:
|
||||
responses.append(FileDownloadResponse(path=path, content=None, error="invalid_path"))
|
||||
continue
|
||||
|
||||
descriptor = self._files.get(normalized_path)
|
||||
if descriptor is None:
|
||||
if normalized_path in self._dir_paths:
|
||||
responses.append(FileDownloadResponse(path=path, content=None, error="is_directory"))
|
||||
else:
|
||||
responses.append(FileDownloadResponse(path=path, content=None, error="file_not_found"))
|
||||
continue
|
||||
|
||||
try:
|
||||
content = self._ensure_local_file(descriptor).read_bytes()
|
||||
except FileNotFoundError:
|
||||
responses.append(FileDownloadResponse(path=path, content=None, error="file_not_found"))
|
||||
continue
|
||||
except ValueError:
|
||||
responses.append(FileDownloadResponse(path=path, content=None, error="invalid_path"))
|
||||
continue
|
||||
|
||||
responses.append(FileDownloadResponse(path=path, content=content, error=None))
|
||||
return responses
|
||||
|
||||
@ -1,5 +1,4 @@
|
||||
from yuxi.utils.paths import (
|
||||
VIRTUAL_KBS_PATH,
|
||||
VIRTUAL_PATH_OUTPUTS,
|
||||
VIRTUAL_PATH_PREFIX,
|
||||
VIRTUAL_PATH_UPLOADS,
|
||||
@ -25,9 +24,8 @@ PROMPT = f"""
|
||||
非必要不写入其他路径
|
||||
|
||||
<| 知识库访问 |>
|
||||
当 query_kb 中没有找到相关的内容，或者需要进一步基于检索到的内容获取更加详细的上下文的时候，还可以直接访问知识库文件系统
|
||||
（路径为 {VIRTUAL_KBS_PATH}）来获取信息。
|
||||
源文件可能无法直接读取，可以在 {VIRTUAL_KBS_PATH}/<db_name>/parsed/ 中找到解析后的 markdown 文件。
|
||||
当 query_kb 中没有找到相关内容，或者需要进一步基于检索结果获取更详细上下文时，
|
||||
使用 open_kb_document 按 resource_id 和 file_id 打开知识库文档。
|
||||
"""
|
||||
|
||||
# 效果不好，暂时不启用
|
||||
|
||||
@ -41,24 +41,19 @@ async def list_kbs(dummy: str, runtime: ToolRuntime) -> str: # Now has 2 params
|
||||
|
||||
enabled_kb_names = getattr(runtime_context, "knowledges", None)
|
||||
|
||||
# 获取用户可访问的知识库列表（包含名称和描述）
|
||||
try:
|
||||
result = await knowledge_base.get_databases_by_raw_id(user_id)
|
||||
all_kbs = result.get("databases", [])
|
||||
from yuxi.agents.backends.knowledge_base_backend import resolve_visible_knowledge_bases_for_context
|
||||
|
||||
available_kbs = await resolve_visible_knowledge_bases_for_context(runtime_context)
|
||||
except Exception as e:
|
||||
logger.error(f"获取用户知识库列表失败: {e}")
|
||||
return f"获取知识库列表失败: {str(e)}"
|
||||
|
||||
all_kb_names = [kb["name"] for kb in all_kbs]
|
||||
all_kb_names = [kb["name"] for kb in available_kbs]
|
||||
|
||||
logger.debug(f"用户 {user_id} 可访问的知识库列表: {all_kb_names}")
|
||||
logger.debug(f"用户 {user_id} 当前对话启用的知识库列表: {enabled_kb_names}")
|
||||
|
||||
if enabled_kb_names is None:
|
||||
available_kbs = all_kbs
|
||||
else:
|
||||
available_kbs = [kb for kb in all_kbs if kb["name"] in enabled_kb_names]
|
||||
|
||||
if not available_kbs:
|
||||
return "当前没有可访问的知识库"
|
||||
|
||||
@ -188,7 +183,7 @@ async def _resolve_visible_knowledge_bases_for_query(runtime: ToolRuntime | None
|
||||
|
||||
return await resolve_visible_knowledge_bases_for_context(context)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning(f"解析会话可见知识库失败，跳过 filepath 注入: {exc}")
|
||||
logger.warning(f"解析会话可见知识库失败: {exc}")
|
||||
return []
|
||||
|
||||
|
||||
@ -197,21 +192,21 @@ def _find_query_target(
|
||||
kb_name: str,
|
||||
retrievers: dict[str, Any],
|
||||
visible_kbs: list[dict[str, Any]],
|
||||
) -> tuple[str | None, dict[str, Any] | None, str | None]:
|
||||
) -> tuple[dict[str, Any] | None, str | None]:
|
||||
if not visible_kbs:
|
||||
return None, None, "无法获取当前会话可访问的知识库"
|
||||
return None, "无法获取当前会话可访问的知识库"
|
||||
|
||||
matched_kbs = [db for db in visible_kbs if str(db.get("name") or "").strip() == kb_name]
|
||||
if not matched_kbs:
|
||||
return None, None, f"知识库 '{kb_name}' 不存在或当前会话未启用"
|
||||
return None, f"知识库 '{kb_name}' 不存在或当前会话未启用"
|
||||
if len(matched_kbs) > 1:
|
||||
return None, None, f"知识库 '{kb_name}' 存在重名，请先调整名称后重试"
|
||||
return None, f"知识库 '{kb_name}' 存在重名，请先调整名称后重试"
|
||||
|
||||
target_db_id = str(matched_kbs[0].get("db_id") or "")
|
||||
target_info = retrievers.get(target_db_id)
|
||||
if target_info is None:
|
||||
return None, None, f"知识库 '{kb_name}' 不存在"
|
||||
return target_db_id, target_info, None
|
||||
return None, f"知识库 '{kb_name}' 不存在"
|
||||
return target_info, None
|
||||
|
||||
|
||||
def _normalize_retrieval_result_metadata(result: Any) -> Any:
|
||||
@ -265,7 +260,7 @@ async def query_kb(kb_name: str, query_text: str, file_name: str | None = None,
|
||||
|
||||
visible_kbs = await _resolve_visible_knowledge_bases_for_query(runtime)
|
||||
|
||||
target_db_id, target_info, target_error = _find_query_target(
|
||||
target_info, target_error = _find_query_target(
|
||||
kb_name=kb_name,
|
||||
retrievers=retrievers,
|
||||
visible_kbs=visible_kbs,
|
||||
@ -273,9 +268,6 @@ async def query_kb(kb_name: str, query_text: str, file_name: str | None = None,
|
||||
if target_error:
|
||||
return target_error
|
||||
|
||||
metadata = target_info.get("metadata") if isinstance(target_info, dict) else None
|
||||
kb_type = str((metadata or {}).get("kb_type") or "").strip().lower()
|
||||
|
||||
try:
|
||||
retriever = target_info["retriever"]
|
||||
kwargs = {}
|
||||
@ -287,23 +279,7 @@ async def query_kb(kb_name: str, query_text: str, file_name: str | None = None,
|
||||
else:
|
||||
result = retriever(query_text, **kwargs)
|
||||
|
||||
result = _normalize_retrieval_result_metadata(result)
|
||||
|
||||
if kb_type != "milvus":
|
||||
return result
|
||||
|
||||
if not isinstance(result, list):
|
||||
return f"知识库 '{kb_name}' 返回结果不是 Milvus chunks 列表，无法注入文件路径"
|
||||
|
||||
from yuxi.agents.backends.knowledge_base_backend import inject_filepaths_into_retrieval_result
|
||||
|
||||
# 只有 Milvus 结果的 file_id 对应本地文件系统，可补充沙盒可读路径。
|
||||
return await inject_filepaths_into_retrieval_result(
|
||||
retrieval_chunks=result,
|
||||
visible_kbs=visible_kbs,
|
||||
target_db_id=target_db_id,
|
||||
target_kb_name=kb_name,
|
||||
)
|
||||
return _normalize_retrieval_result_metadata(result)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"检索失败: {e}")
|
||||
|
||||
@ -1,169 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from yuxi.config import config
|
||||
from yuxi.knowledge import knowledge_base
|
||||
from yuxi.knowledge.utils.kb_utils import parse_minio_url
|
||||
from yuxi.repositories.knowledge_file_repository import KnowledgeFileRepository
|
||||
from yuxi.storage.minio import get_minio_client
|
||||
|
||||
KBS_PATH = "/home/gem/kbs"
|
||||
PARSED_DIR_NAME = "parsed"
|
||||
_INVALID_MOUNT_NAME_CHARS = re.compile(r'[\\/:*?"<>|\x00-\x1f]')
|
||||
_MULTISPACE = re.compile(r"\s+")
|
||||
|
||||
|
||||
def get_kb_cache_root() -> Path:
|
||||
return Path(config.save_dir) / "knowledge_base_data" / "kb-cache"
|
||||
|
||||
|
||||
def normalize_knowledge_mount_name(name: str) -> str:
|
||||
normalized = _MULTISPACE.sub(" ", str(name or "").strip())
|
||||
if _INVALID_MOUNT_NAME_CHARS.search(normalized):
|
||||
raise ValueError("知识库名称包含不能映射为目录名的非法字符")
|
||||
normalized = normalized.strip(" .")
|
||||
if not normalized or normalized in {".", ".."}:
|
||||
raise ValueError("知识库名称不能映射为有效目录名")
|
||||
if "/" in normalized or "\\" in normalized:
|
||||
raise ValueError("知识库名称不能包含路径分隔符")
|
||||
return normalized
|
||||
|
||||
|
||||
def validate_knowledge_mount_name(name: str) -> str:
|
||||
return normalize_knowledge_mount_name(name)
|
||||
|
||||
|
||||
def _normalize_selected_knowledges(selected: list[str] | None) -> list[str]:
|
||||
normalized: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for item in selected or []:
|
||||
if not isinstance(item, str):
|
||||
continue
|
||||
value = item.strip()
|
||||
if not value:
|
||||
continue
|
||||
key = value.casefold()
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
normalized.append(value)
|
||||
return normalized
|
||||
|
||||
|
||||
def _derive_parsed_filename(filename: str, file_id: str, used_names: set[str]) -> str:
|
||||
raw_name = (filename or file_id or "file").strip() or file_id or "file"
|
||||
suffix = Path(raw_name).suffix
|
||||
stem = Path(raw_name).name[: -len(suffix)] if suffix else Path(raw_name).name
|
||||
candidate = f"{stem or file_id}.md"
|
||||
lowered = candidate.casefold()
|
||||
if lowered in used_names:
|
||||
candidate = f"{stem or file_id}__{file_id}.md"
|
||||
lowered = candidate.casefold()
|
||||
used_names.add(lowered)
|
||||
return candidate
|
||||
|
||||
|
||||
def _serialize_file_record(record) -> dict[str, Any]:
|
||||
return {
|
||||
"file_id": record.file_id,
|
||||
"db_id": record.db_id,
|
||||
"parent_id": record.parent_id,
|
||||
"filename": record.filename,
|
||||
"original_filename": record.original_filename,
|
||||
"file_type": record.file_type,
|
||||
"path": record.path,
|
||||
"minio_url": record.minio_url,
|
||||
"markdown_file": record.markdown_file,
|
||||
"status": record.status,
|
||||
"content_hash": record.content_hash,
|
||||
"file_size": record.file_size,
|
||||
"content_type": record.content_type,
|
||||
"processing_params": record.processing_params,
|
||||
"is_folder": bool(record.is_folder),
|
||||
}
|
||||
|
||||
|
||||
async def build_visible_knowledge_mounts(
|
||||
*,
|
||||
user_id: str,
|
||||
selected_knowledges: list[str] | None,
|
||||
) -> list[dict[str, Any]]:
|
||||
accessible = (await knowledge_base.get_databases_by_user_id(user_id)).get("databases", [])
|
||||
selected = _normalize_selected_knowledges(selected_knowledges)
|
||||
selected_keys = {item.casefold() for item in selected}
|
||||
|
||||
file_repo = KnowledgeFileRepository()
|
||||
mounts: list[dict[str, Any]] = []
|
||||
used_mount_names: dict[str, str] = {}
|
||||
|
||||
for db in accessible:
|
||||
db_id = str(db.get("db_id") or "").strip()
|
||||
db_name = str(db.get("name") or db_id).strip()
|
||||
if not db_id or not db_name:
|
||||
continue
|
||||
if selected_keys and db_name.casefold() not in selected_keys and db_id.casefold() not in selected_keys:
|
||||
continue
|
||||
|
||||
mount_name = normalize_knowledge_mount_name(db_name)
|
||||
conflict_db_id = used_mount_names.get(mount_name.casefold())
|
||||
if conflict_db_id and conflict_db_id != db_id:
|
||||
raise ValueError(f"知识库名称映射冲突: '{db_name}' -> '{mount_name}'")
|
||||
used_mount_names[mount_name.casefold()] = db_id
|
||||
|
||||
records = await file_repo.list_by_db_id(db_id)
|
||||
mounts.append(
|
||||
{
|
||||
"db_id": db_id,
|
||||
"db_name": db_name,
|
||||
"mount_name": mount_name,
|
||||
"files": [_serialize_file_record(record) for record in records],
|
||||
}
|
||||
)
|
||||
|
||||
mounts.sort(key=lambda item: item["mount_name"].casefold())
|
||||
return mounts
|
||||
|
||||
|
||||
def cache_minio_object(*, source_url: str, metadata: dict[str, Any] | None = None) -> Path:
|
||||
bucket_name, object_name = parse_minio_url(source_url)
|
||||
minio_client = get_minio_client()
|
||||
stat = minio_client.client.stat_object(bucket_name=bucket_name, object_name=object_name)
|
||||
etag = str(getattr(stat, "etag", "") or "")
|
||||
last_modified = getattr(stat, "last_modified", None)
|
||||
version_key = etag or (last_modified.isoformat() if last_modified else "")
|
||||
cache_key = hashlib.sha256(f"{bucket_name}:{object_name}:{version_key}".encode()).hexdigest()
|
||||
|
||||
suffix = Path(object_name).suffix
|
||||
objects_root = get_kb_cache_root() / "objects"
|
||||
manifests_root = get_kb_cache_root() / "manifests"
|
||||
objects_root.mkdir(parents=True, exist_ok=True)
|
||||
manifests_root.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
cached_path = objects_root / f"{cache_key}{suffix}"
|
||||
if not cached_path.exists():
|
||||
payload = minio_client.download_file(bucket_name=bucket_name, object_name=object_name)
|
||||
with tempfile.NamedTemporaryFile(dir=objects_root, delete=False) as tmp:
|
||||
tmp.write(payload)
|
||||
tmp_path = Path(tmp.name)
|
||||
tmp_path.replace(cached_path)
|
||||
|
||||
manifest_path = manifests_root / f"{cache_key}.json"
|
||||
if not manifest_path.exists():
|
||||
manifest = {
|
||||
"bucket_name": bucket_name,
|
||||
"object_name": object_name,
|
||||
"etag": etag,
|
||||
"last_modified": last_modified.isoformat() if last_modified else None,
|
||||
"source_url": source_url,
|
||||
"cached_path": str(cached_path),
|
||||
"metadata": metadata or {},
|
||||
}
|
||||
manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
return cached_path
|
||||
@ -12,7 +12,6 @@ import aiofiles
|
||||
from fastapi import HTTPException, UploadFile
|
||||
from fastapi.responses import FileResponse, StreamingResponse
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from yuxi.agents.backends import KBS_PATH, KnowledgeBaseReadonlyBackend, resolve_visible_knowledge_bases_for_context
|
||||
from yuxi.agents.backends.sandbox import (
|
||||
SKILLS_PATH,
|
||||
USER_DATA_PATH,
|
||||
@ -90,6 +89,7 @@ _PROTECTED_USER_DATA_ROOTS = frozenset(
|
||||
VIRTUAL_PATH_OUTPUTS,
|
||||
}
|
||||
)
|
||||
_LEGACY_KBS_PATH = "/home/gem/kbs"
|
||||
|
||||
|
||||
def _detect_preview_type(path: str, raw_content: bytes) -> tuple[str, bool, str | None]:
|
||||
@ -139,7 +139,7 @@ def _normalize_path(path: str | None) -> str:
|
||||
normalized = (path or "/").strip() or "/"
|
||||
if not normalized.startswith("/"):
|
||||
normalized = f"/{normalized}"
|
||||
return normalized.rstrip("/") if normalized not in {"/", KBS_PATH, SKILLS_PATH, USER_DATA_PATH} else normalized
|
||||
return normalized.rstrip("/") if normalized not in {"/", SKILLS_PATH, USER_DATA_PATH} else normalized
|
||||
|
||||
|
||||
def _is_path_within(path: Path, root: Path) -> bool:
|
||||
@ -180,10 +180,6 @@ def _is_skills_path(path: str) -> bool:
|
||||
return path == SKILLS_PATH or path.startswith(f"{SKILLS_PATH}/")
|
||||
|
||||
|
||||
def _is_kbs_path(path: str) -> bool:
|
||||
return path == KBS_PATH or path.startswith(f"{KBS_PATH}/")
|
||||
|
||||
|
||||
def _is_in_home_gem(path: str) -> bool:
|
||||
"""检查路径是否在 /home/gem/ 下但不在虚拟挂载点内"""
|
||||
if not path.startswith("/home/gem/"):
|
||||
@ -193,7 +189,7 @@ def _is_in_home_gem(path: str) -> bool:
|
||||
return False
|
||||
if path.startswith(f"{SKILLS_PATH}/") or path == SKILLS_PATH:
|
||||
return False
|
||||
if path.startswith(f"{KBS_PATH}/") or path == KBS_PATH:
|
||||
if path == _LEGACY_KBS_PATH or path.startswith(f"{_LEGACY_KBS_PATH}/"):
|
||||
return False
|
||||
return True
|
||||
|
||||
@ -204,12 +200,6 @@ def _strip_skills_prefix(path: str) -> str:
|
||||
return path[len(SKILLS_PATH) :] or "/"
|
||||
|
||||
|
||||
def _strip_kbs_prefix(path: str) -> str:
|
||||
if path == KBS_PATH:
|
||||
return "/"
|
||||
return path[len(KBS_PATH) :] or "/"
|
||||
|
||||
|
||||
def _remap_prefixed_entry(entry: dict, prefix: str) -> dict:
|
||||
raw_path = str(entry.get("path") or "")
|
||||
is_dir = bool(entry.get("is_dir", False))
|
||||
@ -358,11 +348,9 @@ async def _resolve_viewer_state(
|
||||
agent_id=agent_id,
|
||||
agent_config_id=agent_config_id,
|
||||
)
|
||||
visible_kbs = await resolve_visible_knowledge_bases_for_context(runtime_context)
|
||||
selected_skills = normalize_selected_skills(getattr(runtime_context, "skills", None) or [])
|
||||
skills_backend = SelectedSkillsReadonlyBackend(selected_slugs=selected_skills)
|
||||
kb_backend = KnowledgeBaseReadonlyBackend(visible_kbs=visible_kbs)
|
||||
return sandbox_backend, skills_backend, kb_backend, selected_skills
|
||||
return sandbox_backend, skills_backend, selected_skills
|
||||
|
||||
|
||||
async def list_viewer_filesystem_tree(
|
||||
@ -378,7 +366,7 @@ async def list_viewer_filesystem_tree(
|
||||
raise HTTPException(status_code=422, detail="thread_id 不能为空")
|
||||
|
||||
normalized_path = _normalize_path(path)
|
||||
sandbox_backend, skills_backend, kb_backend, selected_skills = await _resolve_viewer_state(
|
||||
sandbox_backend, skills_backend, selected_skills = await _resolve_viewer_state(
|
||||
thread_id=thread_id,
|
||||
agent_id=agent_id,
|
||||
agent_config_id=agent_config_id,
|
||||
@ -395,8 +383,6 @@ async def list_viewer_filesystem_tree(
|
||||
)
|
||||
if selected_skills:
|
||||
entries.append({"path": f"{SKILLS_PATH}/", "name": "skills", "is_dir": True, "size": 0, "modified_at": ""})
|
||||
if kb_backend.has_entries():
|
||||
entries.append({"path": f"{KBS_PATH}/", "name": "kbs", "is_dir": True, "size": 0, "modified_at": ""})
|
||||
|
||||
return {"entries": _sort_entries(entries)}
|
||||
|
||||
@ -419,10 +405,6 @@ async def list_viewer_filesystem_tree(
|
||||
entries = await asyncio.to_thread(skills_backend.ls_info, _strip_skills_prefix(normalized_path))
|
||||
remapped = [_remap_prefixed_entry(entry, SKILLS_PATH) for entry in entries]
|
||||
return {"entries": _sort_entries(remapped)}
|
||||
if _is_kbs_path(normalized_path):
|
||||
entries = await asyncio.to_thread(kb_backend.ls_info, _strip_kbs_prefix(normalized_path))
|
||||
remapped = [_remap_prefixed_entry(entry, KBS_PATH) for entry in entries]
|
||||
return {"entries": _sort_entries(remapped)}
|
||||
except PermissionError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e)) from e
|
||||
except ValueError as e:
|
||||
@ -444,7 +426,7 @@ async def read_viewer_file_content(
|
||||
raise HTTPException(status_code=422, detail="thread_id 不能为空")
|
||||
normalized_path = _normalize_path(path)
|
||||
|
||||
sandbox_backend, skills_backend, kb_backend, _selected_skills = await _resolve_viewer_state(
|
||||
sandbox_backend, skills_backend, _selected_skills = await _resolve_viewer_state(
|
||||
thread_id=thread_id,
|
||||
agent_id=agent_id,
|
||||
agent_config_id=agent_config_id,
|
||||
@ -476,8 +458,6 @@ async def read_viewer_file_content(
|
||||
}
|
||||
elif _is_skills_path(normalized_path):
|
||||
responses = await asyncio.to_thread(skills_backend.download_files, [_strip_skills_prefix(normalized_path)])
|
||||
elif _is_kbs_path(normalized_path):
|
||||
responses = await asyncio.to_thread(kb_backend.download_files, [_strip_kbs_prefix(normalized_path)])
|
||||
elif _is_in_home_gem(normalized_path):
|
||||
# /home/gem/ 下的其他文件（如 workspace 目录）
|
||||
responses = await asyncio.to_thread(sandbox_backend.download_files, [normalized_path])
|
||||
@ -537,7 +517,7 @@ async def download_viewer_file(
|
||||
db: AsyncSession,
|
||||
) -> StreamingResponse:
|
||||
normalized_path = _normalize_path(path)
|
||||
sandbox_backend, skills_backend, kb_backend, _selected_skills = await _resolve_viewer_state(
|
||||
sandbox_backend, skills_backend, _selected_skills = await _resolve_viewer_state(
|
||||
thread_id=thread_id,
|
||||
agent_id=agent_id,
|
||||
agent_config_id=agent_config_id,
|
||||
@ -562,8 +542,6 @@ async def download_viewer_file(
|
||||
|
||||
if _is_skills_path(normalized_path):
|
||||
responses = await asyncio.to_thread(skills_backend.download_files, [_strip_skills_prefix(normalized_path)])
|
||||
elif _is_kbs_path(normalized_path):
|
||||
responses = await asyncio.to_thread(kb_backend.download_files, [_strip_kbs_prefix(normalized_path)])
|
||||
elif _is_in_home_gem(normalized_path):
|
||||
# /home/gem/ 下的其他文件（如 workspace 目录）
|
||||
responses = await asyncio.to_thread(sandbox_backend.download_files, [normalized_path])
|
||||
|
||||
@ -9,7 +9,6 @@ WORKSPACE_AGENTS_PROMPT_FILE_NAME = "AGENTS.md"
|
||||
UPLOADS_DIR_NAME = "uploads"
|
||||
OUTPUTS_DIR_NAME = "outputs"
|
||||
VIRTUAL_SKILLS_PATH = "/home/gem/skills"
|
||||
VIRTUAL_KBS_PATH = "/home/gem/kbs"
|
||||
|
||||
VIRTUAL_PATH_WORKSPACE = (Path(VIRTUAL_PATH_PREFIX) / WORKSPACE_DIR_NAME).as_posix()
|
||||
VIRTUAL_PATH_UPLOADS = (Path(VIRTUAL_PATH_PREFIX) / UPLOADS_DIR_NAME).as_posix()
|
||||
@ -26,5 +25,4 @@ __all__ = [
|
||||
"VIRTUAL_PATH_UPLOADS",
|
||||
"VIRTUAL_PATH_OUTPUTS",
|
||||
"VIRTUAL_SKILLS_PATH",
|
||||
"VIRTUAL_KBS_PATH",
|
||||
]
|
||||
|
||||
@ -620,7 +620,7 @@ async def test_viewer_create_directory_rejects_invalid_names(test_client, standa
|
||||
assert response.status_code == 422, response.text
|
||||
|
||||
|
||||
async def test_viewer_tree_root_hides_kbs_namespace_when_no_database_is_visible(test_client, standard_user):
|
||||
async def test_viewer_tree_root_hides_kbs_namespace(test_client, standard_user):
|
||||
headers = standard_user["headers"]
|
||||
thread_id = await _create_thread_for_user(test_client, headers)
|
||||
|
||||
@ -633,13 +633,11 @@ async def test_viewer_tree_root_hides_kbs_namespace_when_no_database_is_visible(
|
||||
|
||||
entries = response.json().get("entries", [])
|
||||
paths = {entry.get("path") for entry in entries}
|
||||
if "/home/gem/kbs/" in paths:
|
||||
pytest.skip("Current integration database has visible knowledge bases.")
|
||||
assert "/home/gem/user-data/" in paths
|
||||
assert "/home/gem/kbs/" not in paths
|
||||
|
||||
|
||||
async def test_viewer_kbs_namespace_is_empty_when_no_database_is_visible(test_client, standard_user):
|
||||
async def test_viewer_rejects_kbs_namespace(test_client, standard_user):
|
||||
headers = standard_user["headers"]
|
||||
thread_id = await _create_thread_for_user(test_client, headers)
|
||||
|
||||
@ -648,8 +646,18 @@ async def test_viewer_kbs_namespace_is_empty_when_no_database_is_visible(test_cl
|
||||
params={"thread_id": thread_id, "path": "/home/gem/kbs"},
|
||||
headers=headers,
|
||||
)
|
||||
assert tree_response.status_code == 200, tree_response.text
|
||||
entries = tree_response.json().get("entries", [])
|
||||
if entries:
|
||||
pytest.skip("Current integration database has visible knowledge bases.")
|
||||
assert entries == []
|
||||
assert tree_response.status_code == 400, tree_response.text
|
||||
|
||||
file_response = await test_client.get(
|
||||
"/api/viewer/filesystem/file",
|
||||
params={"thread_id": thread_id, "path": "/home/gem/kbs/demo.md"},
|
||||
headers=headers,
|
||||
)
|
||||
assert file_response.status_code == 400, file_response.text
|
||||
|
||||
download_response = await test_client.get(
|
||||
"/api/viewer/filesystem/download",
|
||||
params={"thread_id": thread_id, "path": "/home/gem/kbs/demo.md"},
|
||||
headers=headers,
|
||||
)
|
||||
assert download_response.status_code == 400, download_response.text
|
||||
|
||||
@ -3,93 +3,7 @@ from __future__ import annotations
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from yuxi.agents.backends.knowledge_base_backend import (
|
||||
KBS_PATH,
|
||||
KnowledgeBaseReadonlyBackend,
|
||||
build_knowledge_base_filepath_map,
|
||||
inject_filepaths_into_retrieval_result,
|
||||
resolve_file_relative_virtual_path,
|
||||
resolve_visible_knowledge_bases_for_context,
|
||||
)
|
||||
|
||||
|
||||
def test_knowledge_base_backend_builds_virtual_tree_and_materializes_files(monkeypatch, tmp_path) -> None:
|
||||
visible_kbs = [
|
||||
{"db_id": "db-1", "name": "FAQ"},
|
||||
{"db_id": "db-2", "name": "FAQ"},
|
||||
]
|
||||
files_meta = {
|
||||
"folder-api": {
|
||||
"file_id": "folder-api",
|
||||
"database_id": "db-1",
|
||||
"parent_id": None,
|
||||
"filename": "API",
|
||||
"is_folder": True,
|
||||
"created_at": "2026-03-26T00:00:00Z",
|
||||
},
|
||||
"file-pdf": {
|
||||
"file_id": "file-pdf",
|
||||
"database_id": "db-1",
|
||||
"parent_id": "folder-api",
|
||||
"filename": "auth-guide.pdf",
|
||||
"path": "http://minio/kb-source/db-1/upload/auth-guide.pdf",
|
||||
"markdown_file": "http://minio/kb-parsed/db-1/parsed/file-pdf.md",
|
||||
"size": 12,
|
||||
"is_folder": False,
|
||||
"created_at": "2026-03-26T00:00:00Z",
|
||||
},
|
||||
"file-pdf-2": {
|
||||
"file_id": "file-pdf-2",
|
||||
"database_id": "db-1",
|
||||
"parent_id": "folder-api",
|
||||
"filename": "auth-guide.pdf",
|
||||
"path": "http://minio/kb-source/db-1/upload/auth-guide-2.pdf",
|
||||
"size": 7,
|
||||
"is_folder": False,
|
||||
"created_at": "2026-03-26T00:00:00Z",
|
||||
},
|
||||
"file-db2": {
|
||||
"file_id": "file-db2",
|
||||
"database_id": "db-2",
|
||||
"parent_id": None,
|
||||
"filename": "overview.txt",
|
||||
"path": "http://minio/kb-source/db-2/upload/overview.txt",
|
||||
"size": 5,
|
||||
"is_folder": False,
|
||||
"created_at": "2026-03-26T00:00:00Z",
|
||||
},
|
||||
}
|
||||
monkeypatch.setattr("yuxi.agents.backends.knowledge_base_backend._all_files_meta", lambda: files_meta)
|
||||
|
||||
class _FakeMinio:
|
||||
def download_file(self, bucket_name: str, object_name: str) -> bytes:
|
||||
return f"{bucket_name}:{object_name}".encode()
|
||||
|
||||
monkeypatch.setattr("yuxi.agents.backends.knowledge_base_backend.get_minio_client", lambda: _FakeMinio())
|
||||
|
||||
backend = KnowledgeBaseReadonlyBackend(visible_kbs=visible_kbs, cache_root=tmp_path)
|
||||
|
||||
root_entries = {entry["path"] for entry in backend.ls_info("/")}
|
||||
assert "/FAQ/" in root_entries
|
||||
assert "/FAQ__db-2/" in root_entries
|
||||
|
||||
faq_entries = {entry["path"] for entry in backend.ls_info("/FAQ")}
|
||||
assert "/FAQ/API/" in faq_entries
|
||||
assert "/FAQ/parsed/" in faq_entries
|
||||
|
||||
api_entries = {entry["path"] for entry in backend.ls_info("/FAQ/API")}
|
||||
assert "/FAQ/API/auth-guide.pdf" in api_entries
|
||||
assert any(path.startswith("/FAQ/API/auth-guide.pdf__file-pdf") for path in api_entries)
|
||||
|
||||
parsed_entries = {entry["path"] for entry in backend.ls_info("/FAQ/parsed/API")}
|
||||
assert "/FAQ/parsed/API/auth-guide.pdf.md" in parsed_entries
|
||||
|
||||
responses = backend.download_files(["/FAQ/API/auth-guide.pdf", "/FAQ/parsed/API/auth-guide.pdf.md"])
|
||||
assert responses[0].content == b"kb-source:db-1/upload/auth-guide.pdf"
|
||||
assert responses[1].content == b"kb-parsed:db-1/parsed/file-pdf.md"
|
||||
|
||||
assert "kb-source" in backend.read("/FAQ/API/auth-guide.pdf")
|
||||
assert backend.write("/FAQ/API/new.txt", "x").error == "Knowledge base path is read-only."
|
||||
from yuxi.agents.backends.knowledge_base_backend import resolve_visible_knowledge_bases_for_context
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@ -110,216 +24,41 @@ async def test_resolve_visible_knowledge_bases_for_context_filters_by_enabled_na
|
||||
assert getattr(context, "_visible_knowledge_bases") == visible
|
||||
|
||||
|
||||
def test_build_knowledge_base_filepath_map_matches_virtual_tree(monkeypatch, tmp_path) -> None:
|
||||
visible_kbs = [
|
||||
{"db_id": "db-1", "name": "FAQ"},
|
||||
{"db_id": "db-2", "name": "FAQ"},
|
||||
]
|
||||
files_meta = {
|
||||
"folder-api": {
|
||||
"file_id": "folder-api",
|
||||
"database_id": "db-1",
|
||||
"parent_id": None,
|
||||
"filename": "API",
|
||||
"is_folder": True,
|
||||
"created_at": "2026-03-26T00:00:00Z",
|
||||
},
|
||||
"file-pdf": {
|
||||
"file_id": "file-pdf",
|
||||
"database_id": "db-1",
|
||||
"parent_id": "folder-api",
|
||||
"filename": "auth-guide.pdf",
|
||||
"path": "http://minio/kb-source/db-1/upload/auth-guide.pdf",
|
||||
"size": 12,
|
||||
"is_folder": False,
|
||||
"created_at": "2026-03-26T00:00:00Z",
|
||||
},
|
||||
"file-pdf-2": {
|
||||
"file_id": "file-pdf-2",
|
||||
"database_id": "db-1",
|
||||
"parent_id": "folder-api",
|
||||
"filename": "auth-guide.pdf",
|
||||
"path": "http://minio/kb-source/db-1/upload/auth-guide-2.pdf",
|
||||
"size": 7,
|
||||
"is_folder": False,
|
||||
"created_at": "2026-03-26T00:00:00Z",
|
||||
},
|
||||
"file-db2": {
|
||||
"file_id": "file-db2",
|
||||
"database_id": "db-2",
|
||||
"parent_id": None,
|
||||
"filename": "overview.txt",
|
||||
"path": "http://minio/kb-source/db-2/upload/overview.txt",
|
||||
"size": 5,
|
||||
"is_folder": False,
|
||||
"created_at": "2026-03-26T00:00:00Z",
|
||||
},
|
||||
}
|
||||
monkeypatch.setattr("yuxi.agents.backends.knowledge_base_backend._all_files_meta", lambda: files_meta)
|
||||
|
||||
filepath_map = build_knowledge_base_filepath_map(visible_kbs=visible_kbs, files_meta=files_meta)
|
||||
backend = KnowledgeBaseReadonlyBackend(visible_kbs=visible_kbs, cache_root=tmp_path)
|
||||
|
||||
assert filepath_map["file-pdf"] == f"{KBS_PATH}/FAQ/API/auth-guide.pdf"
|
||||
assert filepath_map["file-db2"] == f"{KBS_PATH}/FAQ__db-2/overview.txt"
|
||||
assert filepath_map["file-pdf-2"].startswith(f"{KBS_PATH}/FAQ/API/auth-guide.pdf__")
|
||||
|
||||
mapped_virtual_paths = {path[len(KBS_PATH) :] for path in filepath_map.values()}
|
||||
for virtual_path in mapped_virtual_paths:
|
||||
assert virtual_path in backend._files
|
||||
|
||||
|
||||
def test_resolve_file_relative_virtual_path_by_file_id(monkeypatch) -> None:
|
||||
visible_kbs = [{"db_id": "db-1", "name": "食品 相关文献"}]
|
||||
files_meta = {
|
||||
"folder-1": {
|
||||
"file_id": "folder-1",
|
||||
"database_id": "db-1",
|
||||
"parent_id": None,
|
||||
"filename": "课程",
|
||||
"is_folder": True,
|
||||
"created_at": "2026-03-26T00:00:00Z",
|
||||
},
|
||||
"file_79c496": {
|
||||
"file_id": "file_79c496",
|
||||
"database_id": "db-1",
|
||||
"parent_id": "folder-1",
|
||||
"filename": "营养与食品课程中的思政建设研究.pdf",
|
||||
"path": "http://minio/knowledgebases/db-1/uploads/file_79c496.pdf",
|
||||
"is_folder": False,
|
||||
"size": 100,
|
||||
"created_at": "2026-03-26T00:00:00Z",
|
||||
},
|
||||
}
|
||||
monkeypatch.setattr("yuxi.agents.backends.knowledge_base_backend._all_files_meta", lambda: files_meta)
|
||||
|
||||
relative_path = resolve_file_relative_virtual_path(file_id="file_79c496", visible_kbs=visible_kbs)
|
||||
absolute_map = build_knowledge_base_filepath_map(visible_kbs=visible_kbs, files_meta=files_meta)
|
||||
|
||||
assert relative_path == "/食品 相关文献/课程/营养与食品课程中的思政建设研究.pdf"
|
||||
assert absolute_map["file_79c496"] == f"{KBS_PATH}{relative_path}"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_inject_filepaths_into_retrieval_result_injects_by_file_id(monkeypatch) -> None:
|
||||
retrieval_chunks = [
|
||||
{
|
||||
"content": "auth guide",
|
||||
"metadata": {
|
||||
"file_id": "file-1",
|
||||
"source": "auth-guide.pdf",
|
||||
},
|
||||
}
|
||||
]
|
||||
async def test_resolve_visible_knowledge_bases_for_context_defaults_to_all_accessible(monkeypatch) -> None:
|
||||
databases = [{"db_id": "db-1", "name": "Alpha"}, {"db_id": "db-2", "name": "Beta"}]
|
||||
|
||||
async def _fake_get_databases_by_raw_id(user_id: int) -> dict:
|
||||
assert user_id == 7
|
||||
return {"databases": databases}
|
||||
|
||||
monkeypatch.setattr(
|
||||
"yuxi.agents.backends.knowledge_base_backend._resolve_virtual_layout",
|
||||
lambda **kwargs: SimpleNamespace(
|
||||
source_filepaths={"file-1": f"{KBS_PATH}/FAQ/auth-guide.pdf"},
|
||||
parsed_filepaths={"file-1": f"{KBS_PATH}/FAQ/parsed/auth-guide.pdf.md"},
|
||||
),
|
||||
"yuxi.agents.backends.knowledge_base_backend.knowledge_base.get_databases_by_raw_id",
|
||||
_fake_get_databases_by_raw_id,
|
||||
)
|
||||
|
||||
result = await inject_filepaths_into_retrieval_result(
|
||||
retrieval_chunks=retrieval_chunks,
|
||||
visible_kbs=[{"db_id": "db-1", "name": "FAQ"}],
|
||||
target_db_id="db-1",
|
||||
)
|
||||
context = SimpleNamespace(user_id="7", knowledges=None)
|
||||
visible = await resolve_visible_knowledge_bases_for_context(context)
|
||||
|
||||
assert result[0]["metadata"]["filepath"] == f"{KBS_PATH}/FAQ/auth-guide.pdf"
|
||||
assert result[0]["metadata"]["parsed_path"] == f"{KBS_PATH}/FAQ/parsed/auth-guide.pdf.md"
|
||||
assert visible == databases
|
||||
assert getattr(context, "_visible_knowledge_bases") == databases
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_inject_filepaths_into_retrieval_result_injects_parsed_path_when_markdown_exists(monkeypatch) -> None:
|
||||
files_meta = {
|
||||
"file-1": {
|
||||
"file_id": "file-1",
|
||||
"database_id": "db-1",
|
||||
"parent_id": None,
|
||||
"filename": "auth-guide.pdf",
|
||||
"path": "http://minio/kb-source/db-1/upload/auth-guide.pdf",
|
||||
"markdown_file": "http://minio/kb-parsed/db-1/parsed/file-1.md",
|
||||
"is_folder": False,
|
||||
"created_at": "2026-03-26T00:00:00Z",
|
||||
}
|
||||
}
|
||||
monkeypatch.setattr("yuxi.agents.backends.knowledge_base_backend._all_files_meta", lambda: files_meta)
|
||||
async def test_resolve_visible_knowledge_bases_for_context_handles_missing_user() -> None:
|
||||
context = SimpleNamespace(user_id="", knowledges=None)
|
||||
|
||||
retrieval_chunks = [{"content": "auth guide", "metadata": {"file_id": "file-1"}}]
|
||||
result = await inject_filepaths_into_retrieval_result(
|
||||
retrieval_chunks=retrieval_chunks,
|
||||
visible_kbs=[{"db_id": "db-1", "name": "FAQ"}],
|
||||
target_db_id="db-1",
|
||||
)
|
||||
visible = await resolve_visible_knowledge_bases_for_context(context)
|
||||
|
||||
assert result[0]["metadata"]["filepath"] == f"{KBS_PATH}/FAQ/auth-guide.pdf"
|
||||
assert result[0]["metadata"]["parsed_path"] == f"{KBS_PATH}/FAQ/parsed/auth-guide.pdf.md"
|
||||
assert visible == []
|
||||
assert getattr(context, "_visible_knowledge_bases") == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_inject_filepaths_into_retrieval_result_does_not_use_filename_fallback(monkeypatch) -> None:
|
||||
retrieval_chunks = [
|
||||
{
|
||||
"id": "chunk-1",
|
||||
"metadata": {
|
||||
"source": "auth-guide.pdf",
|
||||
},
|
||||
}
|
||||
]
|
||||
async def test_resolve_visible_knowledge_bases_for_context_handles_invalid_user() -> None:
|
||||
context = SimpleNamespace(user_id="not-a-number", knowledges=None)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"yuxi.agents.backends.knowledge_base_backend._resolve_virtual_layout",
|
||||
lambda **kwargs: SimpleNamespace(
|
||||
source_filepaths={
|
||||
"file-1": f"{KBS_PATH}/FAQ/API/auth-guide.pdf",
|
||||
"file-2": f"{KBS_PATH}/FAQ/API/another.pdf",
|
||||
},
|
||||
parsed_filepaths={
|
||||
"file-1": f"{KBS_PATH}/FAQ/parsed/API/auth-guide.pdf.md",
|
||||
"file-2": f"{KBS_PATH}/FAQ/parsed/API/another.pdf.md",
|
||||
},
|
||||
),
|
||||
)
|
||||
visible = await resolve_visible_knowledge_bases_for_context(context)
|
||||
|
||||
result = await inject_filepaths_into_retrieval_result(
|
||||
retrieval_chunks=retrieval_chunks,
|
||||
visible_kbs=[{"db_id": "db-1", "name": "FAQ"}],
|
||||
target_db_id="db-1",
|
||||
)
|
||||
|
||||
assert "filepath" not in result[0]["metadata"]
|
||||
assert "parsed_path" not in result[0]["metadata"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_inject_filepaths_into_retrieval_result_requires_explicit_file_id(monkeypatch) -> None:
|
||||
retrieval_chunks = [
|
||||
{
|
||||
"content": "chunk",
|
||||
"metadata": {
|
||||
"source": "http://172.19.13.5:9000/knowledgebases/kb-1/parsed/file_79c496.md",
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
monkeypatch.setattr(
|
||||
"yuxi.agents.backends.knowledge_base_backend._resolve_virtual_layout",
|
||||
lambda **kwargs: SimpleNamespace(
|
||||
source_filepaths={
|
||||
"file_79c496": f"{KBS_PATH}/食品 相关文献/课程/营养与食品课程中的思政建设研究.pdf",
|
||||
},
|
||||
parsed_filepaths={
|
||||
"file_79c496": f"{KBS_PATH}/食品 相关文献/parsed/课程/营养与食品课程中的思政建设研究.pdf.md",
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
result = await inject_filepaths_into_retrieval_result(
|
||||
retrieval_chunks=retrieval_chunks,
|
||||
visible_kbs=[{"db_id": "db-1", "name": "食品 相关文献"}],
|
||||
target_db_id="db-1",
|
||||
)
|
||||
|
||||
assert "filepath" not in result[0]["metadata"]
|
||||
assert "parsed_path" not in result[0]["metadata"]
|
||||
assert visible == []
|
||||
assert getattr(context, "_visible_knowledge_bases") == []
|
||||
|
||||
@ -39,7 +39,7 @@ def test_create_agent_composite_backend_uses_provisioner_default(monkeypatch):
|
||||
assert isinstance(backend.default, ProvisionerSandboxBackend)
|
||||
assert backend.default._visible_skills == ["reporter"]
|
||||
assert "/skills/" in backend.routes
|
||||
assert "/home/gem/kbs/" in backend.routes
|
||||
assert "/home/gem/kbs/" not in backend.routes
|
||||
|
||||
|
||||
def test_create_agent_composite_backend_requires_thread_id():
|
||||
|
||||
@ -62,7 +62,7 @@ def _build_test_window(content: str, offset: int = 0, limit: int = 800) -> dict:
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_query_kb_injects_filepath_into_chunk_metadata(monkeypatch) -> None:
|
||||
async def test_query_kb_returns_milvus_chunks_without_sandbox_paths(monkeypatch) -> None:
|
||||
async def _fake_retriever(query_text: str, **kwargs):
|
||||
assert query_text == "auth"
|
||||
return [
|
||||
@ -92,23 +92,20 @@ async def test_query_kb_injects_filepath_into_chunk_metadata(monkeypatch) -> Non
|
||||
|
||||
monkeypatch.setattr(tools, "_resolve_visible_knowledge_bases_for_query", _fake_visible_kbs)
|
||||
|
||||
async def _fake_inject(*, retrieval_chunks, visible_kbs, target_db_id, target_kb_name=None):
|
||||
assert visible_kbs == [{"db_id": "db-1", "name": "FAQ"}]
|
||||
assert target_db_id == "db-1"
|
||||
retrieval_chunks[0]["metadata"]["filepath"] = "/home/gem/kbs/FAQ/API/auth-guide.pdf"
|
||||
retrieval_chunks[0]["metadata"]["parsed_path"] = "/home/gem/kbs/FAQ/parsed/API/auth-guide.pdf.md"
|
||||
return retrieval_chunks
|
||||
|
||||
monkeypatch.setattr(
|
||||
"yuxi.agents.backends.knowledge_base_backend.inject_filepaths_into_retrieval_result",
|
||||
_fake_inject,
|
||||
)
|
||||
|
||||
runtime = SimpleNamespace(context=SimpleNamespace())
|
||||
result = await _run_query_kb(kb_name="FAQ", query_text="auth", runtime=runtime)
|
||||
|
||||
assert result[0]["metadata"]["filepath"] == "/home/gem/kbs/FAQ/API/auth-guide.pdf"
|
||||
assert result[0]["metadata"]["parsed_path"] == "/home/gem/kbs/FAQ/parsed/API/auth-guide.pdf.md"
|
||||
assert result == [
|
||||
{
|
||||
"content": "auth guide",
|
||||
"metadata": {
|
||||
"file_id": "file-1",
|
||||
"source": "auth-guide.pdf",
|
||||
},
|
||||
}
|
||||
]
|
||||
assert "filepath" not in result[0]["metadata"]
|
||||
assert "parsed_path" not in result[0]["metadata"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@ -142,10 +139,6 @@ async def test_query_kb_allows_dify_knowledge_base(monkeypatch) -> None:
|
||||
return [{"db_id": "db-1", "name": "FAQ"}]
|
||||
|
||||
monkeypatch.setattr(tools, "_resolve_visible_knowledge_bases_for_query", _fake_visible_kbs)
|
||||
monkeypatch.setattr(
|
||||
"yuxi.agents.backends.knowledge_base_backend.inject_filepaths_into_retrieval_result",
|
||||
pytest.fail,
|
||||
)
|
||||
|
||||
runtime = SimpleNamespace(context=SimpleNamespace())
|
||||
result = await _run_query_kb(kb_name="FAQ", query_text="auth", runtime=runtime)
|
||||
@ -163,7 +156,7 @@ async def test_query_kb_allows_dify_knowledge_base(monkeypatch) -> None:
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_query_kb_returns_lightrag_result_without_filepath_injection(monkeypatch) -> None:
|
||||
async def test_query_kb_returns_lightrag_result_without_path_injection(monkeypatch) -> None:
|
||||
async def _fake_retriever(query_text: str, **kwargs):
|
||||
assert query_text == "auth"
|
||||
return "LightRAG context"
|
||||
@ -184,10 +177,6 @@ async def test_query_kb_returns_lightrag_result_without_filepath_injection(monke
|
||||
return [{"db_id": "db-1", "name": "FAQ"}]
|
||||
|
||||
monkeypatch.setattr(tools, "_resolve_visible_knowledge_bases_for_query", _fake_visible_kbs)
|
||||
monkeypatch.setattr(
|
||||
"yuxi.agents.backends.knowledge_base_backend.inject_filepaths_into_retrieval_result",
|
||||
pytest.fail,
|
||||
)
|
||||
|
||||
runtime = SimpleNamespace(context=SimpleNamespace())
|
||||
result = await _run_query_kb(kb_name="FAQ", query_text="auth", runtime=runtime)
|
||||
@ -195,55 +184,6 @@ async def test_query_kb_returns_lightrag_result_without_filepath_injection(monke
|
||||
assert result == "LightRAG context"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_query_kb_uses_backend_filepath_injector(monkeypatch) -> None:
|
||||
async def _fake_retriever(query_text: str, **kwargs):
|
||||
assert query_text == "auth"
|
||||
return [
|
||||
{
|
||||
"content": "auth guide",
|
||||
"metadata": {
|
||||
"file_id": "file-1",
|
||||
"source": "auth-guide.pdf",
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
monkeypatch.setattr(
|
||||
tools.knowledge_base,
|
||||
"get_retrievers",
|
||||
lambda: {
|
||||
"db-1": {
|
||||
"name": "FAQ",
|
||||
"retriever": _fake_retriever,
|
||||
"metadata": {"kb_type": "milvus"},
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
async def _fake_visible_kbs(runtime):
|
||||
return [{"db_id": "db-1", "name": "FAQ"}]
|
||||
|
||||
async def _fake_inject(*, retrieval_chunks, visible_kbs, target_db_id, target_kb_name=None):
|
||||
assert visible_kbs == [{"db_id": "db-1", "name": "FAQ"}]
|
||||
assert target_db_id == "db-1"
|
||||
retrieval_chunks[0]["metadata"]["filepath"] = "/home/gem/kbs/FAQ/auth-guide.pdf"
|
||||
retrieval_chunks[0]["metadata"]["parsed_path"] = "/home/gem/kbs/FAQ/parsed/auth-guide.pdf.md"
|
||||
return retrieval_chunks
|
||||
|
||||
monkeypatch.setattr(tools, "_resolve_visible_knowledge_bases_for_query", _fake_visible_kbs)
|
||||
monkeypatch.setattr(
|
||||
"yuxi.agents.backends.knowledge_base_backend.inject_filepaths_into_retrieval_result",
|
||||
_fake_inject,
|
||||
)
|
||||
|
||||
runtime = SimpleNamespace(context=SimpleNamespace())
|
||||
result = await _run_query_kb(kb_name="FAQ", query_text="auth", runtime=runtime)
|
||||
|
||||
assert result[0]["metadata"]["filepath"] == "/home/gem/kbs/FAQ/auth-guide.pdf"
|
||||
assert result[0]["metadata"]["parsed_path"] == "/home/gem/kbs/FAQ/parsed/auth-guide.pdf.md"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_query_kb_normalizes_file_metadata_for_open(monkeypatch) -> None:
|
||||
async def _fake_retriever(query_text: str, **kwargs):
|
||||
|
||||
@ -6,9 +6,9 @@
|
||||
|
||||
我们在 Yuxi 里引入沙盒，不是为了让架构更“重”，而是因为 Agent 一旦从纯文本对话进入真实执行阶段，就一定会碰到一组很具体的运行时需求：执行命令、读写文件、处理用户上传附件、产出可下载结果，以及在受控目录里保留中间过程文件。如果把这些能力直接放进 API 进程本身，权限边界、租户隔离、环境一致性和后续运维成本都会迅速恶化。
|
||||
|
||||
从设计目标上看，沙盒这一层主要解决三件事。第一，给 Agent 一个可写、可执行、可回收的独立运行空间，而不是让它直接操作应用主进程。第二，把模型可见文件系统整理成稳定的命名空间，例如 `/home/gem/user-data`、`/home/gem/skills`、`/home/gem/kbs`，这样 prompt、工具、viewer 和 artifact 下载接口可以共享同一套路径语义。第三，让这套能力既能在本地 Docker 开发环境里稳定工作，也能在需要时切到 Kubernetes 这类更适合多实例部署的承载方式。
|
||||
从设计目标上看，沙盒这一层主要解决三件事。第一，给 Agent 一个可写、可执行、可回收的独立运行空间，而不是让它直接操作应用主进程。第二，把模型可见文件系统整理成稳定的命名空间，例如 `/home/gem/user-data` 和 `/home/gem/skills`，这样 prompt、工具、viewer 和 artifact 下载接口可以共享同一套路径语义。第三，让这套能力既能在本地 Docker 开发环境里稳定工作，也能在需要时切到 Kubernetes 这类更适合多实例部署的承载方式。
|
||||
|
||||
这份文档说明当前项目中“沙盒”这一层到底是什么、为什么同时会看到 Docker 和 Kubernetes、默认开发环境实际启用的是哪一种模式，以及沙盒如何和 `skills`、知识库、附件、工作区文件系统组合在一起工作。内容以当前仓库实现为准，我们重点解释真实调用链、配置入口、路径语义和运维边界，而不是抽象地介绍容器技术。
|
||||
这份文档说明当前项目中“沙盒”这一层到底是什么、为什么同时会看到 Docker 和 Kubernetes、默认开发环境实际启用的是哪一种模式，以及沙盒如何和 `skills`、附件、工作区文件系统组合在一起工作。内容以当前仓库实现为准，我们重点解释真实调用链、配置入口、路径语义和运维边界，而不是抽象地介绍容器技术。
|
||||
|
||||
## 一、先说明白：Docker 和 K8s 在这里是什么关系
|
||||
|
||||
@ -114,7 +114,7 @@ services:
|
||||
|
||||
## 八、当前项目的沙盒文件系统是如何设计的
|
||||
|
||||
从模型和工具调用的视角看，Yuxi 主要向 Agent 暴露三类路径：`/home/gem/user-data`、`/home/gem/skills` 和 `/home/gem/kbs`。其中 `user-data` 是可写的用户工作区，`skills` 是只读的技能目录，`kbs` 是只读的知识库映射目录。
|
||||
从模型和工具调用的视角看，Yuxi 主要向 Agent 暴露两类路径：`/home/gem/user-data` 和 `/home/gem/skills`。其中 `user-data` 是可写的用户工作区，`skills` 是只读的技能目录。知识库不再映射为沙盒文件系统路径，模型应通过知识库工具检索和打开文档。
|
||||
|
||||
在宿主机侧，和线程相关的数据主要放在 `saves` 目录下。当前可读的目录结构可以概括为下面这样：
|
||||
|
||||
@ -149,7 +149,7 @@ Yuxi 不会把整个容器文件系统都开放给 Agent 或 viewer。当前 vie
|
||||
|
||||
`/home/gem/skills` 是只读目录。它不是简单地把 `saves/skills` 整个暴露进去，而是先根据当前线程可见的 skill 列表，把这些技能从全局 skills 根目录同步复制到 `saves/threads/<thread_id>/skills`，再把这个线程目录只读挂进沙盒。这样做的结果是，不同线程看到的 skill 集可能不同，而且模型永远不能在运行时修改 skills 内容。
|
||||
|
||||
`/home/gem/kbs` 也是只读目录。它不是物理直挂一个宿主机目录，而是由 `KnowledgeBaseReadonlyBackend` 动态组织出来的一棵虚拟树。这个树只暴露“当前用户可访问的知识库”和“当前 Agent 上启用的知识库”的交集，并且会同时组织源文件和解析后的 Markdown 视图。对于模型来说，这个目录更像一个只读文件系统投影，而不是原始磁盘路径。
|
||||
知识库访问不属于沙盒文件系统暴露规则。当前 Agent 可见知识库仍由用户权限和 Agent 配置共同决定，但只通过 `query_kb`、`open_kb_document` 等工具访问，不提供沙盒目录投影。
|
||||
|
||||
## 十、skills、知识库、附件是怎么和沙盒结合的
|
||||
|
||||
@ -157,7 +157,7 @@ skills 的结合方式分成两层。第一层是提示词层，`SkillsMiddlewar
|
||||
|
||||
附件的结合方式更偏向“先落盘，再把路径告诉模型”。用户上传文件后，系统会先把原始文件写入 `saves/threads/<thread_id>/user-data/uploads`。如果该文件可以被解析，系统还会额外生成一个 Markdown 副本，写到 `saves/threads/<thread_id>/user-data/uploads/attachments/<name>.md`。随后，LangGraph state 中会维护一份 `uploads` 列表，`AttachmentMiddleware` 会把这些可读路径注入系统提示，告诉模型优先用 `read_file` 去读取这些路径。因此，附件并不是“作为消息大段内联塞给模型”，而是被转换成沙盒文件系统中的路径对象。
|
||||
|
||||
知识库则是另一种只读投影。它不会被复制到每个线程目录，而是按当前运行上下文动态生成 `/home/gem/kbs` 虚拟树。模型既可以通过专门的知识库工具检索，也可以在某些需要高精度定位原始内容的场景下直接遍历 `/home/gem/kbs/<db_name>/...`。内置 prompt 里已经明确提到，解析后的 Markdown 通常位于 `parsed` 视图下，这样模型在工具检索不足时还有一个明确的文件系统后备路径。
|
||||
知识库不再与沙盒文件系统结合。它不会被复制到每个线程目录，也不会生成虚拟目录；模型通过专门的知识库工具检索，并在需要更完整上下文时用 `open_kb_document` 按 `resource_id` 和 `file_id` 打开文档内容。
|
||||
|
||||
## 十一、当前推荐如何使用 Docker 沙盒
|
||||
|
||||
@ -186,9 +186,9 @@ curl http://localhost:8002/health
|
||||
|
||||
## 十二、如何理解文件管理与暴露边界
|
||||
|
||||
从产品行为上看，viewer 文件系统和 artifact 下载接口优先走的是宿主机路径解析，而不是无条件透传到沙盒容器内部。这么设计有两个直接收益。第一，浏览 `/` 或 `/home/gem/user-data` 这样的树形入口时，不需要为了只读查看而冷启动沙盒。第二，权限边界更好做，因为 `resolve_virtual_path` 会把用户可见路径严格限制在预定义的 `user-data`、`skills`、`kbs` 命名空间内。
|
||||
从产品行为上看，viewer 文件系统和 artifact 下载接口优先走的是宿主机路径解析，而不是无条件透传到沙盒容器内部。这么设计有两个直接收益。第一，浏览 `/` 或 `/home/gem/user-data` 这样的树形入口时，不需要为了只读查看而冷启动沙盒。第二，权限边界更好做，因为 `resolve_virtual_path` 会把用户可见路径严格限制在预定义的 `user-data` 和 `skills` 命名空间内。
|
||||
|
||||
从工程上看，当前实现更像“双层文件系统”。对 Agent 执行来说，真正工作的对象是远程沙盒进程暴露的文件 API；对 viewer、附件下载和一部分 artifact 查看来说，系统会优先在宿主机侧解析虚拟路径，再用本地文件读取或只读 backend 下载内容。这也是为什么你会看到既有 `ProvisionerSandboxBackend`，又有 `viewer_filesystem_service`、`SelectedSkillsReadonlyBackend`、`KnowledgeBaseReadonlyBackend` 这样的配套实现。
|
||||
从工程上看，当前实现更像“双层文件系统”。对 Agent 执行来说，真正工作的对象是远程沙盒进程暴露的文件 API；对 viewer、附件下载和一部分 artifact 查看来说，系统会优先在宿主机侧解析虚拟路径，再用本地文件读取或只读 backend 下载内容。这也是为什么你会看到既有 `ProvisionerSandboxBackend`，又有 `viewer_filesystem_service`、`SelectedSkillsReadonlyBackend` 这样的配套实现。
|
||||
|
||||
## 十三、环境变量配置与传递链
|
||||
|
||||
@ -302,4 +302,4 @@ CHECK_YUXI_SANDBOX_ENV_EXISTS=True
|
||||
|
||||
如果怀疑是 Docker 地址不可达，重点检查 `SANDBOX_DOCKER_SANDBOX_HOST` 和随机映射端口是否从 `api` 容器可访问。可以在 `api` 容器内直接 `curl` provisioner 返回的 `sandbox_url`。如果怀疑是 Kubernetes 地址不可达，重点检查 `NODE_HOST` 和 NodePort 的外部连通性，因为当前实现并不是通过集群内部 Service 名称回连。
|
||||
|
||||
如果怀疑是文件看得到但模型读不到，或者模型写了但 viewer 看不到，优先把问题拆成两层：一层是宿主机路径是否存在于 `saves/...` 下，另一层是该路径是否真的被当前线程沙盒挂载并暴露到了 `/home/gem/user-data`、`/home/gem/skills` 或 `/home/gem/kbs`。只要先分清“宿主机侧文件语义”和“沙盒侧运行时挂载语义”，定位问题通常会快很多。
|
||||
如果怀疑是文件看得到但模型读不到，或者模型写了但 viewer 看不到，优先把问题拆成两层：一层是宿主机路径是否存在于 `saves/...` 下，另一层是该路径是否真的被当前线程沙盒挂载并暴露到了 `/home/gem/user-data` 或 `/home/gem/skills`。只要先分清“宿主机侧文件语义”和“沙盒侧运行时挂载语义”，定位问题通常会快很多。
|
||||
|
||||
@ -31,6 +31,7 @@
|
||||
- 调整 Milvus 混合检索实现：集合 schema 增加 BM25 稀疏向量字段、BM25 函数和中文 analyzer 配置
|
||||
- 重构 MCP 运行时配置加载模型：移除 `MCP_SERVERS` 作为运行正确性前提的设计，改为每次直接从数据库读取最新 MCP 配置
|
||||
- 为知识库检索工具补充 `metadata.filepath` 注入：在 `query_kb` 统一出口基于会话可见知识库构建 `file_id -> /home/gem/kbs/...` 映射并回填 Milvus 检索结果
|
||||
- 移除知识库沙盒文件系统映射：Agent 不再通过 `/home/gem/kbs` 遍历知识库文件，继续通过 `query_kb` 和 `open_kb_document` 检索与打开文档。
|
||||
|
||||
## v0.6.0 (2026-04-01)
|
||||
|
||||
|
||||
@ -18,8 +18,7 @@
|
||||
- 拓宽检索的知识源，统一多知识源（channel），目前已知知识库/知识图谱/网页，可拓展：个人知识库、数据库、历史对话等
|
||||
- 前置任务，多知识库并行检索（扩展 query_kb）
|
||||
- 新增 query_keywords 工具，专门用于基于关键词命中的排序，也结合词频（和 BM25 的区别？）
|
||||
- 调研将当前知识库映射为虚拟文件系统工具的可行性，先明确文件树映射、权限边界、内容读取与 Agent 工具调用形态，再决定是否实现
|
||||
- 参考 AgenticRAG 方案扩展当前 Search 工具：等待虚拟文件系统构建完成后，改进 Search 返回递增文件序列 ID，新增 Find 与 Open 能力；Summary 暂缓
|
||||
- 参考 AgenticRAG 方案扩展当前 Search 工具：基于知识库工具返回的 resource_id/file_id 改进 Search 返回递增文件序列 ID，完善 Find 与 Open 能力；Summary 暂缓
|
||||
- 评估，基于 Agent 的评估，这里应该是结合 Langfuse 实现
|
||||
|
||||
### Bugs
|
||||
@ -40,6 +39,7 @@
|
||||
<!-- 0.6.2 的内容请放在这里 -->
|
||||
- 下放扩展管理权限：普通管理员现在可进入扩展管理并完整管理 Tools、MCP、SubAgent、Skills；同步放开 Skill 管理接口权限并补充权限测试。
|
||||
- 调整 Agent 知识库默认选择：未显式配置知识库时默认启用当前用户可访问的全部知识库，显式保存空列表仍表示不启用知识库。
|
||||
- 移除知识库沙盒文件系统映射：不再通过 `/home/gem/kbs` 暴露知识库文件树，Agent 继续使用 `query_kb` 与 `open_kb_document` 访问知识库内容。
|
||||
- 优化评估基准自动生成：仅支持 commonrag/Milvus 知识库，默认参考 chunks 数量改为 1；多 chunk 场景复用知识库向量检索选择相似 chunks，不再对全量 chunks 重新计算 embedding，并移除前端 Embedding 模型选择。
|
||||
- 修复知识库文档入库状态回退：当已解析文件缺失 `markdown_file` 解析产物时，索引流程会将文件状态恢复为未解析，便于重新解析而不是停留在索引失败。
|
||||
- 优化 Agent 输入框文件 mention：用户级 workspace 文件候选改为从独立 workspace API 递归加载，不再依赖 active thread；插入时仍转换为 `/home/gem/user-data/workspace/` 沙盒虚拟路径，并修复附件上传后未立即刷新 mention 候选的问题。
|
||||
|
||||
@ -154,7 +154,7 @@ const props = defineProps({
|
||||
|
||||
const emit = defineEmits(['refresh', 'resize', 'resizing'])
|
||||
const INLINE_PREVIEW_MIN_WIDTH = 920
|
||||
const DEFAULT_EXPANDED_ROOT_DIRECTORY_NAME = 'user-data'
|
||||
const DISPLAY_ROOT_DIRECTORY_NAME = 'user-data'
|
||||
|
||||
const panelRef = ref(null)
|
||||
const modalVisible = ref(false)
|
||||
@ -303,12 +303,6 @@ const loadDirectoryChildren = async (directoryPath) => {
|
||||
return sortEntries(res?.entries || []).map((entry) => createTreeNode(entry))
|
||||
}
|
||||
|
||||
const findDefaultExpandedRootNode = (nodes) => {
|
||||
return nodes.find(
|
||||
(node) => !node.isLeaf && buildDisplayName(node.key) === DEFAULT_EXPANDED_ROOT_DIRECTORY_NAME
|
||||
)
|
||||
}
|
||||
|
||||
const refreshFileSystem = async () => {
|
||||
if (!props.threadId) {
|
||||
dynamicTreeData.value = []
|
||||
@ -327,21 +321,13 @@ const refreshFileSystem = async () => {
|
||||
props.agentConfigId
|
||||
)
|
||||
if (res?.entries) {
|
||||
const rootNodes = sortEntries(res.entries).map((entry) => createTreeNode(entry))
|
||||
const defaultExpandedNode = findDefaultExpandedRootNode(rootNodes)
|
||||
const displayRootEntry = res.entries.find(
|
||||
(entry) => entry?.is_dir && entry.name === DISPLAY_ROOT_DIRECTORY_NAME
|
||||
)
|
||||
|
||||
dynamicTreeData.value = rootNodes
|
||||
expandedKeys.value = defaultExpandedNode ? [defaultExpandedNode.key] : []
|
||||
dynamicTreeData.value = displayRootEntry ? await loadDirectoryChildren(displayRootEntry.path) : []
|
||||
expandedKeys.value = []
|
||||
selectedKeys.value = []
|
||||
|
||||
if (defaultExpandedNode) {
|
||||
try {
|
||||
const children = await loadDirectoryChildren(defaultExpandedNode.key)
|
||||
dynamicTreeData.value = updateTreeChildren(rootNodes, defaultExpandedNode.key, children)
|
||||
} catch (error) {
|
||||
console.error('Failed to load default expanded directory', error)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
dynamicTreeData.value = []
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user