diff --git a/docs/latest/changelog/roadmap.md b/docs/latest/changelog/roadmap.md index 40251712..07b4aac9 100644 --- a/docs/latest/changelog/roadmap.md +++ b/docs/latest/changelog/roadmap.md @@ -15,12 +15,12 @@ - 同名文件处理逻辑：遇到同名文件则在上传区域提示，是否删除旧文件 - conversation 待修改为异步的版本 - DBManager 需要将数据库修改为异步的aiosqlite或者异步mysql，缓存使用Redis存储 -- agent 状态中的文件区域，新增可以下载 ### Bugs - 部分异常状态下，智能体的模型名称出现重叠[#279](https://github.com/xerrors/Yuxi-Know/issues/279) - DeepSeek 官方接口适配会出现问题 - 目前的知识库的图片存在公开访问风险 +- 深度分析智能体需要考虑上下文超限的问题 ### 新增 - 优化知识库详情页面，更加简洁清晰 @@ -34,6 +34,8 @@ - 新增自定义模型支持、新增 dashscope rerank/embeddings 模型的支持 - 新增文档解析的图片支持，已支持 MinerU Officical、Docs、Markdown Zip格式 - 新增暗色模式支持并调整整体 UI（[#343](https://github.com/xerrors/Yuxi-Know/pull/343)） +- agent 状态中的文件区域，新增可以下载 +- 移除 Chroma 的支持，当前版本标记为移除 ### 修复 - 修复重排序模型实际未生效的问题 diff --git a/server/routers/knowledge_router.py b/server/routers/knowledge_router.py index cd6a00f2..e9655fd3 100644 --- a/server/routers/knowledge_router.py +++ b/server/routers/knowledge_router.py @@ -87,10 +87,12 @@ async def create_database( """创建知识库""" logger.debug( f"Create database {database_name} with kb_type {kb_type}, " - f"additional_params {additional_params}, llm_info {llm_info}" + f"additional_params {additional_params}, llm_info {llm_info}, " + f"embed_model_name {embed_model_name}" ) try: additional_params = {**(additional_params or {})} + additional_params["auto_generate_questions"] = False # 默认不生成问题 def normalize_reranker_config(kb: str, params: dict) -> None: reranker_cfg = params.get("reranker_config") @@ -112,12 +114,12 @@ async def create_database( if not isinstance(reranker_cfg, Mapping): raise HTTPException(status_code=400, detail="reranker_config must be an object") - enabled = bool(reranker_cfg.get("enabled", False)) + reranker_enabled = bool(reranker_cfg.get("enabled", False)) model = (reranker_cfg.get("model") or "").strip() recall_top_k = max(1, int(reranker_cfg.get("recall_top_k", 50))) final_top_k = max(1, int(reranker_cfg.get("final_top_k", 10))) - if enabled: + if reranker_enabled: if not model: raise HTTPException(status_code=400, detail="reranker_config.model is required when enabled") if model not in config.reranker_names: @@ -132,7 +134,7 @@ async def create_database( model = model if model in config.reranker_names else "" params["reranker_config"] = { - "enabled": enabled, + "enabled": reranker_enabled, "model": model, "recall_top_k": recall_top_k, "final_top_k": final_top_k, diff --git a/src/agents/common/mcp_repos/arxiv-mcp-server b/src/agents/common/mcp_repos/arxiv-mcp-server new file mode 160000 index 00000000..057e2000 --- /dev/null +++ b/src/agents/common/mcp_repos/arxiv-mcp-server @@ -0,0 +1 @@ +Subproject commit 057e2000be7b56823239815b0fe7c7fc0dbced96 diff --git a/src/agents/common/mcp_repos/mcp-server-mysql b/src/agents/common/mcp_repos/mcp-server-mysql new file mode 160000 index 00000000..6a036783 --- /dev/null +++ b/src/agents/common/mcp_repos/mcp-server-mysql @@ -0,0 +1 @@ +Subproject commit 6a0367834ea0fb5e5c94b9711e3e2756966789ea diff --git a/src/config/static/models.py b/src/config/static/models.py index 897ea980..53b5fcbf 100644 --- a/src/config/static/models.py +++ b/src/config/static/models.py @@ -29,7 +29,7 @@ class EmbedModelInfo(BaseModel): dimension: int = Field(..., description="向量维度") base_url: str = Field(..., description="API 基础 URL") api_key: str = Field(..., description="API Key 或环境变量名") - + model_id: str | None = Field(None, description="可选的模型 ID") class RerankerInfo(BaseModel): """重排序模型配置""" @@ -158,42 +158,49 @@ DEFAULT_CHAT_MODEL_PROVIDERS: dict[str, ChatModelProvider] = { DEFAULT_EMBED_MODELS: dict[str, EmbedModelInfo] = { "siliconflow/BAAI/bge-m3": EmbedModelInfo( + model_id="siliconflow/BAAI/bge-m3", name="BAAI/bge-m3", dimension=1024, base_url="https://api.siliconflow.cn/v1/embeddings", api_key="SILICONFLOW_API_KEY", ), "siliconflow/Pro/BAAI/bge-m3": EmbedModelInfo( + model_id="siliconflow/Pro/BAAI/bge-m3", name="Pro/BAAI/bge-m3", dimension=1024, base_url="https://api.siliconflow.cn/v1/embeddings", api_key="SILICONFLOW_API_KEY", ), "siliconflow/Qwen/Qwen3-Embedding-0.6B": EmbedModelInfo( + model_id="siliconflow/Qwen/Qwen3-Embedding-0.6B", name="Qwen/Qwen3-Embedding-0.6B", dimension=1024, base_url="https://api.siliconflow.cn/v1/embeddings", api_key="SILICONFLOW_API_KEY", ), "vllm/Qwen/Qwen3-Embedding-0.6B": EmbedModelInfo( + model_id="vllm/Qwen/Qwen3-Embedding-0.6B", name="Qwen3-Embedding-0.6B", dimension=1024, base_url="http://localhost:8000/v1/embeddings", api_key="no_api_key", ), "ollama/nomic-embed-text": EmbedModelInfo( + model_id="ollama/nomic-embed-text", name="nomic-embed-text", dimension=768, base_url="http://localhost:11434/api/embed", api_key="no_api_key", ), "ollama/bge-m3": EmbedModelInfo( + model_id="ollama/bge-m3", name="bge-m3", dimension=1024, base_url="http://localhost:11434/api/embed", api_key="no_api_key", ), "dashscope/text-embedding-v4": EmbedModelInfo( + model_id="dashscope/text-embedding-v4", name="text-embedding-v4", dimension=1024, base_url="https://dashscope.aliyuncs.com/compatible-mode/v1/embeddings", diff --git a/src/knowledge/implementations/milvus.py b/src/knowledge/implementations/milvus.py index b914b53c..0b22d674 100644 --- a/src/knowledge/implementations/milvus.py +++ b/src/knowledge/implementations/milvus.py @@ -7,6 +7,7 @@ from typing import Any from pymilvus import Collection, CollectionSchema, DataType, FieldSchema, connections, db, utility +from src import config from src.knowledge.base import KnowledgeBase from src.knowledge.indexing import process_file_to_markdown from src.knowledge.utils.kb_utils import ( @@ -91,10 +92,14 @@ class MilvusKB(KnowledgeBase): """创建 Milvus 集合""" logger.info(f"Creating Milvus collection for {db_id}") - if db_id not in self.databases_meta: + if not (metadata := self.databases_meta.get(db_id)): raise ValueError(f"Database {db_id} not found") - embed_info = self.databases_meta[db_id].get("embed_info", {}) + # embed_info = metadata.get("embed_info", {}) + if not (embed_info := metadata.get("embed_info")): + logger.error(f"Embedding info not found for database {db_id}, using default model") + embed_info = config.embed_model_names[config.embed_model] + collection_name = db_id try: @@ -117,8 +122,8 @@ class MilvusKB(KnowledgeBase): except Exception: # 创建新集合 - embedding_dim = getattr(embed_info, "dimension", 1024) if embed_info else 1024 - model_name = getattr(embed_info, "name", "default") if embed_info else "default" + embedding_dim = embed_info.get("dimension", 1024) + model_name = embed_info.get("name", "default") # 定义集合Schema fields = [ @@ -142,7 +147,7 @@ class MilvusKB(KnowledgeBase): index_params = {"metric_type": "COSINE", "index_type": "IVF_FLAT", "params": {"nlist": 1024}} collection.create_index("embedding", index_params) - logger.info(f"Created new Milvus collection: {collection_name}") + logger.info(f"Created new Milvus collection: {collection_name}: {model_name=}, {embedding_dim=}") return collection @@ -154,25 +159,29 @@ class MilvusKB(KnowledgeBase): except Exception as e: logger.warning(f"Failed to load collection into memory: {e}") - def _get_async_embedding_function(self, embed_info: dict): + def _get_async_embedding(self, embed_info: dict): """获取 embedding 函数""" + # 检查是否有 model_id 字段，优先使用 select_embedding_model + if embed_info and "model_id" in embed_info: + from src.models.embed import select_embedding_model + return select_embedding_model(embed_info["model_id"]) + + # 使用原有的逻辑（兼容模式）） config_dict = get_embedding_config(embed_info) - embedding_model = OtherEmbedding( + return OtherEmbedding( model=config_dict.get("model"), base_url=config_dict.get("base_url"), api_key=config_dict.get("api_key"), ) + def _get_async_embedding_function(self, embed_info: dict): + """获取 embedding 函数""" + embedding_model = self._get_async_embedding(embed_info) return partial(embedding_model.abatch_encode, batch_size=40) def _get_embedding_function(self, embed_info: dict): """获取 embedding 函数""" - config_dict = get_embedding_config(embed_info) - embedding_model = OtherEmbedding( - model=config_dict.get("model"), - base_url=config_dict.get("base_url"), - api_key=config_dict.get("api_key"), - ) + embedding_model = self._get_async_embedding(embed_info) return partial(embedding_model.batch_encode, batch_size=40) diff --git a/src/knowledge/manager.py b/src/knowledge/manager.py index 3e912c3f..e52d094f 100644 --- a/src/knowledge/manager.py +++ b/src/knowledge/manager.py @@ -246,16 +246,12 @@ class KnowledgeBaseManager: db_id = db_info["db_id"] async with self._metadata_lock: - # 准备 additional_params，包含 auto_generate_questions - saved_params = kwargs.copy() - saved_params["auto_generate_questions"] = False - self.global_databases_meta[db_id] = { "name": database_name, "description": description, "kb_type": kb_type, "created_at": utc_isoformat(), - "additional_params": saved_params, + "additional_params": kwargs.copy(), } self._save_global_metadata() diff --git a/src/knowledge/utils/kb_utils.py b/src/knowledge/utils/kb_utils.py index 0b06459f..c8d9abcb 100644 --- a/src/knowledge/utils/kb_utils.py +++ b/src/knowledge/utils/kb_utils.py @@ -247,15 +247,23 @@ def get_embedding_config(embed_info: dict) -> dict: try: if embed_info: - # 处理 embed_info 可能是字典或 EmbedModelInfo 对象的情况 - if hasattr(embed_info, "name"): + # 优先检查是否有 model_id 字段 + if "model_id" in embed_info: + from src.models.embed import select_embedding_model + + model = select_embedding_model(embed_info["model_id"]) + config_dict["model"] = model.model + config_dict["api_key"] = model.api_key + config_dict["base_url"] = model.base_url + config_dict["dimension"] = getattr(model, "dimension", 1024) + elif hasattr(embed_info, "name"): # EmbedModelInfo 对象 config_dict["model"] = embed_info.name config_dict["api_key"] = os.getenv(embed_info.api_key) or embed_info.api_key config_dict["base_url"] = embed_info.base_url config_dict["dimension"] = embed_info.dimension else: - # 字典形式 + # 字典形式（保持向后兼容） config_dict["model"] = embed_info["name"] config_dict["api_key"] = os.getenv(embed_info["api_key"]) or embed_info["api_key"] config_dict["base_url"] = embed_info["base_url"] diff --git a/src/models/embed.py b/src/models/embed.py index ebf0ece9..a63bbc58 100644 --- a/src/models/embed.py +++ b/src/models/embed.py @@ -11,7 +11,7 @@ from src.utils import get_docker_safe_url, hashstr, logger class BaseEmbeddingModel(ABC): - def __init__(self, model=None, name=None, dimension=None, url=None, base_url=None, api_key=None): + def __init__(self, model=None, name=None, dimension=None, url=None, base_url=None, api_key=None, model_id=None): """ Args: model: 模型名称，冗余设计，同name @@ -140,6 +140,7 @@ class OllamaEmbedding(BaseEmbeddingModel): payload = {"model": self.model, "input": message} async with httpx.AsyncClient() as client: try: + print(f"\n\n\nOllama Embedding request: {payload}\n\n\n") response = await client.post(self.base_url, json=payload, timeout=60) response.raise_for_status() result = response.json() diff --git a/web/src/views/DataBaseView.vue b/web/src/views/DataBaseView.vue index 88453597..0c38fd6a 100644 --- a/web/src/views/DataBaseView.vue +++ b/web/src/views/DataBaseView.vue @@ -14,15 +14,23 @@

知识库类型*

{{ getKbTypeLabel(typeKey) }} + + 弃用 +
{{ typeInfo.description }}
@@ -71,7 +79,7 @@

隐私设置

@@ -158,6 +166,18 @@

正在加载知识库...

+ +
+

暂无知识库

+

创建您的第一个知识库，开始管理文档和知识

+ + + 创建知识库 + +
+
({ name: '', description: '', embed_model_name: configStore.config?.embed_model, - kb_type: 'chroma', + kb_type: 'milvus', is_private: false, storage: '', language: 'English', @@ -295,6 +315,27 @@ const llmModelSpec = computed(() => { // 支持的知识库类型 const supportedKbTypes = ref({}) +// 有序的知识库类型（Chroma 排在最后） +const orderedKbTypes = computed(() => { + const types = { ...supportedKbTypes.value } + const ordered = {} + const chromaData = types.chroma + + // 先添加除了 Chroma 之外的所有类型 + Object.keys(types).forEach(key => { + if (key !== 'chroma') { + ordered[key] = types[key] + } + }) + + // 最后添加 Chroma（如果存在） + if (chromaData) { + ordered.chroma = chromaData + } + + return ordered +}) + // 加载支持的知识库类型 const loadSupportedKbTypes = async () => { try { @@ -368,24 +409,6 @@ const getKbTypeIcon = (type) => { return icons[type] || Database } -// const getKbTypeDescription = (type) => { -// const descriptions = { -// lightrag: '🔥 图结构索引 • 智能查询 • 关系挖掘 • 复杂推理', -// chroma: '⚡ 轻量向量 • 快速开发 • 本地部署 • 简单易用', -// milvus: '🚀 生产级 • 高性能 • 分布式 • 企业级部署' -// } -// return descriptions[type] || '' -// } - -const getKbTypeAlertType = (type) => { - const types = { - lightrag: 'info', - chroma: 'success', - milvus: 'warning' - } - return types[type] || 'info' -} - const getKbTypeColor = (type) => { const colors = { lightrag: 'purple', @@ -395,7 +418,6 @@ const getKbTypeColor = (type) => { return colors[type] || 'blue' } - // 格式化创建时间 const formatCreatedTime = (createdAt) => { if (!createdAt) return '' @@ -680,24 +702,12 @@ onMounted(() => { border-color: var(--main-color); } - // 为不同知识库类型设置不同的悬停颜色与主题色 - &:nth-child(1):hover, - &:nth-child(1).active { - border-color: var(--color-accent-100); - .type-icon { color: var(--color-accent-500); } + &.active { + border-color: var(--main-color); + background: var(--main-10); + .type-icon { color: var(--main-color); } } - &:nth-child(2):hover, - &:nth-child(2).active { - border-color: var(--color-warning-100); - .type-icon { color: var(--color-warning-500); } - } - - &:nth-child(3):hover, - &:nth-child(3).active { - border-color: var(--color-error-100); - .type-icon { color: var(--color-error-500); } - } .card-header { display: flex; align-items: center; @@ -725,6 +735,26 @@ onMounted(() => { margin-bottom: 0; // min-height: 40px; } + + .deprecated-badge { + background: var(--color-error-100); + color: var(--color-error-600); + font-size: 10px; + font-weight: 600; + padding: 2px 6px; + border-radius: 4px; + margin-left: auto; + text-transform: uppercase; + letter-spacing: 0.5px; + cursor: help; + transition: all 0.2s ease; + + &:hover { + background: var(--color-error-200); + color: var(--color-error-700); + } + } + } } @@ -900,8 +930,6 @@ onMounted(() => { font-weight: 400; flex: 1; } - - } .database-empty { @@ -913,6 +941,38 @@ onMounted(() => { color: var(--gray-900); } +.empty-state { + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + padding: 100px 20px; + text-align: center; + + .empty-title { + font-size: 20px; + font-weight: 600; + color: var(--gray-900); + margin: 0 0 12px 0; + letter-spacing: -0.02em; + } + + .empty-description { + font-size: 14px; + color: var(--gray-600); + margin: 0 0 32px 0; + line-height: 1.5; + max-width: 320px; + } + + .ant-btn { + height: 44px; + padding: 0 24px; + font-size: 15px; + font-weight: 500; + } +} + .database-container { padding: 0; }