diff --git a/docs/latest/changelog/roadmap.md b/docs/latest/changelog/roadmap.md index 8edc0e5c..0ca0be10 100644 --- a/docs/latest/changelog/roadmap.md +++ b/docs/latest/changelog/roadmap.md @@ -2,7 +2,6 @@ 路线图可能会经常变更，如果有强烈的建议，可以在 [issue](https://github.com/xerrors/Yuxi-Know/issues) 中提。 -v0.5 ### 看板 @@ -15,9 +14,10 @@ v0.5 - skills 如何实现还需要继续调研 - 优化 paddle 的命名，paddlex 有歧义，修改为 PP-StructureV3 - 增加 paddle-vl 以及 deepseek-ocr 的支持（deepseek-ocr 已支持） -- RAG 检索的时候，支持限定文件类型 - 将现有的 Milvus 的命名调整为通用 RAG - 将工具与知识库解耦，在 context 中就完成解耦，虽然最终都是在 Agent 中的 get_tools 中获取 +- 系统层面添加 apikey，在智能体、知识库调用中支持 apikey 以支持外部调用 +- 支持更多类型的文档源的导入功能 ### Bugs - 部分异常状态下，智能体的模型名称出现重叠[#279](https://github.com/xerrors/Yuxi-Know/issues/279) @@ -27,6 +27,20 @@ v0.5 - 目前的知识库的图片存在公开访问风险 - 工具传递给模型的时候，使用英文，但部分模型不支持中文函数名（如gpt-4o-mini） - 首页加载的问题 +- 当前的 upload 图谱查询为同步操作，可能会导致页面卡顿 + +## v0.5 + +### 新增 + +- 优化 OCR 体验并新增对 Deepseek OCR 的支持 +- 优化 RAG 检索，支持根据文件 pattern 来检索（Agentic Mode） +- 重构智能体对于“工具变更/模型变更”的处理逻辑，无需导入更复杂的中间件 +- 重构知识库的 Agentic 配置逻辑，与 Tools 解耦 + +### 修复 + +- 修复知识图谱上传的向量配置错误，并新增模型选择以及 batch size 选择 ## v0.4 diff --git a/server/routers/graph_router.py b/server/routers/graph_router.py index 2cbafb6e..80c3373e 100644 --- a/server/routers/graph_router.py +++ b/server/routers/graph_router.py @@ -251,7 +251,7 @@ async def index_neo4j_entities(data: dict = Body(default={}), current_user: User raise HTTPException(status_code=400, detail="图数据库未启动") kgdb_name = data.get("kgdb_name", "neo4j") - count = graph_base.add_embedding_to_nodes(kgdb_name=kgdb_name) + count = await graph_base.add_embedding_to_nodes(kgdb_name=kgdb_name) return { "success": True, @@ -269,6 +269,7 @@ async def add_neo4j_entities( file_path: str = Body(...), kgdb_name: str | None = Body(None), embed_model_name: str | None = Body(None), + batch_size: int | None = Body(None), current_user: User = Depends(get_admin_user), ): """通过JSONL文件添加图谱实体到Neo4j""" @@ -284,7 +285,7 @@ async def add_neo4j_entities( if not file_path.endswith(".jsonl"): return {"success": False, "message": "文件格式错误，请上传jsonl文件", "status": "failed"} - await graph_base.jsonl_file_add_entity(file_path, kgdb_name, embed_model_name) + await graph_base.jsonl_file_add_entity(file_path, kgdb_name, embed_model_name, batch_size) return {"success": True, "message": "实体添加成功", "status": "success"} except Exception as e: logger.error(f"添加实体失败: {e}, {traceback.format_exc()}") diff --git a/src/knowledge/services/upload_graph_service.py b/src/knowledge/services/upload_graph_service.py index c2abd30b..94b5fc1e 100644 --- a/src/knowledge/services/upload_graph_service.py +++ b/src/knowledge/services/upload_graph_service.py @@ -25,8 +25,8 @@ class UploadGraphService: self.connection = db_manager or Neo4jConnectionManager() self.files = [] self.kgdb_name = "neo4j" - self.embed_model_name = os.getenv("GRAPH_EMBED_MODEL_NAME") or "siliconflow/BAAI/bge-m3" - self.embed_model = select_embedding_model(self.embed_model_name) + self.embed_model_name = None # self.load_graph_info() 时加载 + self.embed_model = None # self.load_graph_info() 时加载 self.work_dir = os.path.join(config.save_dir, "knowledge_graph", self.kgdb_name) os.makedirs(self.work_dir, exist_ok=True) self.is_initialized_from_file = False @@ -51,7 +51,6 @@ class UploadGraphService: if not self.connection.is_running(): self.connection._connect() logger.info(f"Connected to Neo4j: {self.get_graph_info(self.kgdb_name)}") - # 连接成功后保存图数据库信息 def close(self): """关闭数据库连接""" @@ -84,7 +83,7 @@ class UploadGraphService: if self.status == "closed": self.start() - async def jsonl_file_add_entity(self, file_path, kgdb_name="neo4j", embed_model_name=None): + async def jsonl_file_add_entity(self, file_path, kgdb_name="neo4j", embed_model_name=None, batch_size=None): """从JSONL文件添加实体三元组到Neo4j""" assert self.driver is not None, "Database is not connected" self.connection.status = "processing" @@ -134,7 +133,7 @@ class UploadGraphService: triples = list(read_triples(actual_file_path)) - await self.txt_add_vector_entity(triples, kgdb_name, embed_model_name) + await self.txt_add_vector_entity(triples, kgdb_name, embed_model_name, batch_size) except Exception as e: logger.error(f"处理文件失败: {e}") @@ -153,7 +152,7 @@ class UploadGraphService: self.save_graph_info() return kgdb_name - async def txt_add_vector_entity(self, triples, kgdb_name="neo4j", embed_model_name=None): + async def txt_add_vector_entity(self, triples, kgdb_name="neo4j", embed_model_name=None, batch_size=None): """添加实体三元组""" assert self.driver is not None, "Database is not connected" self.use_database(kgdb_name) @@ -274,9 +273,7 @@ class UploadGraphService: # 允许 self.embed_model_name 与 config.embed_model 不同（用户自定义选择的情况） # 但必须在支持的模型列表中 - assert self.embed_model_name in config.embed_model_names, ( - f"Unsupported embed model: {self.embed_model_name}" - ) + assert self.embed_model_name in config.embed_model_names, f"Unsupported embed model: {self.embed_model_name}" with self.driver.session() as session: logger.info(f"Adding entity to {kgdb_name}") @@ -316,7 +313,7 @@ class UploadGraphService: ) # 批量获取嵌入向量 - batch_embeddings = await self.aget_embedding(batch_entities) + batch_embeddings = await self.aget_embedding(batch_entities, batch_size=batch_size) # 将实体名称和嵌入向量配对 entity_embedding_pairs = list(zip(batch_entities, batch_embeddings)) @@ -327,12 +324,13 @@ class UploadGraphService: # 数据添加完成后保存图信息 self.save_graph_info() - def add_embedding_to_nodes(self, node_names=None, kgdb_name="neo4j"): + async def add_embedding_to_nodes(self, node_names=None, kgdb_name="neo4j", batch_size=None): """为节点添加嵌入向量 Args: node_names (list, optional): 要添加嵌入向量的节点名称列表，None表示所有没有嵌入向量的节点 kgdb_name (str, optional): 图数据库名称，默认为'neo4j' + batch_size (int, optional): 嵌入批次大小 Returns: int: 成功添加嵌入向量的节点数量 @@ -348,7 +346,7 @@ class UploadGraphService: with self.driver.session() as session: for node_name in node_names: try: - embedding = self.get_embedding(node_name) + embedding = await self.aget_embedding(node_name, batch_size=batch_size) session.execute_write(self.set_embedding, node_name, embedding) count += 1 except Exception as e: @@ -500,23 +498,24 @@ class UploadGraphService: logger.error(f"加载图数据库信息失败：{e}") return False - async def aget_embedding(self, text): + async def aget_embedding(self, text, batch_size=40): if isinstance(text, list): - outputs = await self.embed_model.abatch_encode(text, batch_size=40) + outputs = await self.embed_model.abatch_encode(text, batch_size=batch_size) return outputs else: outputs = await self.embed_model.aencode(text) return outputs - def get_embedding(self, text): + def get_embedding(self, text, batch_size=40): if isinstance(text, list): - outputs = self.embed_model.batch_encode(text, batch_size=40) + outputs = self.embed_model.batch_encode(text, batch_size=batch_size) return outputs else: outputs = self.embed_model.encode([text])[0] return outputs def set_embedding(self, tx, entity_name, embedding): + """为单个实体设置嵌入向量""" tx.run( """ MATCH (e:Entity {name: $name}) @@ -564,9 +563,8 @@ class UploadGraphService: entity_to_score[name] = max(entity_to_score.get(name, 0.0), 0.3) # 排序并截断 - qualified_entities = [name for name, _ in sorted(entity_to_score.items(), key=lambda x: x[1], reverse=True)][ - :max_entities - ] + sorted_entity_to_score = sorted(entity_to_score.items(), key=lambda x: x[1], reverse=True) + qualified_entities = [name for name, _ in sorted_entity_to_score][:max_entities] logger.debug(f"Graph Query Entities: {keyword}, {qualified_entities=}") diff --git a/web/src/apis/graph_api.js b/web/src/apis/graph_api.js index 6c7d53d9..643c9331 100644 --- a/web/src/apis/graph_api.js +++ b/web/src/apis/graph_api.js @@ -123,13 +123,15 @@ export const neo4jApi = { * @param {string} file_path - JSONL文件路径 * @param {string} kgdb_name - Neo4j数据库名称（默认为'neo4j'） * @param {string} embed_model_name - 嵌入模型名称 (可选) + * @param {number} batch_size - 批处理大小 (可选) * @returns {Promise} - 添加结果 */ - addEntities: async (file_path, kgdb_name = 'neo4j', embed_model_name = null) => { + addEntities: async (file_path, kgdb_name = 'neo4j', embed_model_name = null, batch_size = null) => { return await apiPost('/api/graph/neo4j/add-entities', { file_path: file_path, kgdb_name: kgdb_name, - embed_model_name: embed_model_name + embed_model_name: embed_model_name, + batch_size: batch_size }, {}, true) }, diff --git a/web/src/layouts/AppLayout.vue b/web/src/layouts/AppLayout.vue index a5c7b324..e5d60cfc 100644 --- a/web/src/layouts/AppLayout.vue +++ b/web/src/layouts/AppLayout.vue @@ -327,8 +327,8 @@ div.header, #app-router-view { display: flex; align-items: center; justify-content: center; - width: 40px; - height: 40px; + width: 36px; + height: 36px; padding: 4px; border: 1px solid transparent; border-radius: 12px; diff --git a/web/src/views/GraphView.vue b/web/src/views/GraphView.vue index 209bcd8c..35ae4e5e 100644 --- a/web/src/views/GraphView.vue +++ b/web/src/views/GraphView.vue @@ -115,16 +115,34 @@

上传的文件内容参考 test/data/A_Dream_of_Red_Mansions_tiny.jsonl 中的格式：

-
- 嵌入模型: - -
+
+
+ +
+ +
+
+
* 图数据库已有数据或已设定模型，不可更改
+
+ +
+ +
+
+
+ 默认值: 40，范围: 1-1000 +
{ @@ -343,7 +362,7 @@ const addDocumentByFile = () => { return } - neo4jApi.addEntities(filePath, 'neo4j', state.embedModelName) + neo4jApi.addEntities(filePath, 'neo4j', state.embedModelName, state.batchSize) .then((data) => { if (data.status === 'success') { message.success(data.message); @@ -640,6 +659,49 @@ const goToDatabasePage = () => { .upload-dragger { margin: 0px; } + + .upload-config { + margin: 24px 0; + padding: 16px; + background-color: var(--gray-0); + border-radius: 4px; + + .config-row { + display: flex; + align-items: center; + margin-bottom: 16px; + + &:last-of-type { + margin-bottom: 0; + } + + .config-label { + width: 100px; + flex-shrink: 0; + font-size: 14px; + color: var(--color-text); + text-align: right; + margin-right: 16px; + } + + .config-field { + flex: 1; + min-width: 0; + } + } + + .config-hint-row { + margin-bottom: 16px; + padding-left: 116px; + font-size: 12px; + color: var(--color-text-secondary); + line-height: 1.5; + + &:last-child { + margin-bottom: 0; + } + } + } } .container-outter {