fix: 修复 lightrag 序列化与 knowledge router 路径问题 Fixes: #584
- 修正 lightrag.py 中的序列化逻辑 - 修复 knowledge_router.py 路由路径 - 更新 roadmap.md 文档记录
This commit is contained in:
parent
4f6353d555
commit
d1a02579bc
@ -1,3 +1,4 @@
|
|||||||
|
import asyncio
|
||||||
import os
|
import os
|
||||||
import traceback
|
import traceback
|
||||||
from functools import partial
|
from functools import partial
|
||||||
@ -34,6 +35,9 @@ class LightRagKB(KnowledgeBase):
|
|||||||
|
|
||||||
# 存储 LightRAG 实例映射 {db_id: LightRAG}
|
# 存储 LightRAG 实例映射 {db_id: LightRAG}
|
||||||
self.instances: dict[str, LightRAG] = {}
|
self.instances: dict[str, LightRAG] = {}
|
||||||
|
self._db_write_locks: dict[str, asyncio.Lock] = {}
|
||||||
|
self._db_instance_locks: dict[str, asyncio.Lock] = {}
|
||||||
|
self._lock_guard = asyncio.Lock()
|
||||||
|
|
||||||
logger.info("LightRagKB initialized")
|
logger.info("LightRagKB initialized")
|
||||||
|
|
||||||
@ -197,6 +201,12 @@ class LightRagKB(KnowledgeBase):
|
|||||||
if db_id not in self.databases_meta:
|
if db_id not in self.databases_meta:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
instance_lock = await self._get_db_instance_lock(db_id)
|
||||||
|
async with instance_lock:
|
||||||
|
if db_id in self.instances:
|
||||||
|
logger.info(f"Using cached LightRAG instance for {db_id}")
|
||||||
|
return self.instances[db_id]
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# 创建实例
|
# 创建实例
|
||||||
rag = await self._create_kb_instance(db_id, {})
|
rag = await self._create_kb_instance(db_id, {})
|
||||||
@ -212,6 +222,14 @@ class LightRagKB(KnowledgeBase):
|
|||||||
logger.error(f"Traceback: {traceback.format_exc()}")
|
logger.error(f"Traceback: {traceback.format_exc()}")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
async def _get_db_write_lock(self, db_id: str) -> asyncio.Lock:
|
||||||
|
async with self._lock_guard:
|
||||||
|
return self._db_write_locks.setdefault(db_id, asyncio.Lock())
|
||||||
|
|
||||||
|
async def _get_db_instance_lock(self, db_id: str) -> asyncio.Lock:
|
||||||
|
async with self._lock_guard:
|
||||||
|
return self._db_instance_locks.setdefault(db_id, asyncio.Lock())
|
||||||
|
|
||||||
def _get_llm_func(self, llm_info: dict):
|
def _get_llm_func(self, llm_info: dict):
|
||||||
"""获取 LLM 函数"""
|
"""获取 LLM 函数"""
|
||||||
from yuxi.models import select_model
|
from yuxi.models import select_model
|
||||||
@ -299,6 +317,8 @@ class LightRagKB(KnowledgeBase):
|
|||||||
if db_id not in self.databases_meta:
|
if db_id not in self.databases_meta:
|
||||||
raise ValueError(f"Database {db_id} not found")
|
raise ValueError(f"Database {db_id} not found")
|
||||||
|
|
||||||
|
db_write_lock = await self._get_db_write_lock(db_id)
|
||||||
|
async with db_write_lock:
|
||||||
rag = await self._get_lightrag_instance(db_id)
|
rag = await self._get_lightrag_instance(db_id)
|
||||||
if not rag:
|
if not rag:
|
||||||
raise ValueError(f"Failed to get LightRAG instance for {db_id}")
|
raise ValueError(f"Failed to get LightRAG instance for {db_id}")
|
||||||
@ -354,7 +374,9 @@ class LightRagKB(KnowledgeBase):
|
|||||||
await self._save_metadata()
|
await self._save_metadata()
|
||||||
|
|
||||||
chunks = chunk_markdown(markdown_content, file_id, filename, processing_params)
|
chunks = chunk_markdown(markdown_content, file_id, filename, processing_params)
|
||||||
chunk_input, split_by_character, split_by_character_only = self._prepare_lightrag_insert_payload(chunks)
|
chunk_input, split_by_character, split_by_character_only = self._prepare_lightrag_insert_payload(
|
||||||
|
chunks
|
||||||
|
)
|
||||||
if not chunk_input:
|
if not chunk_input:
|
||||||
chunk_input = markdown_content
|
chunk_input = markdown_content
|
||||||
|
|
||||||
@ -404,6 +426,8 @@ class LightRagKB(KnowledgeBase):
|
|||||||
if db_id not in self.databases_meta:
|
if db_id not in self.databases_meta:
|
||||||
raise ValueError(f"Database {db_id} not found")
|
raise ValueError(f"Database {db_id} not found")
|
||||||
|
|
||||||
|
db_write_lock = await self._get_db_write_lock(db_id)
|
||||||
|
async with db_write_lock:
|
||||||
rag = await self._get_lightrag_instance(db_id)
|
rag = await self._get_lightrag_instance(db_id)
|
||||||
if not rag:
|
if not rag:
|
||||||
raise ValueError(f"Failed to get LightRAG instance for {db_id}")
|
raise ValueError(f"Failed to get LightRAG instance for {db_id}")
|
||||||
@ -448,7 +472,9 @@ class LightRagKB(KnowledgeBase):
|
|||||||
logger.info(f"Markdown content: {markdown_content_lines}...")
|
logger.info(f"Markdown content: {markdown_content_lines}...")
|
||||||
filename = file_meta.get("filename") or file_id
|
filename = file_meta.get("filename") or file_id
|
||||||
chunks = chunk_markdown(markdown_content, file_id, filename, resolved_params)
|
chunks = chunk_markdown(markdown_content, file_id, filename, resolved_params)
|
||||||
chunk_input, split_by_character, split_by_character_only = self._prepare_lightrag_insert_payload(chunks)
|
chunk_input, split_by_character, split_by_character_only = self._prepare_lightrag_insert_payload(
|
||||||
|
chunks
|
||||||
|
)
|
||||||
if not chunk_input:
|
if not chunk_input:
|
||||||
chunk_input = markdown_content
|
chunk_input = markdown_content
|
||||||
|
|
||||||
|
|||||||
@ -392,6 +392,7 @@ async def add_documents(
|
|||||||
try:
|
try:
|
||||||
# 2. Parse file (PARSING -> PARSED)
|
# 2. Parse file (PARSING -> PARSED)
|
||||||
file_meta = await knowledge_base.parse_file(db_id, file_id, operator_id=current_user.user_id)
|
file_meta = await knowledge_base.parse_file(db_id, file_id, operator_id=current_user.user_id)
|
||||||
|
added_files[item] = (file_id, file_meta)
|
||||||
processed_items.append(file_meta)
|
processed_items.append(file_meta)
|
||||||
parse_success_count += 1
|
parse_success_count += 1
|
||||||
except Exception as parse_error:
|
except Exception as parse_error:
|
||||||
|
|||||||
@ -65,6 +65,7 @@
|
|||||||
- 修复前端工具图标与渲染匹配不准确的问题:工具管理列表与工具调用结果统一改为基于工具 `id` 的精确映射,避免模糊匹配导致的误渲染,未命中的工具不再显示默认扳手图标
|
- 修复前端工具图标与渲染匹配不准确的问题:工具管理列表与工具调用结果统一改为基于工具 `id` 的精确映射,避免模糊匹配导致的误渲染,未命中的工具不再显示默认扳手图标
|
||||||
- 修复 GitHub Pages 文档部署工作流失败:移除 `actions/setup-node@v4` 对不存在 `docs/package-lock.json` 的缓存依赖,并将 `docs` 目录安装命令从 `npm ci` 调整为 `npm install`,避免因未提交锁文件导致 CI 在依赖缓存和安装阶段直接失败
|
- 修复 GitHub Pages 文档部署工作流失败:移除 `actions/setup-node@v4` 对不存在 `docs/package-lock.json` 的缓存依赖,并将 `docs` 目录安装命令从 `npm ci` 调整为 `npm install`,避免因未提交锁文件导致 CI 在依赖缓存和安装阶段直接失败
|
||||||
- 修正沙盒 provisioner backend 命名与配置说明:统一对外使用 `docker` / `kubernetes`,保留 `local` 作为兼容别名;同步清理 compose 中未生效的 provisioner 环境变量、补齐 K8s 相关变量注释,并更新沙盒架构文档中的默认模式与 backend 描述
|
- 修正沙盒 provisioner backend 命名与配置说明:统一对外使用 `docker` / `kubernetes`,保留 `local` 作为兼容别名;同步清理 compose 中未生效的 provisioner 环境变量、补齐 K8s 相关变量注释,并更新沙盒架构文档中的默认模式与 backend 描述
|
||||||
|
- 修复 LightRAG 同库写入并发导致的入库失败:为 `index_file` / `update_content` 增加按知识库维度的串行锁,并补齐 `documents` 接口 `auto_index` 阶段对最新解析状态的回写与回归测试,避免长时间入库任务进行中再次选择同库文件时直接并发写入报错
|
||||||
|
|
||||||
<!-- 添加到这里 -->
|
<!-- 添加到这里 -->
|
||||||
|
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user