kris/ForcePilot
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

refactor(metadata): 将_save_metadata()替换为_persist_file()以优化文件持久化逻辑 #513

Browse Source
This commit is contained in:
Wenjie Zhang 2026-02-25 15:53:38 +08:00
parent 74cb569569
commit 52f9ed0eb9
5 changed files with 102 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
src/knowledge/base.py
View File

@ -208,7 +208,7 @@ class KnowledgeBase(ABC):
# Save to metadata
self.files_meta[file_id] = metadata
await self._save_metadata()
await self._persist_file(file_id)
return metadata
@ -256,7 +256,7 @@ class KnowledgeBase(ABC):
self.files_meta[file_id]["updated_at"] = utc_isoformat()
if operator_id:
self.files_meta[file_id]["updated_by"] = operator_id
await self._save_metadata()
await self._persist_file(file_id)
# Add to processing queue
self._add_to_processing_queue(file_id)
@ -293,7 +293,7 @@ class KnowledgeBase(ABC):
self.files_meta[file_id]["updated_at"] = utc_isoformat()
if operator_id:
self.files_meta[file_id]["updated_by"] = operator_id
await self._save_metadata()
await self._persist_file(file_id)
return self.files_meta[file_id]
@ -306,7 +306,7 @@ class KnowledgeBase(ABC):
self.files_meta[file_id]["updated_at"] = utc_isoformat()
if operator_id:
self.files_meta[file_id]["updated_by"] = operator_id
await self._save_metadata()
await self._persist_file(file_id)
raise
@ -338,7 +338,7 @@ class KnowledgeBase(ABC):
logger.debug(f"[update_file_params] file_id={file_id}, updated_params={current_params}")
await self._save_metadata()
await self._persist_file(file_id)
async def _save_markdown_to_minio(self, db_id: str, file_id: str, content: str) -> str:
"""Save markdown content to MinIO and return HTTP URL"""
@ -431,7 +431,7 @@ class KnowledgeBase(ABC):
"metadata": kwargs,
"created_at": utc_isoformat(),
}
await self._save_metadata()
await self._persist_kb(db_id)
# 创建工作目录
working_dir = os.path.join(self.work_dir, db_id)
@ -496,7 +496,7 @@ class KnowledgeBase(ABC):
"path": folder_name,
"file_type": "folder",
}
await self._save_metadata()
await self._persist_file(folder_id)
return self.files_meta[folder_id]
@abstractmethod
@ -838,7 +838,7 @@ class KnowledgeBase(ABC):
current = parent_meta.get("parent_id")
meta["parent_id"] = new_parent_id
await self._save_metadata()
await self._persist_file(file_id)
return meta
@abstractmethod
@ -1120,3 +1120,78 @@ class KnowledgeBase(ABC):
}
if existing is None:
await eval_repo.create_benchmark(payload)
async def _persist_file(self, file_id: str) -> None:
"""只保存单个文件到数据库，避免全量遍历"""
from src.repositories.knowledge_file_repository import KnowledgeFileRepository
file_repo = KnowledgeFileRepository()
if file_id not in self.files_meta:
return
meta = self.files_meta[file_id]
db_id = meta.get("database_id")
if not db_id:
return
await file_repo.upsert(
file_id=file_id,
data={
"db_id": db_id,
"parent_id": meta.get("parent_id"),
"filename": meta.get("filename") or "",
"original_filename": meta.get("original_filename"),
"file_type": meta.get("file_type"),
"path": meta.get("path"),
"minio_url": meta.get("minio_url"),
"markdown_file": meta.get("markdown_file"),
"status": meta.get("status"),
"content_hash": meta.get("content_hash"),
"file_size": meta.get("size"),
"content_type": meta.get("content_type"),
"processing_params": meta.get("processing_params"),
"is_folder": meta.get("is_folder", False),
"error_message": meta.get("error"),
"created_by": str(meta.get("created_by")) if meta.get("created_by") else None,
"updated_by": str(meta.get("updated_by")) if meta.get("updated_by") else None,
},
)
async def _persist_kb(self, db_id: str) -> None:
"""只保存单个知识库到数据库，避免全量遍历"""
from src.repositories.knowledge_base_repository import KnowledgeBaseRepository
kb_repo = KnowledgeBaseRepository()
if db_id not in self.databases_meta:
return
meta = self.databases_meta[db_id]
existing = await kb_repo.get_by_id(db_id)
payload = {
"db_id": db_id,
"name": meta.get("name") or db_id,
"description": meta.get("description"),
"kb_type": meta.get("kb_type") or self.kb_type,
"embed_info": meta.get("embed_info"),
"llm_info": meta.get("llm_info"),
"query_params": meta.get("query_params"),
"additional_params": meta.get("metadata") or {},
}
if existing is None:
await kb_repo.create(payload)
else:
await kb_repo.update(
db_id,
{
"name": payload["name"],
"description": payload["description"],
"kb_type": payload["kb_type"],
"embed_info": payload["embed_info"],
"llm_info": payload["llm_info"],
"query_params": payload["query_params"],
"additional_params": payload["additional_params"],
},
)

13
src/knowledge/implementations/lightrag.py
View File

@ -284,7 +284,7 @@ class LightRagKB(KnowledgeBase):
self.files_meta[file_id]["updated_at"] = utc_isoformat()
if operator_id:
self.files_meta[file_id]["updated_by"] = operator_id
await self._save_metadata()
await self._persist_file(file_id)
# Add to processing queue
self._add_to_processing_queue(file_id)
@ -307,7 +307,7 @@ class LightRagKB(KnowledgeBase):
self.files_meta[file_id]["updated_at"] = utc_isoformat()
if operator_id:
self.files_meta[file_id]["updated_by"] = operator_id
await self._save_metadata()
await self._persist_file(file_id)
return self.files_meta[file_id]
@ -318,7 +318,7 @@ class LightRagKB(KnowledgeBase):
self.files_meta[file_id]["updated_at"] = utc_isoformat()
if operator_id:
self.files_meta[file_id]["updated_by"] = operator_id
await self._save_metadata()
await self._persist_file(file_id)
raise
finally:
@ -360,7 +360,7 @@ class LightRagKB(KnowledgeBase):
# 更新状态为处理中
self.files_meta[file_id]["processing_params"] = params.copy()
self.files_meta[file_id]["status"] = "processing"
await self._save_metadata()
await self._persist_file(file_id)
# 重新解析文件为 markdown
if content_type != "file":
@ -379,7 +379,7 @@ class LightRagKB(KnowledgeBase):
# 更新元数据状态
self.files_meta[file_id]["status"] = "done"
await self._save_metadata()
await self._persist_file(file_id)
# 从处理队列中移除
self._remove_from_processing_queue(file_id)
@ -395,7 +395,7 @@ class LightRagKB(KnowledgeBase):
logger.error(f"更新{content_type} {file_path} 失败: {error_msg}, {traceback.format_exc()}")
self.files_meta[file_id]["status"] = "failed"
self.files_meta[file_id]["error"] = error_msg
await self._save_metadata()
await self._persist_file(file_id)
# 从处理队列中移除
self._remove_from_processing_queue(file_id)
@ -508,7 +508,6 @@ class LightRagKB(KnowledgeBase):
from src.repositories.knowledge_file_repository import KnowledgeFileRepository
await KnowledgeFileRepository().delete(file_id)
await self._save_metadata()
async def get_file_basic_info(self, db_id: str, file_id: str) -> dict:
"""获取文件基本信息（仅元数据）"""

13
src/knowledge/implementations/milvus.py
View File

@ -281,7 +281,7 @@ class MilvusKB(KnowledgeBase):
self.files_meta[file_id]["updated_at"] = utc_isoformat()
if operator_id:
self.files_meta[file_id]["updated_by"] = operator_id
await self._save_metadata()
await self._persist_file(file_id)
# Read processing params inside lock to ensure we get the latest values
params = file_meta.get("processing_params", {}) or {}
@ -334,7 +334,7 @@ class MilvusKB(KnowledgeBase):
self.files_meta[file_id]["updated_at"] = utc_isoformat()
if operator_id:
self.files_meta[file_id]["updated_by"] = operator_id
await self._save_metadata()
await self._persist_file(file_id)
return self.files_meta[file_id]
except Exception as e:
@ -345,7 +345,7 @@ class MilvusKB(KnowledgeBase):
self.files_meta[file_id]["updated_at"] = utc_isoformat()
if operator_id:
self.files_meta[file_id]["updated_by"] = operator_id
await self._save_metadata()
await self._persist_file(file_id)
raise
finally:
@ -393,7 +393,7 @@ class MilvusKB(KnowledgeBase):
async with self._metadata_lock:
self.files_meta[file_id]["processing_params"] = params.copy()
self.files_meta[file_id]["status"] = "processing"
await self._save_metadata()
await self._persist_file(file_id)
# 重新解析文件为 markdown
if content_type != "file":
@ -431,7 +431,7 @@ class MilvusKB(KnowledgeBase):
# 更新元数据状态
async with self._metadata_lock:
self.files_meta[file_id]["status"] = "done"
await self._save_metadata()
await self._persist_file(file_id)
# 从处理队列中移除
self._remove_from_processing_queue(file_id)
@ -446,7 +446,7 @@ class MilvusKB(KnowledgeBase):
logger.error(f"更新{content_type} {file_path} 失败: {e}, {traceback.format_exc()}")
async with self._metadata_lock:
self.files_meta[file_id]["status"] = "failed"
await self._save_metadata()
await self._persist_file(file_id)
# 从处理队列中移除
self._remove_from_processing_queue(file_id)
@ -703,7 +703,6 @@ class MilvusKB(KnowledgeBase):
from src.repositories.knowledge_file_repository import KnowledgeFileRepository
await KnowledgeFileRepository().delete(file_id)
await self._save_metadata()
async def get_file_basic_info(self, db_id: str, file_id: str) -> dict:
"""获取文件基本信息（仅元数据）"""

2
web/src/components/AgentPanel.vue
View File

@ -817,10 +817,8 @@ const stopResize = () => {
min-height: 300px;
max-height: 60vh;
overflow-y: auto;
background: var(--main-5);
border-radius: 6px;
padding: 16px;
border: 1px solid var(--gray-200);
&::-webkit-scrollbar {
width: 8px;

13
web/src/views/DataBaseInfoView.vue
View File

@ -589,20 +589,21 @@ const handleMouseUp = () => {
display: flex;
align-items: center;
gap: 6px;
padding: 4px 8px;
background: var(--color-warning-50);
border-radius: 4px;
padding: 6px 12px;
background: var(--color-info-50);
border-left: 3px solid var(--color-info-500);
border-radius: 2px;
font-size: 13px;
color: var(--color-warning-700);
color: var(--color-info-700);
cursor: pointer;
transition: all 0.2s;
&:hover {
background: var(--color-warning-100);
background: var(--color-info-100);
}
svg {
color: var(--color-warning-700);
color: var(--color-info-500);
}
}
}