Merge branch 'main' into feat/model-config
This commit is contained in:
commit
2920cbeec3
@ -302,7 +302,7 @@ class LightRagKB(KnowledgeBase):
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
async def index_file(self, db_id: str, file_id: str, operator_id: str | None = None) -> dict:
|
async def index_file(self, db_id: str, file_id: str, operator_id: str | None = None, params: dict | None = None) -> dict:
|
||||||
"""
|
"""
|
||||||
Index parsed file (Status: INDEXING -> INDEXED/ERROR_INDEXING)
|
Index parsed file (Status: INDEXING -> INDEXED/ERROR_INDEXING)
|
||||||
|
|
||||||
@ -310,6 +310,7 @@ class LightRagKB(KnowledgeBase):
|
|||||||
db_id: Database ID
|
db_id: Database ID
|
||||||
file_id: File ID
|
file_id: File ID
|
||||||
operator_id: ID of the user performing the operation
|
operator_id: ID of the user performing the operation
|
||||||
|
params: Override processing params to apply during indexing (merged on top of stored params)
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Updated file metadata
|
Updated file metadata
|
||||||
@ -366,14 +367,15 @@ class LightRagKB(KnowledgeBase):
|
|||||||
markdown_content = await self._read_markdown_from_minio(file_meta["markdown_file"])
|
markdown_content = await self._read_markdown_from_minio(file_meta["markdown_file"])
|
||||||
file_path = file_meta.get("path")
|
file_path = file_meta.get("path")
|
||||||
filename = file_meta.get("filename") or file_id
|
filename = file_meta.get("filename") or file_id
|
||||||
processing_params = resolve_chunk_processing_params(
|
params = resolve_chunk_processing_params(
|
||||||
kb_additional_params=self.databases_meta.get(db_id, {}).get("metadata"),
|
kb_additional_params=self.databases_meta.get(db_id, {}).get("metadata"),
|
||||||
file_processing_params=file_meta.get("processing_params"),
|
file_processing_params=file_meta.get("processing_params"),
|
||||||
|
request_params=params,
|
||||||
)
|
)
|
||||||
self.files_meta[file_id]["processing_params"] = processing_params
|
self.files_meta[file_id]["processing_params"] = params
|
||||||
await self._save_metadata()
|
await self._save_metadata()
|
||||||
|
|
||||||
chunks = chunk_markdown(markdown_content, file_id, filename, processing_params)
|
chunks = chunk_markdown(markdown_content, file_id, filename, params)
|
||||||
chunk_input, split_by_character, split_by_character_only = self._prepare_lightrag_insert_payload(chunks)
|
chunk_input, split_by_character, split_by_character_only = self._prepare_lightrag_insert_payload(chunks)
|
||||||
if not chunk_input:
|
if not chunk_input:
|
||||||
chunk_input = markdown_content
|
chunk_input = markdown_content
|
||||||
|
|||||||
@ -284,7 +284,7 @@ class MilvusKB(KnowledgeBase):
|
|||||||
"""将文本分割成块"""
|
"""将文本分割成块"""
|
||||||
return chunk_markdown(text, file_id, filename, params)
|
return chunk_markdown(text, file_id, filename, params)
|
||||||
|
|
||||||
async def index_file(self, db_id: str, file_id: str, operator_id: str | None = None) -> dict:
|
async def index_file(self, db_id: str, file_id: str, operator_id: str | None = None, params: dict | None = None) -> dict:
|
||||||
"""
|
"""
|
||||||
Index parsed file (Status: INDEXING -> INDEXED/ERROR_INDEXING)
|
Index parsed file (Status: INDEXING -> INDEXED/ERROR_INDEXING)
|
||||||
|
|
||||||
@ -292,6 +292,7 @@ class MilvusKB(KnowledgeBase):
|
|||||||
db_id: Database ID
|
db_id: Database ID
|
||||||
file_id: File ID
|
file_id: File ID
|
||||||
operator_id: ID of the user performing the operation
|
operator_id: ID of the user performing the operation
|
||||||
|
params: Override processing params to apply during indexing (merged on top of stored params)
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Updated file metadata
|
Updated file metadata
|
||||||
@ -342,10 +343,11 @@ class MilvusKB(KnowledgeBase):
|
|||||||
if operator_id:
|
if operator_id:
|
||||||
self.files_meta[file_id]["updated_by"] = operator_id
|
self.files_meta[file_id]["updated_by"] = operator_id
|
||||||
|
|
||||||
# Read processing params inside lock to ensure we get the latest values
|
# 将传入的 params 作为 request_params,确保用户指定的参数始终覆盖存储的参数
|
||||||
params = resolve_chunk_processing_params(
|
params = resolve_chunk_processing_params(
|
||||||
kb_additional_params=self.databases_meta.get(db_id, {}).get("metadata"),
|
kb_additional_params=self.databases_meta.get(db_id, {}).get("metadata"),
|
||||||
file_processing_params=file_meta.get("processing_params"),
|
file_processing_params=file_meta.get("processing_params"),
|
||||||
|
request_params=params,
|
||||||
)
|
)
|
||||||
self.files_meta[file_id]["processing_params"] = params
|
self.files_meta[file_id]["processing_params"] = params
|
||||||
await self._save_metadata()
|
await self._save_metadata()
|
||||||
|
|||||||
@ -415,10 +415,10 @@ class KnowledgeBaseManager:
|
|||||||
kb_instance = await self._get_kb_for_database(db_id)
|
kb_instance = await self._get_kb_for_database(db_id)
|
||||||
return await kb_instance.parse_file(db_id, file_id, operator_id)
|
return await kb_instance.parse_file(db_id, file_id, operator_id)
|
||||||
|
|
||||||
async def index_file(self, db_id: str, file_id: str, operator_id: str | None = None) -> dict:
|
async def index_file(self, db_id: str, file_id: str, operator_id: str | None = None, params: dict | None = None) -> dict:
|
||||||
"""Index parsed file"""
|
"""Index parsed file"""
|
||||||
kb_instance = await self._get_kb_for_database(db_id)
|
kb_instance = await self._get_kb_for_database(db_id)
|
||||||
return await kb_instance.index_file(db_id, file_id, operator_id)
|
return await kb_instance.index_file(db_id, file_id, operator_id, params=params)
|
||||||
|
|
||||||
async def update_file_params(self, db_id: str, file_id: str, params: dict, operator_id: str | None = None) -> None:
|
async def update_file_params(self, db_id: str, file_id: str, params: dict, operator_id: str | None = None) -> None:
|
||||||
"""Update file processing params"""
|
"""Update file processing params"""
|
||||||
|
|||||||
@ -442,8 +442,8 @@ async def add_documents(
|
|||||||
await knowledge_base.update_file_params(
|
await knowledge_base.update_file_params(
|
||||||
db_id, file_id, indexing_params, operator_id=current_user.user_id
|
db_id, file_id, indexing_params, operator_id=current_user.user_id
|
||||||
)
|
)
|
||||||
# 2. 执行入库
|
# 2. 执行入库(传入 indexing_params 确保使用的参数与用户设置一致)
|
||||||
result = await knowledge_base.index_file(db_id, file_id, operator_id=current_user.user_id)
|
result = await knowledge_base.index_file(db_id, file_id, operator_id=current_user.user_id, params=indexing_params)
|
||||||
processed_items.append(result)
|
processed_items.append(result)
|
||||||
except Exception as index_error:
|
except Exception as index_error:
|
||||||
logger.error(f"自动入库失败 {item} (file_id={file_id}): {index_error}")
|
logger.error(f"自动入库失败 {item} (file_id={file_id}): {index_error}")
|
||||||
@ -605,7 +605,7 @@ async def index_documents(
|
|||||||
await context.set_progress(progress, f"正在入库第 {idx}/{total} 个文档")
|
await context.set_progress(progress, f"正在入库第 {idx}/{total} 个文档")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
result = await knowledge_base.index_file(db_id, file_id, operator_id=operator_id)
|
result = await knowledge_base.index_file(db_id, file_id, operator_id=operator_id, params=params)
|
||||||
processed_items.append(result)
|
processed_items.append(result)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Index failed for {file_id}: {e}")
|
logger.error(f"Index failed for {file_id}: {e}")
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user