fix(kb): 修剪文件列表负载并清理处理参数
This commit is contained in:
Wenjie Zhang
parent
ecb2252e36
commit
1d8a5e88af
@ -7,6 +7,7 @@ from yuxi.knowledge.chunking.ragflow_like.presets import (
|
||||
ensure_chunk_defaults_in_additional_params,
|
||||
resolve_chunk_processing_params,
|
||||
)
|
||||
from yuxi.knowledge.utils import sanitize_processing_params
|
||||
from yuxi.utils import logger
|
||||
from yuxi.utils.datetime_utils import coerce_any_to_utc_datetime, utc_isoformat
|
||||
|
||||
@ -667,7 +668,6 @@ class KnowledgeBase(ABC):
|
||||
"type": file_info.get("file_type", ""),
|
||||
"status": file_info.get("status", "done"),
|
||||
"created_at": created_at,
|
||||
"processing_params": file_info.get("processing_params", None),
|
||||
"is_folder": file_info.get("is_folder", False),
|
||||
"parent_id": file_info.get("parent_id", None),
|
||||
}
|
||||
@ -1057,9 +1057,11 @@ class KnowledgeBase(ABC):
|
||||
"content_hash": record.content_hash,
|
||||
"size": record.file_size,
|
||||
"content_type": record.content_type,
|
||||
"processing_params": resolve_chunk_processing_params(
|
||||
kb_additional_params=kb_additional_params,
|
||||
file_processing_params=record.processing_params,
|
||||
"processing_params": sanitize_processing_params(
|
||||
resolve_chunk_processing_params(
|
||||
kb_additional_params=kb_additional_params,
|
||||
file_processing_params=record.processing_params,
|
||||
)
|
||||
),
|
||||
"is_folder": record.is_folder,
|
||||
"error": record.error_message,
|
||||
@ -1153,7 +1155,7 @@ class KnowledgeBase(ABC):
|
||||
"content_hash": meta.get("content_hash"),
|
||||
"file_size": meta.get("size"),
|
||||
"content_type": meta.get("content_type"),
|
||||
"processing_params": meta.get("processing_params"),
|
||||
"processing_params": sanitize_processing_params(meta.get("processing_params")),
|
||||
"is_folder": meta.get("is_folder", False),
|
||||
"error_message": meta.get("error"),
|
||||
"created_by": str(meta.get("created_by")) if meta.get("created_by") else None,
|
||||
@ -1207,7 +1209,7 @@ class KnowledgeBase(ABC):
|
||||
"content_hash": meta.get("content_hash"),
|
||||
"file_size": meta.get("size"),
|
||||
"content_type": meta.get("content_type"),
|
||||
"processing_params": meta.get("processing_params"),
|
||||
"processing_params": sanitize_processing_params(meta.get("processing_params")),
|
||||
"is_folder": meta.get("is_folder", False),
|
||||
"error_message": meta.get("error"),
|
||||
"created_by": str(meta.get("created_by")) if meta.get("created_by") else None,
|
||||
|
||||
@ -10,6 +10,7 @@ from .kb_utils import (
|
||||
get_embedding_config,
|
||||
merge_processing_params,
|
||||
prepare_item_metadata,
|
||||
sanitize_processing_params,
|
||||
split_text_into_chunks,
|
||||
validate_file_path,
|
||||
)
|
||||
@ -18,6 +19,7 @@ __all__ = [
|
||||
"calculate_content_hash",
|
||||
"get_embedding_config",
|
||||
"prepare_item_metadata",
|
||||
"sanitize_processing_params",
|
||||
"split_text_into_chunks",
|
||||
"merge_processing_params",
|
||||
"validate_file_path",
|
||||
|
||||
@ -91,6 +91,17 @@ def _unescape_separator(separator: str | None) -> str | None:
|
||||
return separator
|
||||
|
||||
|
||||
def sanitize_processing_params(params: dict | None) -> dict | None:
|
||||
"""移除批次级临时字段，避免写入单文件元数据。"""
|
||||
if not params:
|
||||
return None
|
||||
|
||||
sanitized = params.copy()
|
||||
sanitized.pop("_preprocessed_map", None)
|
||||
sanitized.pop("content_hashes", None)
|
||||
return sanitized
|
||||
|
||||
|
||||
def split_text_into_chunks(text: str, file_id: str, filename: str, params: dict = {}) -> list[dict]:
|
||||
"""
|
||||
将文本分割成块，使用 LangChain 的 MarkdownTextSplitter 进行智能分割
|
||||
@ -222,9 +233,7 @@ async def prepare_item_metadata(item: str, content_type: str, db_id: str, params
|
||||
}
|
||||
|
||||
if params:
|
||||
# 移除内部参数以免污染 metadata
|
||||
safe_params = params.copy()
|
||||
safe_params.pop("_preprocessed_map", None)
|
||||
safe_params = sanitize_processing_params(params) or {}
|
||||
# 覆盖 content_type 为 file，确保后续解析走文件流程（MinIO 下载 -> HTML 解析）
|
||||
# 而不是再次尝试作为 URL 抓取
|
||||
safe_params["content_type"] = "file"
|
||||
@ -314,7 +323,7 @@ async def prepare_item_metadata(item: str, content_type: str, db_id: str, params
|
||||
|
||||
# 保存处理参数到元数据
|
||||
if params:
|
||||
metadata["processing_params"] = params.copy()
|
||||
metadata["processing_params"] = sanitize_processing_params(params)
|
||||
|
||||
return metadata
|
||||
|
||||
|
||||
@ -13,6 +13,7 @@ from yuxi.knowledge.chunking.ragflow_like.presets import (
|
||||
map_to_internal_parser_id,
|
||||
resolve_chunk_processing_params,
|
||||
)
|
||||
from yuxi.knowledge.utils.kb_utils import sanitize_processing_params
|
||||
|
||||
|
||||
def test_general_maps_to_naive() -> None:
|
||||
@ -218,3 +219,15 @@ def test_laws_markdown_articles_should_not_collapse_into_chapter_chunk() -> None
|
||||
# 条级切分时，第一条与第二条不应被合并到同一块。
|
||||
assert all("第二条" not in chunk for chunk in first_article_chunks)
|
||||
assert max(count_tokens(ck["content"]) for ck in chunks) <= 120
|
||||
|
||||
|
||||
def test_sanitize_processing_params_should_drop_batch_only_fields() -> None:
|
||||
sanitized = sanitize_processing_params(
|
||||
{
|
||||
"chunk_preset_id": "general",
|
||||
"content_hashes": {"a.md": "hash-a"},
|
||||
"_preprocessed_map": {"a.md": {"path": "/tmp/a.md"}},
|
||||
}
|
||||
)
|
||||
|
||||
assert sanitized == {"chunk_preset_id": "general"}
|
||||
|
||||
@ -72,6 +72,7 @@
|
||||
- 修复沙盒后端接入回归：补齐 composite backend 的 `sandbox_backend` 参数、限制 `/api/sandbox/prepare` 仅允许访问当前用户线程、确保 `release()` 之后的 `destroy()` 会真正停止热池容器，并恢复 docker-compose 的完整模式默认值
|
||||
- 重构沙盒为 deer-flow 风格的 AIO provider：切换为 thread-local sandbox、统一 `/home/yuxi/user-data/{workspace,uploads,outputs}` 固定路径、移除公开 `/api/sandbox/*` 生命周期接口，并补充 lite 模式下的 provider 生命周期、filesystem API 与 sandbox 复用/隔离 E2E 验证
|
||||
- 调整聊天附件存储链路：线程附件改为直接落盘到 `saves/threads/<thread_id>/user-data/uploads`，解析成功后额外生成 `uploads/attachments/*.md`，不再依赖 MinIO 或显式上传到 sandbox
|
||||
- 修复知识库文件列表包体异常膨胀：上传阶段不再把批次级 `content_hashes` 写入每个文件的 `processing_params`，并从数据库详情列表接口中移除该字段，改为按需读取单文件详情
|
||||
|
||||
## v0.4
|
||||
|
||||
|
||||
@ -1157,23 +1157,32 @@ const handleParseFile = async (record) => {
|
||||
await store.parseFiles([record.file_id])
|
||||
}
|
||||
|
||||
const defaultIndexParams = {
|
||||
chunk_size: 1000,
|
||||
chunk_overlap: 200,
|
||||
qa_separator: '',
|
||||
chunk_preset_id: ''
|
||||
}
|
||||
|
||||
const loadRecordProcessingParams = async (record) => {
|
||||
if (record?.processing_params) {
|
||||
return record.processing_params
|
||||
}
|
||||
|
||||
const detail = await documentApi.getDocumentInfo(store.databaseId, record.file_id)
|
||||
return detail?.processing_params || null
|
||||
}
|
||||
|
||||
const handleIndexFile = async (record) => {
|
||||
closePopover(record.file_id)
|
||||
// 打开参数配置弹窗
|
||||
currentIndexFileIds.value = [record.file_id]
|
||||
isBatchIndexOperation.value = false
|
||||
indexConfigModalTitle.value = '入库参数配置'
|
||||
|
||||
if (record?.processing_params) {
|
||||
Object.assign(indexParams.value, record.processing_params)
|
||||
} else {
|
||||
// Reset to defaults if no existing params
|
||||
Object.assign(indexParams.value, {
|
||||
chunk_size: 1000,
|
||||
chunk_overlap: 200,
|
||||
qa_separator: '',
|
||||
chunk_preset_id: ''
|
||||
})
|
||||
Object.assign(indexParams.value, defaultIndexParams)
|
||||
const processingParams = await loadRecordProcessingParams(record)
|
||||
if (processingParams) {
|
||||
Object.assign(indexParams.value, processingParams)
|
||||
}
|
||||
|
||||
indexConfigModalVisible.value = true
|
||||
@ -1185,11 +1194,12 @@ const handleReindexFile = async (record) => {
|
||||
isBatchIndexOperation.value = false
|
||||
indexConfigModalTitle.value = '重新入库参数配置'
|
||||
|
||||
if (record?.processing_params) {
|
||||
Object.assign(indexParams.value, record.processing_params)
|
||||
Object.assign(indexParams.value, defaultIndexParams)
|
||||
const processingParams = await loadRecordProcessingParams(record)
|
||||
if (processingParams) {
|
||||
Object.assign(indexParams.value, processingParams)
|
||||
}
|
||||
|
||||
// 显示参数配置模态框
|
||||
indexConfigModalVisible.value = true
|
||||
}
|
||||
|
||||
@ -1230,12 +1240,7 @@ const handleIndexConfigCancel = () => {
|
||||
currentIndexFileIds.value = []
|
||||
isBatchIndexOperation.value = false
|
||||
// 重置参数为默认值
|
||||
Object.assign(indexParams.value, {
|
||||
chunk_size: 1000,
|
||||
chunk_overlap: 200,
|
||||
qa_separator: '',
|
||||
chunk_preset_id: ''
|
||||
})
|
||||
Object.assign(indexParams.value, defaultIndexParams)
|
||||
}
|
||||
|
||||
// 导入工具函数
|
||||
|
||||
Loading…
Reference in New Issue
Block a user