diff --git a/backend/package/yuxi/knowledge/base.py b/backend/package/yuxi/knowledge/base.py index 0fd7d8da..278c8e8c 100644 --- a/backend/package/yuxi/knowledge/base.py +++ b/backend/package/yuxi/knowledge/base.py @@ -7,6 +7,7 @@ from yuxi.knowledge.chunking.ragflow_like.presets import ( ensure_chunk_defaults_in_additional_params, resolve_chunk_processing_params, ) +from yuxi.knowledge.utils import sanitize_processing_params from yuxi.utils import logger from yuxi.utils.datetime_utils import coerce_any_to_utc_datetime, utc_isoformat @@ -667,7 +668,6 @@ class KnowledgeBase(ABC): "type": file_info.get("file_type", ""), "status": file_info.get("status", "done"), "created_at": created_at, - "processing_params": file_info.get("processing_params", None), "is_folder": file_info.get("is_folder", False), "parent_id": file_info.get("parent_id", None), } @@ -1057,9 +1057,11 @@ class KnowledgeBase(ABC): "content_hash": record.content_hash, "size": record.file_size, "content_type": record.content_type, - "processing_params": resolve_chunk_processing_params( - kb_additional_params=kb_additional_params, - file_processing_params=record.processing_params, + "processing_params": sanitize_processing_params( + resolve_chunk_processing_params( + kb_additional_params=kb_additional_params, + file_processing_params=record.processing_params, + ) ), "is_folder": record.is_folder, "error": record.error_message, @@ -1153,7 +1155,7 @@ class KnowledgeBase(ABC): "content_hash": meta.get("content_hash"), "file_size": meta.get("size"), "content_type": meta.get("content_type"), - "processing_params": meta.get("processing_params"), + "processing_params": sanitize_processing_params(meta.get("processing_params")), "is_folder": meta.get("is_folder", False), "error_message": meta.get("error"), "created_by": str(meta.get("created_by")) if meta.get("created_by") else None, @@ -1207,7 +1209,7 @@ class KnowledgeBase(ABC): "content_hash": meta.get("content_hash"), "file_size": meta.get("size"), "content_type": meta.get("content_type"), - "processing_params": meta.get("processing_params"), + "processing_params": sanitize_processing_params(meta.get("processing_params")), "is_folder": meta.get("is_folder", False), "error_message": meta.get("error"), "created_by": str(meta.get("created_by")) if meta.get("created_by") else None, diff --git a/backend/package/yuxi/knowledge/utils/__init__.py b/backend/package/yuxi/knowledge/utils/__init__.py index 551ab5db..1a983f9e 100644 --- a/backend/package/yuxi/knowledge/utils/__init__.py +++ b/backend/package/yuxi/knowledge/utils/__init__.py @@ -10,6 +10,7 @@ from .kb_utils import ( get_embedding_config, merge_processing_params, prepare_item_metadata, + sanitize_processing_params, split_text_into_chunks, validate_file_path, ) @@ -18,6 +19,7 @@ __all__ = [ "calculate_content_hash", "get_embedding_config", "prepare_item_metadata", + "sanitize_processing_params", "split_text_into_chunks", "merge_processing_params", "validate_file_path", diff --git a/backend/package/yuxi/knowledge/utils/kb_utils.py b/backend/package/yuxi/knowledge/utils/kb_utils.py index 20040ed1..3641c65e 100644 --- a/backend/package/yuxi/knowledge/utils/kb_utils.py +++ b/backend/package/yuxi/knowledge/utils/kb_utils.py @@ -91,6 +91,17 @@ def _unescape_separator(separator: str | None) -> str | None: return separator +def sanitize_processing_params(params: dict | None) -> dict | None: + """移除批次级临时字段,避免写入单文件元数据。""" + if not params: + return None + + sanitized = params.copy() + sanitized.pop("_preprocessed_map", None) + sanitized.pop("content_hashes", None) + return sanitized + + def split_text_into_chunks(text: str, file_id: str, filename: str, params: dict = {}) -> list[dict]: """ 将文本分割成块,使用 LangChain 的 MarkdownTextSplitter 进行智能分割 @@ -222,9 +233,7 @@ async def prepare_item_metadata(item: str, content_type: str, db_id: str, params } if params: - # 移除内部参数以免污染 metadata - safe_params = params.copy() - safe_params.pop("_preprocessed_map", None) + safe_params = sanitize_processing_params(params) or {} # 覆盖 content_type 为 file,确保后续解析走文件流程(MinIO 下载 -> HTML 解析) # 而不是再次尝试作为 URL 抓取 safe_params["content_type"] = "file" @@ -314,7 +323,7 @@ async def prepare_item_metadata(item: str, content_type: str, db_id: str, params # 保存处理参数到元数据 if params: - metadata["processing_params"] = params.copy() + metadata["processing_params"] = sanitize_processing_params(params) return metadata diff --git a/backend/test/test_ragflow_like_chunking.py b/backend/test/test_ragflow_like_chunking.py index 88f7d684..3e900c63 100644 --- a/backend/test/test_ragflow_like_chunking.py +++ b/backend/test/test_ragflow_like_chunking.py @@ -13,6 +13,7 @@ from yuxi.knowledge.chunking.ragflow_like.presets import ( map_to_internal_parser_id, resolve_chunk_processing_params, ) +from yuxi.knowledge.utils.kb_utils import sanitize_processing_params def test_general_maps_to_naive() -> None: @@ -218,3 +219,15 @@ def test_laws_markdown_articles_should_not_collapse_into_chapter_chunk() -> None # 条级切分时,第一条与第二条不应被合并到同一块。 assert all("第二条" not in chunk for chunk in first_article_chunks) assert max(count_tokens(ck["content"]) for ck in chunks) <= 120 + + +def test_sanitize_processing_params_should_drop_batch_only_fields() -> None: + sanitized = sanitize_processing_params( + { + "chunk_preset_id": "general", + "content_hashes": {"a.md": "hash-a"}, + "_preprocessed_map": {"a.md": {"path": "/tmp/a.md"}}, + } + ) + + assert sanitized == {"chunk_preset_id": "general"} diff --git a/docs/develop-guides/roadmap.md b/docs/develop-guides/roadmap.md index 17ea45c2..844fc671 100644 --- a/docs/develop-guides/roadmap.md +++ b/docs/develop-guides/roadmap.md @@ -72,6 +72,7 @@ - 修复沙盒后端接入回归:补齐 composite backend 的 `sandbox_backend` 参数、限制 `/api/sandbox/prepare` 仅允许访问当前用户线程、确保 `release()` 之后的 `destroy()` 会真正停止热池容器,并恢复 docker-compose 的完整模式默认值 - 重构沙盒为 deer-flow 风格的 AIO provider:切换为 thread-local sandbox、统一 `/home/yuxi/user-data/{workspace,uploads,outputs}` 固定路径、移除公开 `/api/sandbox/*` 生命周期接口,并补充 lite 模式下的 provider 生命周期、filesystem API 与 sandbox 复用/隔离 E2E 验证 - 调整聊天附件存储链路:线程附件改为直接落盘到 `saves/threads//user-data/uploads`,解析成功后额外生成 `uploads/attachments/*.md`,不再依赖 MinIO 或显式上传到 sandbox +- 修复知识库文件列表包体异常膨胀:上传阶段不再把批次级 `content_hashes` 写入每个文件的 `processing_params`,并从数据库详情列表接口中移除该字段,改为按需读取单文件详情 ## v0.4 diff --git a/web/src/components/FileTable.vue b/web/src/components/FileTable.vue index 38100b48..c1e7c22a 100644 --- a/web/src/components/FileTable.vue +++ b/web/src/components/FileTable.vue @@ -1157,23 +1157,32 @@ const handleParseFile = async (record) => { await store.parseFiles([record.file_id]) } +const defaultIndexParams = { + chunk_size: 1000, + chunk_overlap: 200, + qa_separator: '', + chunk_preset_id: '' +} + +const loadRecordProcessingParams = async (record) => { + if (record?.processing_params) { + return record.processing_params + } + + const detail = await documentApi.getDocumentInfo(store.databaseId, record.file_id) + return detail?.processing_params || null +} + const handleIndexFile = async (record) => { closePopover(record.file_id) - // 打开参数配置弹窗 currentIndexFileIds.value = [record.file_id] isBatchIndexOperation.value = false indexConfigModalTitle.value = '入库参数配置' - if (record?.processing_params) { - Object.assign(indexParams.value, record.processing_params) - } else { - // Reset to defaults if no existing params - Object.assign(indexParams.value, { - chunk_size: 1000, - chunk_overlap: 200, - qa_separator: '', - chunk_preset_id: '' - }) + Object.assign(indexParams.value, defaultIndexParams) + const processingParams = await loadRecordProcessingParams(record) + if (processingParams) { + Object.assign(indexParams.value, processingParams) } indexConfigModalVisible.value = true @@ -1185,11 +1194,12 @@ const handleReindexFile = async (record) => { isBatchIndexOperation.value = false indexConfigModalTitle.value = '重新入库参数配置' - if (record?.processing_params) { - Object.assign(indexParams.value, record.processing_params) + Object.assign(indexParams.value, defaultIndexParams) + const processingParams = await loadRecordProcessingParams(record) + if (processingParams) { + Object.assign(indexParams.value, processingParams) } - // 显示参数配置模态框 indexConfigModalVisible.value = true } @@ -1230,12 +1240,7 @@ const handleIndexConfigCancel = () => { currentIndexFileIds.value = [] isBatchIndexOperation.value = false // 重置参数为默认值 - Object.assign(indexParams.value, { - chunk_size: 1000, - chunk_overlap: 200, - qa_separator: '', - chunk_preset_id: '' - }) + Object.assign(indexParams.value, defaultIndexParams) } // 导入工具函数