diff --git a/src/knowledge/implementations/chroma.py b/src/knowledge/implementations/chroma.py index 81949dc1..d335d45f 100644 --- a/src/knowledge/implementations/chroma.py +++ b/src/knowledge/implementations/chroma.py @@ -14,7 +14,6 @@ from src.knowledge.utils.kb_utils import ( get_embedding_config, prepare_item_metadata, split_text_into_chunks, - split_text_into_qa_chunks, ) from src.utils import logger from src.utils.datetime_utils import utc_isoformat @@ -151,16 +150,7 @@ class ChromaKB(KnowledgeBase): def _split_text_into_chunks(self, text: str, file_id: str, filename: str, params: dict) -> list[dict]: """将文本分割成块""" - # 检查是否使用QA分割模式 - use_qa_split = params.get("use_qa_split", False) - - if use_qa_split: - # 使用QA分割模式 - qa_separator = params.get("qa_separator", "\n\n\n") - chunks = split_text_into_qa_chunks(text, file_id, filename, qa_separator, params) - else: - # 使用传统分割模式 - chunks = split_text_into_chunks(text, file_id, filename, params) + chunks = split_text_into_chunks(text, file_id, filename, params) # 为 ChromaDB 添加特定的 metadata 格式 for chunk in chunks: diff --git a/src/knowledge/implementations/milvus.py b/src/knowledge/implementations/milvus.py index a1d81c57..adbb1def 100644 --- a/src/knowledge/implementations/milvus.py +++ b/src/knowledge/implementations/milvus.py @@ -14,7 +14,6 @@ from src.knowledge.utils.kb_utils import ( get_embedding_config, prepare_item_metadata, split_text_into_chunks, - split_text_into_qa_chunks, ) from src.models.embed import OtherEmbedding from src.utils import hashstr, logger @@ -222,16 +221,7 @@ class MilvusKB(KnowledgeBase): def _split_text_into_chunks(self, text: str, file_id: str, filename: str, params: dict) -> list[dict]: """将文本分割成块""" - # 检查是否使用QA分割模式 - use_qa_split = params.get("use_qa_split", False) - - if use_qa_split: - # 使用QA分割模式 - qa_separator = params.get("qa_separator", "\n\n\n") - return split_text_into_qa_chunks(text, file_id, filename, qa_separator, params) - else: - # 使用传统分割模式 - return split_text_into_chunks(text, file_id, filename, params) + return split_text_into_chunks(text, file_id, filename, params) async def add_content(self, db_id: str, items: list[str], params: dict | None = {}) -> list[dict]: """添加内容（文件/URL）""" diff --git a/src/knowledge/utils/__init__.py b/src/knowledge/utils/__init__.py index 4863799a..551ab5db 100644 --- a/src/knowledge/utils/__init__.py +++ b/src/knowledge/utils/__init__.py @@ -11,7 +11,6 @@ from .kb_utils import ( merge_processing_params, prepare_item_metadata, split_text_into_chunks, - split_text_into_qa_chunks, validate_file_path, ) @@ -20,7 +19,6 @@ __all__ = [ "get_embedding_config", "prepare_item_metadata", "split_text_into_chunks", - "split_text_into_qa_chunks", "merge_processing_params", "validate_file_path", ] diff --git a/src/knowledge/utils/kb_utils.py b/src/knowledge/utils/kb_utils.py index c4cd7935..2506fa21 100644 --- a/src/knowledge/utils/kb_utils.py +++ b/src/knowledge/utils/kb_utils.py @@ -74,6 +74,23 @@ def validate_file_path(file_path: str, db_id: str = None) -> str: raise ValueError(f"Invalid file path: {file_path}") +def _unescape_separator(separator: str | None) -> str | None: + """将前端传入的字面量转义字符转换为实际字符 + + 例如: "\\n\\n\\n" -> "\n\n\n" + """ + if not separator: + return None + + # 处理常见的转义序列 + separator = separator.replace("\\n", "\n") + separator = separator.replace("\\r", "\r") + separator = separator.replace("\\t", "\t") + separator = separator.replace("\\\\", "\\") + + return separator + + def split_text_into_chunks(text: str, file_id: str, filename: str, params: dict = {}) -> list[dict]: """ 将文本分割成块，使用 LangChain 的 MarkdownTextSplitter 进行智能分割 @@ -82,6 +99,16 @@ def split_text_into_chunks(text: str, file_id: str, filename: str, params: dict chunk_size = params.get("chunk_size", 1000) chunk_overlap = params.get("chunk_overlap", 200) + # 获取分隔符并转换为实际字符 + separator = params.get("qa_separator") + separator = _unescape_separator(separator) + + # 向后兼容：如果旧配置设置了 use_qa_split=True 但未指定 separator，使用默认分隔符 + use_qa_split = params.get("use_qa_split", False) + if use_qa_split and not separator: + separator = "\n\n\n" + logger.debug("启用了向后兼容模式：use_qa_split=True，使用默认分隔符 \\n\\n\\n") + # 使用 MarkdownTextSplitter 进行智能分割 # MarkdownTextSplitter 会尝试沿着 Markdown 格式的标题进行分割 text_splitter = MarkdownTextSplitter( @@ -89,7 +116,18 @@ def split_text_into_chunks(text: str, file_id: str, filename: str, params: dict chunk_overlap=chunk_overlap, ) - text_chunks = text_splitter.split_text(text) + # 如果设置了分隔符，先分割后以当前的分割逻辑处理 + if separator: + # 转换分隔符为可视格式（换行符显示为 \n） + separator_display = separator.replace('\n', '\\n').replace('\r', '\\r').replace('\t', '\\t') + logger.debug(f"启用预分割模式，使用分隔符: '{separator_display}'") + pre_chunks = text.split(separator) + text_chunks = [] + for pre_chunk in pre_chunks: + if pre_chunk.strip(): + text_chunks.extend(text_splitter.split_text(pre_chunk)) + else: + text_chunks = text_splitter.split_text(text) # 转换为标准格式 for chunk_index, chunk_content in enumerate(text_chunks): @@ -136,7 +174,8 @@ async def calculate_content_hash(data: bytes | bytearray | str | os.PathLike[str return sha256.hexdigest() - raise TypeError(f"Unsupported data type for hashing: {type(data)!r}") + # 理论上不会执行到这里，但保留作为防御性编程 + raise TypeError(f"Unsupported data type for hashing: {type(data)!r}") # type: ignore[unreachable] async def prepare_item_metadata(item: str, content_type: str, db_id: str, params: dict | None = None) -> dict: @@ -223,39 +262,6 @@ async def prepare_item_metadata(item: str, content_type: str, db_id: str, params return metadata -def split_text_into_qa_chunks( - text: str, file_id: str, filename: str, qa_separator: None | str = None, params: dict = {} -) -> list[dict]: - """ - 将文本按QA对分割成块，使用 LangChain 的 CharacterTextSplitter 进行分割""" - qa_separator = qa_separator or "\n\n" - text_chunks = text.split(qa_separator) - - # 转换为标准格式 - chunks = [] - for chunk_index, chunk_content in enumerate(text_chunks): - if chunk_content.strip(): # 跳过空块 - chunk_content = chunk_content.strip()[:4096] - chunks.append( - { - "id": f"{file_id}_qa_chunk_{chunk_index}", - "content": chunk_content.strip(), - "file_id": file_id, - "filename": filename, - "chunk_index": chunk_index, - "source": filename, - "chunk_id": f"{file_id}_qa_chunk_{chunk_index}", - "chunk_type": "qa", # 标识为QA类型的chunk - } - ) - - logger.debug(f"QA chunks: {chunks[0]}") - logger.debug( - f"Successfully split QA text into {len(chunks)} chunks using CharacterTextSplitter with `{qa_separator=}`" - ) - return chunks - - def merge_processing_params(metadata_params: dict | None, request_params: dict | None) -> dict: """ 合并处理参数：优先使用请求参数，缺失时使用元数据中的参数 diff --git a/web/src/components/ChunkParamsConfig.vue b/web/src/components/ChunkParamsConfig.vue index b7287584..d0659b96 100644 --- a/web/src/components/ChunkParamsConfig.vue +++ b/web/src/components/ChunkParamsConfig.vue @@ -17,13 +17,13 @@

相邻文本片段间的重叠字符数

- - -

启用后将按QA对分割，忽略上述chunk大小设置

- - - -

用于分割不同QA对的分隔符

+ + +

支持 \n、\t 等转义字符。留空则不启用预分割

 diff --git a/web/src/components/FileTable.vue b/web/src/components/FileTable.vue index e1e8043c..8dd2a31b 100644 --- a/web/src/components/FileTable.vue +++ b/web/src/components/FileTable.vue @@ -379,8 +379,7 @@ const rechunkModalLoading = computed(() => store.state.chunkLoading); const rechunkParams = ref({ chunk_size: 1000, chunk_overlap: 200, - use_qa_split: false, - qa_separator: '\n\n\n' + qa_separator: '\\n\\n\\n' }); const currentRechunkFileIds = ref([]); const isBatchRechunk = ref(false); @@ -788,8 +787,7 @@ const handleRechunkConfirm = async () => { rechunkParams.value = { chunk_size: 1000, chunk_overlap: 200, - use_qa_split: false, - qa_separator: '\n\n\n' + qa_separator: '\\n\\n\\n' }; } else { message.error(`重新分块失败: ${result.message}`); @@ -811,8 +809,7 @@ const handleRechunkCancel = () => { rechunkParams.value = { chunk_size: 1000, chunk_overlap: 200, - use_qa_split: false, - qa_separator: '\n\n\n' + qa_separator: '\\n\\n\\n' }; }; diff --git a/web/src/components/FileUploadModal.vue b/web/src/components/FileUploadModal.vue index 83f069dd..2bf6e951 100644 --- a/web/src/components/FileUploadModal.vue +++ b/web/src/components/FileUploadModal.vue @@ -365,8 +365,7 @@ const chunkParams = ref({ chunk_size: 1000, chunk_overlap: 200, enable_ocr: 'disable', - use_qa_split: false, - qa_separator: '\n\n\n', + qa_separator: '', }); // 分块参数配置弹窗 @@ -376,8 +375,7 @@ const chunkConfigModalVisible = ref(false); const tempChunkParams = ref({ chunk_size: 1000, chunk_overlap: 200, - use_qa_split: false, - qa_separator: '\n\n\n', + qa_separator: '', }); // 计算属性：是否支持QA分割 @@ -739,7 +737,6 @@ const showChunkConfigModal = () => { tempChunkParams.value = { chunk_size: chunkParams.value.chunk_size, chunk_overlap: chunkParams.value.chunk_overlap, - use_qa_split: isQaSplitSupported.value ? chunkParams.value.use_qa_split : false, qa_separator: chunkParams.value.qa_separator, }; chunkConfigModalVisible.value = true; @@ -750,10 +747,7 @@ const handleChunkConfigSubmit = () => { chunkParams.value.chunk_overlap = tempChunkParams.value.chunk_overlap; // 只有支持QA分割的知识库类型才保存QA分割配置 if (isQaSplitSupported.value) { - chunkParams.value.use_qa_split = tempChunkParams.value.use_qa_split; chunkParams.value.qa_separator = tempChunkParams.value.qa_separator; - } else { - chunkParams.value.use_qa_split = false; } chunkConfigModalVisible.value = false; message.success('分块参数配置已更新'); diff --git a/web/src/components/MarkdownContentViewer.vue b/web/src/components/MarkdownContentViewer.vue index f6cca296..6114f5d2 100644 --- a/web/src/components/MarkdownContentViewer.vue +++ b/web/src/components/MarkdownContentViewer.vue @@ -114,7 +114,7 @@ const currentChunk = ref(null); const tooltipStyle = ref({ top: '0px', left: '0px' }); const activeChunkIndex = ref(null); const highlightedChunkIndex = ref(null); -const chunkPanelVisible = ref(true); +const chunkPanelVisible = ref(false); // 主题设置 - 根据系统主题动态切换 const theme = computed(() => themeStore.isDark ? 'dark' : 'light');