refactor(kb): 移除QA分割功能并改进分隔符处理

移除不再使用的QA分割功能及相关代码，简化文本分块逻辑
添加分隔符转义处理以支持前端传入的转义字符
改进分块参数配置界面的描述和默认值

src/knowledge/implementations/chroma.py

@@ -14,7 +14,6 @@ from src.knowledge.utils.kb_utils import (
     get_embedding_config,
     prepare_item_metadata,
     split_text_into_chunks,
-    split_text_into_qa_chunks,
 )
 from src.utils import logger
 from src.utils.datetime_utils import utc_isoformat
@@ -151,16 +150,7 @@ class ChromaKB(KnowledgeBase):
 
     def _split_text_into_chunks(self, text: str, file_id: str, filename: str, params: dict) -> list[dict]:
         """将文本分割成块"""
-        # 检查是否使用QA分割模式
+        chunks = split_text_into_chunks(text, file_id, filename, params)
-        use_qa_split = params.get("use_qa_split", False)
-
-        if use_qa_split:
-            # 使用QA分割模式
-            qa_separator = params.get("qa_separator", "\n\n\n")
-            chunks = split_text_into_qa_chunks(text, file_id, filename, qa_separator, params)
-        else:
-            # 使用传统分割模式
-            chunks = split_text_into_chunks(text, file_id, filename, params)
 
         # 为 ChromaDB 添加特定的 metadata 格式
         for chunk in chunks:

src/knowledge/implementations/milvus.py

@@ -14,7 +14,6 @@ from src.knowledge.utils.kb_utils import (
     get_embedding_config,
     prepare_item_metadata,
     split_text_into_chunks,
-    split_text_into_qa_chunks,
 )
 from src.models.embed import OtherEmbedding
 from src.utils import hashstr, logger
@@ -222,16 +221,7 @@ class MilvusKB(KnowledgeBase):
 
     def _split_text_into_chunks(self, text: str, file_id: str, filename: str, params: dict) -> list[dict]:
         """将文本分割成块"""
-        # 检查是否使用QA分割模式
+        return split_text_into_chunks(text, file_id, filename, params)
-        use_qa_split = params.get("use_qa_split", False)
-
-        if use_qa_split:
-            # 使用QA分割模式
-            qa_separator = params.get("qa_separator", "\n\n\n")
-            return split_text_into_qa_chunks(text, file_id, filename, qa_separator, params)
-        else:
-            # 使用传统分割模式
-            return split_text_into_chunks(text, file_id, filename, params)
 
     async def add_content(self, db_id: str, items: list[str], params: dict | None = {}) -> list[dict]:
         """添加内容（文件/URL）"""

src/knowledge/utils/__init__.py

@@ -11,7 +11,6 @@ from .kb_utils import (
     merge_processing_params,
     prepare_item_metadata,
     split_text_into_chunks,
-    split_text_into_qa_chunks,
     validate_file_path,
 )
 
@@ -20,7 +19,6 @@ __all__ = [
     "get_embedding_config",
     "prepare_item_metadata",
     "split_text_into_chunks",
-    "split_text_into_qa_chunks",
     "merge_processing_params",
     "validate_file_path",
 ]

src/knowledge/utils/kb_utils.py

@@ -74,6 +74,23 @@ def validate_file_path(file_path: str, db_id: str = None) -> str:
         raise ValueError(f"Invalid file path: {file_path}")
 
 
+def _unescape_separator(separator: str | None) -> str | None:
+    """将前端传入的字面量转义字符转换为实际字符
+
+    例如: "\\n\\n\\n" -> "\n\n\n"
+    """
+    if not separator:
+        return None
+
+    # 处理常见的转义序列
+    separator = separator.replace("\\n", "\n")
+    separator = separator.replace("\\r", "\r")
+    separator = separator.replace("\\t", "\t")
+    separator = separator.replace("\\\\", "\\")
+
+    return separator
+
+
 def split_text_into_chunks(text: str, file_id: str, filename: str, params: dict = {}) -> list[dict]:
     """
     将文本分割成块，使用 LangChain 的 MarkdownTextSplitter 进行智能分割
@@ -82,6 +99,16 @@ def split_text_into_chunks(text: str, file_id: str, filename: str, params: dict
     chunk_size = params.get("chunk_size", 1000)
     chunk_overlap = params.get("chunk_overlap", 200)
 
+    # 获取分隔符并转换为实际字符
+    separator = params.get("qa_separator")
+    separator = _unescape_separator(separator)
+
+    # 向后兼容：如果旧配置设置了 use_qa_split=True 但未指定 separator，使用默认分隔符
+    use_qa_split = params.get("use_qa_split", False)
+    if use_qa_split and not separator:
+        separator = "\n\n\n"
+        logger.debug("启用了向后兼容模式：use_qa_split=True，使用默认分隔符 \\n\\n\\n")
+
     # 使用 MarkdownTextSplitter 进行智能分割
     # MarkdownTextSplitter 会尝试沿着 Markdown 格式的标题进行分割
     text_splitter = MarkdownTextSplitter(
@@ -89,7 +116,18 @@ def split_text_into_chunks(text: str, file_id: str, filename: str, params: dict
         chunk_overlap=chunk_overlap,
     )
 
-    text_chunks = text_splitter.split_text(text)
+    # 如果设置了分隔符，先分割后以当前的分割逻辑处理
+    if separator:
+        # 转换分隔符为可视格式（换行符显示为 \n）
+        separator_display = separator.replace('\n', '\\n').replace('\r', '\\r').replace('\t', '\\t')
+        logger.debug(f"启用预分割模式，使用分隔符: '{separator_display}'")
+        pre_chunks = text.split(separator)
+        text_chunks = []
+        for pre_chunk in pre_chunks:
+            if pre_chunk.strip():
+                text_chunks.extend(text_splitter.split_text(pre_chunk))
+    else:
+        text_chunks = text_splitter.split_text(text)
 
     # 转换为标准格式
     for chunk_index, chunk_content in enumerate(text_chunks):
@@ -136,7 +174,8 @@ async def calculate_content_hash(data: bytes | bytearray | str | os.PathLike[str
 
         return sha256.hexdigest()
 
-    raise TypeError(f"Unsupported data type for hashing: {type(data)!r}")
+    # 理论上不会执行到这里，但保留作为防御性编程
+    raise TypeError(f"Unsupported data type for hashing: {type(data)!r}")  # type: ignore[unreachable]
 
 
 async def prepare_item_metadata(item: str, content_type: str, db_id: str, params: dict | None = None) -> dict:
@@ -223,39 +262,6 @@ async def prepare_item_metadata(item: str, content_type: str, db_id: str, params
     return metadata
 
 
-def split_text_into_qa_chunks(
-    text: str, file_id: str, filename: str, qa_separator: None | str = None, params: dict = {}
-) -> list[dict]:
-    """
-    将文本按QA对分割成块，使用 LangChain 的 CharacterTextSplitter 进行分割"""
-    qa_separator = qa_separator or "\n\n"
-    text_chunks = text.split(qa_separator)
-
-    # 转换为标准格式
-    chunks = []
-    for chunk_index, chunk_content in enumerate(text_chunks):
-        if chunk_content.strip():  # 跳过空块
-            chunk_content = chunk_content.strip()[:4096]
-            chunks.append(
-                {
-                    "id": f"{file_id}_qa_chunk_{chunk_index}",
-                    "content": chunk_content.strip(),
-                    "file_id": file_id,
-                    "filename": filename,
-                    "chunk_index": chunk_index,
-                    "source": filename,
-                    "chunk_id": f"{file_id}_qa_chunk_{chunk_index}",
-                    "chunk_type": "qa",  # 标识为QA类型的chunk
-                }
-            )
-
-    logger.debug(f"QA chunks: {chunks[0]}")
-    logger.debug(
-        f"Successfully split QA text into {len(chunks)} chunks using CharacterTextSplitter with `{qa_separator=}`"
-    )
-    return chunks
-
-
 def merge_processing_params(metadata_params: dict | None, request_params: dict | None) -> dict:
     """
     合并处理参数：优先使用请求参数，缺失时使用元数据中的参数

web/src/components/ChunkParamsConfig.vue

@@ -17,13 +17,13 @@
           <a-input-number v-model:value="tempChunkParams.chunk_overlap" :min="0" :max="1000" style="width: 100%;" />
           <p class="param-description">相邻文本片段间的重叠字符数</p>
         </a-form-item>
-        <a-form-item v-if="showQaSplit" label="QA分割模式" name="use_qa_split">
+        <a-form-item v-if="showQaSplit" label="分隔符 (Separator)" name="qa_separator">
-          <a-switch v-model:checked="tempChunkParams.use_qa_split" />
+          <a-input
-          <p class="param-description">启用后将按QA对分割，忽略上述chunk大小设置</p>
+            v-model:value="tempChunkParams.qa_separator"
-        </a-form-item>
+            placeholder="输入分隔符，例如 \n\n\n 或 ---"
-        <a-form-item v-if="tempChunkParams?.use_qa_split && showQaSplit" label="QA分隔符" name="qa_separator">
+            style="width: 100%;"
-          <a-input v-model:value="tempChunkParams.qa_separator" placeholder="输入QA分隔符" style="width: 100%;" />
+          />
-          <p class="param-description">用于分割不同QA对的分隔符</p>
+          <p class="param-description">支持 \n、\t 等转义字符。留空则不启用预分割</p>
         </a-form-item>
       </a-form>
   </div>

web/src/components/FileTable.vue

@@ -379,8 +379,7 @@ const rechunkModalLoading = computed(() => store.state.chunkLoading);
 const rechunkParams = ref({
   chunk_size: 1000,
   chunk_overlap: 200,
-  use_qa_split: false,
+  qa_separator: '\\n\\n\\n'
-  qa_separator: '\n\n\n'
 });
 const currentRechunkFileIds = ref([]);
 const isBatchRechunk = ref(false);
@@ -788,8 +787,7 @@ const handleRechunkConfirm = async () => {
       rechunkParams.value = {
         chunk_size: 1000,
         chunk_overlap: 200,
-        use_qa_split: false,
+        qa_separator: '\\n\\n\\n'
-        qa_separator: '\n\n\n'
       };
     } else {
       message.error(`重新分块失败: ${result.message}`);
@@ -811,8 +809,7 @@ const handleRechunkCancel = () => {
   rechunkParams.value = {
     chunk_size: 1000,
     chunk_overlap: 200,
-    use_qa_split: false,
+    qa_separator: '\\n\\n\\n'
-    qa_separator: '\n\n\n'
   };
 };
 

web/src/components/FileUploadModal.vue

@@ -365,8 +365,7 @@ const chunkParams = ref({
   chunk_size: 1000,
   chunk_overlap: 200,
   enable_ocr: 'disable',
-  use_qa_split: false,
+  qa_separator: '',
-  qa_separator: '\n\n\n',
 });
 
 // 分块参数配置弹窗
@@ -376,8 +375,7 @@ const chunkConfigModalVisible = ref(false);
 const tempChunkParams = ref({
   chunk_size: 1000,
   chunk_overlap: 200,
-  use_qa_split: false,
+  qa_separator: '',
-  qa_separator: '\n\n\n',
 });
 
 // 计算属性：是否支持QA分割
@@ -739,7 +737,6 @@ const showChunkConfigModal = () => {
   tempChunkParams.value = {
     chunk_size: chunkParams.value.chunk_size,
     chunk_overlap: chunkParams.value.chunk_overlap,
-    use_qa_split: isQaSplitSupported.value ? chunkParams.value.use_qa_split : false,
     qa_separator: chunkParams.value.qa_separator,
   };
   chunkConfigModalVisible.value = true;
@@ -750,10 +747,7 @@ const handleChunkConfigSubmit = () => {
   chunkParams.value.chunk_overlap = tempChunkParams.value.chunk_overlap;
   // 只有支持QA分割的知识库类型才保存QA分割配置
   if (isQaSplitSupported.value) {
-    chunkParams.value.use_qa_split = tempChunkParams.value.use_qa_split;
     chunkParams.value.qa_separator = tempChunkParams.value.qa_separator;
-  } else {
-    chunkParams.value.use_qa_split = false;
   }
   chunkConfigModalVisible.value = false;
   message.success('分块参数配置已更新');

web/src/components/MarkdownContentViewer.vue

@@ -114,7 +114,7 @@ const currentChunk = ref(null);
 const tooltipStyle = ref({ top: '0px', left: '0px' });
 const activeChunkIndex = ref(null);
 const highlightedChunkIndex = ref(null);
-const chunkPanelVisible = ref(true);
+const chunkPanelVisible = ref(false);
 
 // 主题设置 - 根据系统主题动态切换
 const theme = computed(() => themeStore.isDark ? 'dark' : 'light');