diff --git a/src/knowledge/utils/kb_utils.py b/src/knowledge/utils/kb_utils.py index 676a9195..66645f10 100644 --- a/src/knowledge/utils/kb_utils.py +++ b/src/knowledge/utils/kb_utils.py @@ -90,7 +90,7 @@ def split_text_into_chunks(text: str, file_id: str, filename: str, params: dict chunks.append( { "id": f"{file_id}_chunk_{chunk_index}", - "content": chunk_content.strip(), + "content": chunk_content, # .strip(), "file_id": file_id, "filename": filename, "chunk_index": chunk_index, diff --git a/web/src/utils/chunkUtils.js b/web/src/utils/chunkUtils.js index 850f247d..a3ad5bc5 100644 --- a/web/src/utils/chunkUtils.js +++ b/web/src/utils/chunkUtils.js @@ -16,7 +16,7 @@ export function findOverlap(str1, str2) { let overlap = ''; // 从最长可能的重叠开始检查 - for (let i = maxOverlap; i > 0; i--) { + for (let i = maxOverlap; i > 10; i--) { const endStr1 = str1.slice(-i); const startStr2 = str2.slice(0, i); @@ -63,7 +63,11 @@ export function mergeChunks(chunks) { if (newContent.length > 0) { const startOffset = currentContent.length; - currentContent += newContent; + if (overlap.length > 0) { + currentContent += newContent; + } else { + currentContent += `\n${newContent}`; + } merged.push({ ...chunk, startOffset,