fix: 保留文本块原始空格并优化块合并逻辑

修复kb_utils.py中文本块内容被意外去除空格的问题
优化chunkUtils.js中块合并逻辑,当无重叠时添加换行符
This commit is contained in:
Wenjie Zhang 2025-11-30 17:00:42 +08:00
parent 103b3affee
commit 13d31cfb37
2 changed files with 7 additions and 3 deletions

View File

@ -90,7 +90,7 @@ def split_text_into_chunks(text: str, file_id: str, filename: str, params: dict
chunks.append(
{
"id": f"{file_id}_chunk_{chunk_index}",
"content": chunk_content.strip(),
"content": chunk_content, # .strip(),
"file_id": file_id,
"filename": filename,
"chunk_index": chunk_index,

View File

@ -16,7 +16,7 @@ export function findOverlap(str1, str2) {
let overlap = '';
// 从最长可能的重叠开始检查
for (let i = maxOverlap; i > 0; i--) {
for (let i = maxOverlap; i > 10; i--) {
const endStr1 = str1.slice(-i);
const startStr2 = str2.slice(0, i);
@ -63,7 +63,11 @@ export function mergeChunks(chunks) {
if (newContent.length > 0) {
const startOffset = currentContent.length;
currentContent += newContent;
if (overlap.length > 0) {
currentContent += newContent;
} else {
currentContent += `\n${newContent}`;
}
merged.push({
...chunk,
startOffset,