fix: 保留文本块原始空格并优化块合并逻辑
修复kb_utils.py中文本块内容被意外去除空格的问题 优化chunkUtils.js中块合并逻辑,当无重叠时添加换行符
This commit is contained in:
parent
103b3affee
commit
13d31cfb37
@ -90,7 +90,7 @@ def split_text_into_chunks(text: str, file_id: str, filename: str, params: dict
|
||||
chunks.append(
|
||||
{
|
||||
"id": f"{file_id}_chunk_{chunk_index}",
|
||||
"content": chunk_content.strip(),
|
||||
"content": chunk_content, # .strip(),
|
||||
"file_id": file_id,
|
||||
"filename": filename,
|
||||
"chunk_index": chunk_index,
|
||||
|
||||
@ -16,7 +16,7 @@ export function findOverlap(str1, str2) {
|
||||
let overlap = '';
|
||||
|
||||
// 从最长可能的重叠开始检查
|
||||
for (let i = maxOverlap; i > 0; i--) {
|
||||
for (let i = maxOverlap; i > 10; i--) {
|
||||
const endStr1 = str1.slice(-i);
|
||||
const startStr2 = str2.slice(0, i);
|
||||
|
||||
@ -63,7 +63,11 @@ export function mergeChunks(chunks) {
|
||||
|
||||
if (newContent.length > 0) {
|
||||
const startOffset = currentContent.length;
|
||||
currentContent += newContent;
|
||||
if (overlap.length > 0) {
|
||||
currentContent += newContent;
|
||||
} else {
|
||||
currentContent += `\n${newContent}`;
|
||||
}
|
||||
merged.push({
|
||||
...chunk,
|
||||
startOffset,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user