kris/ForcePilot
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix: 保留文本块原始空格并优化块合并逻辑

Browse Source 
修复kb_utils.py中文本块内容被意外去除空格的问题
优化chunkUtils.js中块合并逻辑，当无重叠时添加换行符
This commit is contained in:
Wenjie Zhang 2025-11-30 17:00:42 +08:00
parent 103b3affee
commit 13d31cfb37
2 changed files with 7 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/knowledge/utils/kb_utils.py
View File

@ -90,7 +90,7 @@ def split_text_into_chunks(text: str, file_id: str, filename: str, params: dict
chunks.append( chunks.append(
{ {
"id": f"{file_id}_chunk_{chunk_index}", "id": f"{file_id}_chunk_{chunk_index}",
"content": chunk_content.strip(), "content": chunk_content, # .strip(),
"file_id": file_id, "file_id": file_id,
"filename": filename, "filename": filename,
"chunk_index": chunk_index, "chunk_index": chunk_index,

8
web/src/utils/chunkUtils.js
View File

@ -16,7 +16,7 @@ export function findOverlap(str1, str2) {
let overlap = ''; let overlap = '';
// 从最长可能的重叠开始检查 // 从最长可能的重叠开始检查
for (let i = maxOverlap; i > 0; i--) { for (let i = maxOverlap; i > 10; i--) {
const endStr1 = str1.slice(-i); const endStr1 = str1.slice(-i);
const startStr2 = str2.slice(0, i); const startStr2 = str2.slice(0, i);
@ -63,7 +63,11 @@ export function mergeChunks(chunks) {
if (newContent.length > 0) { if (newContent.length > 0) {
const startOffset = currentContent.length; const startOffset = currentContent.length;
currentContent += newContent; if (overlap.length > 0) {
currentContent += newContent;
} else {
currentContent += `\n${newContent}`;
}
merged.push({ merged.push({
...chunk, ...chunk,
startOffset, startOffset,