kris/ForcePilot
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix: 修复分块硬切句末标点归属

Browse Source
This commit is contained in:
Wenjie Zhang 2026-05-17 01:18:03 +08:00
parent 77ed3b4a4d
commit f29ad639e8
1 changed files with 6 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
backend/package/yuxi/knowledge/chunking/ragflow_like/nlp.py
View File

@ -68,13 +68,16 @@ def hard_split_by_token_limit(text: str, chunk_token_num: int) -> list[str]:
max_tokens = max(int(chunk_token_num or 0), 1) max_tokens = max(int(chunk_token_num or 0), 1)
while index < len(token_iter): while index < len(token_iter):
end_index = min(index + max_tokens, len(token_iter)) - 1 next_index = min(index + max_tokens, len(token_iter))
end = token_iter[end_index].end() if next_index < len(token_iter):
end = token_iter[next_index].start()
else:
end = len(text)
piece = text[start:end].strip() piece = text[start:end].strip()
if piece: if piece:
chunks.append(piece) chunks.append(piece)
start = end start = end
index = end_index + 1 index = next_index
tail = text[start:].strip() tail = text[start:].strip()
if tail: if tail: