From f29ad639e89ac84d6463e04d7e7b253ad91bd896 Mon Sep 17 00:00:00 2001 From: Wenjie Zhang Date: Sun, 17 May 2026 01:18:03 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E5=88=86=E5=9D=97?= =?UTF-8?q?=E7=A1=AC=E5=88=87=E5=8F=A5=E6=9C=AB=E6=A0=87=E7=82=B9=E5=BD=92?= =?UTF-8?q?=E5=B1=9E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../package/yuxi/knowledge/chunking/ragflow_like/nlp.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/backend/package/yuxi/knowledge/chunking/ragflow_like/nlp.py b/backend/package/yuxi/knowledge/chunking/ragflow_like/nlp.py index 58b0d3f3..32eb948f 100644 --- a/backend/package/yuxi/knowledge/chunking/ragflow_like/nlp.py +++ b/backend/package/yuxi/knowledge/chunking/ragflow_like/nlp.py @@ -68,13 +68,16 @@ def hard_split_by_token_limit(text: str, chunk_token_num: int) -> list[str]: max_tokens = max(int(chunk_token_num or 0), 1) while index < len(token_iter): - end_index = min(index + max_tokens, len(token_iter)) - 1 - end = token_iter[end_index].end() + next_index = min(index + max_tokens, len(token_iter)) + if next_index < len(token_iter): + end = token_iter[next_index].start() + else: + end = len(text) piece = text[start:end].strip() if piece: chunks.append(piece) start = end - index = end_index + 1 + index = next_index tail = text[start:].strip() if tail: