from __future__ import annotations import re _SENTENCE_BOUNDARY = re.compile(r"[。!?.!?\n]") def chunk_text(text: str, limit: int = 20000) -> list[str]: if len(text) <= limit: return [text] chunks: list[str] = [] paragraphs = text.split("\n\n") current = "" for para in paragraphs: if len(current) + len(para) + 2 <= limit: current = f"{current}\n\n{para}" if current else para else: if current: chunks.append(current) if len(para) > limit: sub_chunks = _chunk_long_paragraph(para, limit) if sub_chunks: if sub_chunks[-1]: current = sub_chunks.pop() else: sub_chunks.pop() current = "" chunks.extend(sub_chunks) else: current = "" else: current = para if current: chunks.append(current) return chunks or [text[:limit]] def _chunk_long_paragraph(text: str, limit: int) -> list[str]: chunks: list[str] = [] while len(text) > limit: split_at = _find_split_point(text, limit) chunk = text[:split_at].rstrip() if chunk: chunks.append(chunk) text = text[split_at:].lstrip() if text: chunks.append(text) return chunks def _find_split_point(text: str, limit: int) -> int: candidates = [m.start() for m in _SENTENCE_BOUNDARY.finditer(text, limit // 2, limit)] if candidates: return candidates[-1] + 1 newline = text.rfind("\n", limit // 2, limit) if newline != -1: return newline + 1 space = text.rfind(" ", limit // 2, limit) if space != -1: return space + 1 return limit