from __future__ import annotations import re from collections.abc import Iterator from enum import StrEnum from dataclasses import dataclass class ChunkMode(StrEnum): LENGTH = "length" NEWLINE = "newline" SENTENCE = "sentence" MARKDOWN = "markdown" @dataclass class TextChunk: text: str index: int is_last: bool _NATURAL_BOUNDARIES = [ "\n\n", "\n", "。", "!", "?", ";", ":", ". ", "! ", "? ", ] _MD_FENCE = re.compile(r"^```") _MD_TABLE_SEP = re.compile(r"^\|[-:| ]+\|") _MD_ORDERED_LIST = re.compile(r"^\d+\.\s") _MD_UNORDERED_LIST = re.compile(r"^[-*+]\s") _MD_HEADING = re.compile(r"^#{1,6}\s") class TextChunker: def __init__(self, max_length: int = 2000, mode: ChunkMode = ChunkMode.NEWLINE): self.max_length = max_length self.mode: ChunkMode = mode def chunk(self, text: str) -> list[str]: if len(text) <= self.max_length: return [text] if text else [] if self.mode == ChunkMode.LENGTH: return [c for c in self._chunk_by_length(text) if c] if self.mode == ChunkMode.SENTENCE: return [c for c in self._chunk_by_sentence(text) if c] if self.mode == ChunkMode.MARKDOWN: return [c for c in self._chunk_markdown(text) if c] return [c for c in self._chunk_by_newline(text) if c] def chunk_with_index(self, text: str) -> Iterator[tuple[int, int, str]]: chunks = self.chunk(text) total = len(chunks) for i, chunk in enumerate(chunks): yield i + 1, total, chunk def chunk_as_objects(self, text: str) -> list[TextChunk]: chunks = self.chunk(text) return [TextChunk(text=c, index=i, is_last=(i == len(chunks) - 1)) for i, c in enumerate(chunks)] def _chunk_by_length(self, text: str) -> list[str]: chunks: list[str] = [] remaining = text while len(remaining) > self.max_length: split_at = self._find_split_point(remaining) chunks.append(remaining[:split_at].rstrip()) remaining = remaining[split_at:].lstrip() if remaining: chunks.append(remaining) return chunks def _chunk_by_newline(self, text: str) -> list[str]: paragraphs = re.split(r"\n{2,}", text) chunks: list[str] = [] for para in paragraphs: if len(para) <= self.max_length: if para: chunks.append(para) continue for sub in self._chunk_by_length(para): chunks.append(sub) return chunks def _chunk_by_sentence(self, text: str) -> list[str]: sentences = re.split(r"(?<=[。!?.!?])\s*", text) chunks: list[str] = [] buffer = "" for sentence in sentences: if not sentence.strip(): buffer += sentence continue candidate = buffer + sentence if len(candidate) <= self.max_length: buffer = candidate else: if buffer: chunks.append(buffer) if len(sentence) > self.max_length: for sub in self._chunk_by_length(sentence): chunks.append(sub) buffer = "" else: buffer = sentence if buffer: chunks.append(buffer) return chunks def _chunk_markdown(self, text: str) -> list[str]: segments = self._segment_markdown(text) chunks: list[str] = [] buffer = "" for seg in segments: candidate = buffer + ("\n" if buffer else "") + seg if buffer else seg if len(candidate) <= self.max_length: buffer = candidate else: if buffer: chunks.append(buffer) if len(seg) > self.max_length: for sub in self._chunk_overlong_md(seg): chunks.append(sub) buffer = "" else: buffer = seg if buffer: chunks.append(buffer) return chunks def _segment_markdown(self, text: str) -> list[str]: lines = text.split("\n") segments: list[str] = [] current: list[str] = [] in_fence = False in_table = False in_list = False def flush(): nonlocal current, in_table, in_list if current: segments.append("\n".join(current)) current = [] in_table = False in_list = False for line in lines: is_fence = bool(_MD_FENCE.match(line)) is_table_sep = bool(_MD_TABLE_SEP.match(line)) is_ordered = bool(_MD_ORDERED_LIST.match(line)) is_unordered = bool(_MD_UNORDERED_LIST.match(line)) is_heading = bool(_MD_HEADING.match(line)) if is_fence: if in_fence: current.append(line) flush() in_fence = False continue flush() in_fence = True current.append(line) continue if in_fence: current.append(line) continue if is_table_sep and current and "|" in current[-1]: flush() in_table = True current.append(line) continue if in_table and "|" in line: current.append(line) continue if in_table: flush() if is_ordered or is_unordered: if not in_list: flush() in_list = True current.append(line) continue if is_heading: flush() current.append(line) continue if in_list: flush() if line.strip() == "": flush() continue current.append(line) if len("\n".join(current)) > self.max_length * 2: flush() flush() return segments def _chunk_overlong_md(self, text: str) -> list[str]: lines = text.split("\n") chunks: list[str] = [] buffer = "" for line in lines: candidate = buffer + ("\n" if buffer else "") + line if buffer else line if len(candidate) <= self.max_length: buffer = candidate continue if buffer: chunks.append(buffer) buffer = "" if len(line) <= self.max_length: buffer = line else: for sub in self._chunk_by_length(line): if len(sub) <= self.max_length: if buffer: chunks.append(buffer) buffer = sub else: if buffer: chunks.append(buffer) buffer = "" chunks.append(sub) if buffer: chunks.append(buffer) return chunks def _find_split_point(self, text: str) -> int: limit = self.max_length slice_end = min(limit, len(text)) for sep in _NATURAL_BOUNDARIES: idx = text.rfind(sep, 0, slice_end) if idx > slice_end // 2: return idx + len(sep.rstrip()) space_idx = text.rfind(" ", 0, slice_end) if space_idx > slice_end // 2: return space_idx + 1 while slice_end > 0 and (ord(text[slice_end - 1]) & 0xC0) == 0x80: slice_end -= 1 return slice_end if slice_end > 0 else limit