from __future__ import annotations import re from dataclasses import dataclass from typing import Literal _SENTENCE_BOUNDARY = re.compile(r"[。!?.!?\n]") @dataclass class ChunkConfig: limit: int = 4096 mode: Literal["text", "markdown"] = "text" def chunk_message(content: str, config: ChunkConfig | None = None) -> list[str]: cfg = config or ChunkConfig() if len(content) <= cfg.limit: return [content] if cfg.mode == "markdown": return _chunk_markdown(content, cfg.limit) return _chunk_text(content, cfg.limit) def _chunk_text(text: str, limit: int) -> list[str]: chunks: list[str] = [] paragraphs = text.split("\n\n") current = "" for para in paragraphs: if len(current) + len(para) + 2 <= limit: current = f"{current}\n\n{para}" if current else para else: if current: chunks.append(current) if len(para) > limit: sub = _chunk_long_text(para, limit) if sub: if sub[-1]: current = sub.pop() else: sub.pop() current = "" chunks.extend(sub) else: current = "" else: current = para if current: chunks.append(current) return chunks or [text] def _chunk_long_text(text: str, limit: int) -> list[str]: chunks: list[str] = [] while len(text) > limit: split_at = _find_split_point(text, limit) chunks.append(text[:split_at].rstrip()) text = text[split_at:].lstrip() chunks.append(text) return chunks def _find_split_point(text: str, limit: int) -> int: candidates = [m.start() for m in _SENTENCE_BOUNDARY.finditer(text, limit // 2, limit)] if candidates: return candidates[-1] + 1 newline = text.rfind("\n", limit // 2, limit) if newline != -1: return newline + 1 space = text.rfind(" ", limit // 2, limit) if space != -1: return space + 1 return limit def _chunk_markdown(text: str, limit: int) -> list[str]: lines = text.split("\n") chunks: list[str] = [] current = "" in_code_block = False for line in lines: if line.strip().startswith("```"): if in_code_block: in_code_block = False else: if current and len(current) + len(line) + 1 > limit: chunks.append(current) current = line in_code_block = True continue in_code_block = True if current and len(current) + len(line) + 1 > limit and not in_code_block: chunks.append(current) current = line else: current = f"{current}\n{line}" if current else line if current: chunks.append(current) return chunks or [text]