from __future__ import annotations from dataclasses import dataclass from enum import StrEnum class ChunkMode(StrEnum): LENGTH = "length" NEWLINE = "newline" @dataclass class TextChunk: text: str index: int is_last: bool def convert_table_to_bullets(text: str) -> str: lines = text.strip().split("\n") if len(lines) < 2: return text if "|" not in lines[0] or "|" not in lines[1]: return text header_line = lines[0] if "---" not in lines[1]: return text result_lines: list[str] = [] headers = [h.strip() for h in header_line.split("|") if h.strip()] for line in lines[2:]: if "|" not in line: continue cells = [c.strip() for c in line.split("|") if c.strip()] parts = [f"*{headers[i]}:* {cells[i] if i < len(cells) else 'N/A'}" for i in range(len(headers))] result_lines.append(" • " + " | ".join(parts)) return "\n".join(result_lines) if result_lines else text def resolve_text_chunks( text: str, chunk_limit: int, *, mode: ChunkMode = ChunkMode.LENGTH, convert_table: bool = True, ) -> list[TextChunk]: if not text: return [] processed = text if convert_table: processed = convert_table_to_bullets(processed) if len(processed) <= chunk_limit: return [TextChunk(text=processed, index=0, is_last=True)] if mode == ChunkMode.NEWLINE: return _chunk_by_newline(processed, chunk_limit) return _chunk_by_length(processed, chunk_limit) def _chunk_by_length(text: str, limit: int) -> list[TextChunk]: chunks: list[TextChunk] = [] remaining = text idx = 0 while remaining: if len(remaining) <= limit: chunks.append(TextChunk(text=remaining, index=idx, is_last=True)) break split_point = remaining.rfind("\n", 0, limit) if split_point < limit // 2: split_point = remaining.rfind(" ", 0, limit) if split_point < limit // 2: split_point = limit chunk_text = remaining[:split_point] remaining = remaining[split_point:].lstrip("\n ") is_last = not remaining chunks.append(TextChunk(text=chunk_text, index=idx, is_last=is_last)) idx += 1 if chunks and len(chunks) > 1: chunks[-1].is_last = True return chunks def _chunk_by_newline(text: str, limit: int) -> list[TextChunk]: lines = text.split("\n") chunks: list[TextChunk] = [] current_lines: list[str] = [] current_len = 0 idx = 0 for line in lines: line_len = len(line) + 1 if current_lines and current_len + line_len > limit: chunks.append(TextChunk(text="\n".join(current_lines), index=idx, is_last=False)) idx += 1 current_lines = [] current_len = 0 if line_len > limit: if current_lines: chunks.append(TextChunk(text="\n".join(current_lines), index=idx, is_last=False)) idx += 1 current_lines = [] current_len = 0 sub_chunks = _chunk_by_length(line, limit) for sc in sub_chunks: chunks.append(TextChunk(text=sc.text, index=idx, is_last=False)) idx += 1 continue current_lines.append(line) current_len += line_len if current_lines: chunks.append(TextChunk(text="\n".join(current_lines), index=idx, is_last=True)) elif chunks: chunks[-1].is_last = True return chunks DEFAULT_CHUNK_LIMIT = 8000 DEFAULT_CHUNK_MODE = ChunkMode.LENGTH