35 lines
927 B
Python
35 lines
927 B
Python
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import re
|
||
|
|
|
||
|
|
_CONTROL_CHAR_PATTERN = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]")
|
||
|
|
_MULTI_NEWLINE_PATTERN = re.compile(r"\n{3,}")
|
||
|
|
|
||
|
|
|
||
|
|
def sanitize_text(text: str) -> str:
|
||
|
|
cleaned = _CONTROL_CHAR_PATTERN.sub("", text)
|
||
|
|
cleaned = _MULTI_NEWLINE_PATTERN.sub("\n\n", cleaned)
|
||
|
|
return cleaned.strip()
|
||
|
|
|
||
|
|
|
||
|
|
def chunk_text(text: str, limit: int = 1000) -> list[str]:
|
||
|
|
if len(text) <= limit:
|
||
|
|
return [text]
|
||
|
|
|
||
|
|
chunks = []
|
||
|
|
remaining = text
|
||
|
|
while remaining:
|
||
|
|
if len(remaining) <= limit:
|
||
|
|
chunks.append(remaining)
|
||
|
|
break
|
||
|
|
|
||
|
|
split_at = remaining.rfind("\n", 0, limit)
|
||
|
|
if split_at == -1 or split_at < limit // 2:
|
||
|
|
split_at = remaining.rfind(" ", 0, limit)
|
||
|
|
if split_at == -1:
|
||
|
|
split_at = limit
|
||
|
|
|
||
|
|
chunks.append(remaining[:split_at])
|
||
|
|
remaining = remaining[split_at:].lstrip()
|
||
|
|
return chunks
|