38 lines
1.3 KiB
Python
38 lines
1.3 KiB
Python
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import re
|
||
|
|
|
||
|
|
|
||
|
|
def chunk_text(text: str, limit: int = 1000) -> list[str]:
|
||
|
|
if len(text) <= limit:
|
||
|
|
return [text]
|
||
|
|
|
||
|
|
chunks = []
|
||
|
|
remaining = text
|
||
|
|
while remaining:
|
||
|
|
if len(remaining) <= limit:
|
||
|
|
chunks.append(remaining)
|
||
|
|
break
|
||
|
|
split_at = remaining.rfind("\n", 0, limit + 1)
|
||
|
|
if split_at == -1 or split_at < limit // 2:
|
||
|
|
split_at = remaining.rfind(". ", 0, limit + 1)
|
||
|
|
if split_at == -1 or split_at < limit // 2:
|
||
|
|
split_at = remaining.rfind(" ", 0, limit + 1)
|
||
|
|
if split_at == -1 or split_at < limit // 2:
|
||
|
|
split_at = limit
|
||
|
|
chunks.append(remaining[:split_at].strip())
|
||
|
|
remaining = remaining[split_at:].strip()
|
||
|
|
return chunks
|
||
|
|
|
||
|
|
|
||
|
|
def strip_markdown(text: str) -> str:
|
||
|
|
text = re.sub(r"\*\*(.+?)\*\*", r"\1", text)
|
||
|
|
text = re.sub(r"\*(.+?)\*", r"\1", text)
|
||
|
|
text = re.sub(r"`(.+?)`", r"\1", text)
|
||
|
|
text = re.sub(r"\[(.+?)\]\((.+?)\)", r"\1 (\2)", text)
|
||
|
|
text = re.sub(r"^#{1,6}\s+", "", text, flags=re.MULTILINE)
|
||
|
|
text = re.sub(r"^>\s+", "", text, flags=re.MULTILINE)
|
||
|
|
text = re.sub(r"^[\-\*\+]\s+", "• ", text, flags=re.MULTILINE)
|
||
|
|
text = re.sub(r"^(\d+)\.\s+", r"\1. ", text, flags=re.MULTILINE)
|
||
|
|
text = re.sub(r"~~(.+?)~~", r"\1", text)
|
||
|
|
return text.strip()
|