35 lines
1.2 KiB
Python
35 lines
1.2 KiB
Python
|
import re
|
||
|
|
|
||
|
|
|
||
|
|
def remove_markdown(text: str) -> str:
|
||
|
|
text = re.sub(r"\*\*(.+?)\*\*", r"\1", text)
|
||
|
|
text = re.sub(r"\*(.+?)\*", r"\1", text)
|
||
|
|
text = re.sub(r"__(.+?)__", r"\1", text)
|
||
|
|
text = re.sub(r"_(.+?)_", r"\1", text)
|
||
|
|
text = re.sub(r"~~(.+?)~~", r"\1", text)
|
||
|
|
text = re.sub(r"```[\s\S]*?```", "", text)
|
||
|
|
text = re.sub(r"`{1,3}[^`]*`{1,3}", "", text)
|
||
|
|
text = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", text)
|
||
|
|
text = re.sub(r"!\[.*?\]\(.*?\)", "[图片]", text)
|
||
|
|
text = re.sub(r"^#+ (.*?)$", r"\1", text, flags=re.MULTILINE)
|
||
|
|
text = re.sub(r"^[*-] (.*?)$", r"· \1", text, flags=re.MULTILINE)
|
||
|
|
text = re.sub(r"^> (.*?)$", r" \1", text, flags=re.MULTILINE)
|
||
|
|
text = re.sub(r"\n{3,}", "\n\n", text)
|
||
|
|
return text.strip()
|
||
|
|
|
||
|
|
|
||
|
|
def split_utf8_safe(text: str, max_len: int) -> list[str]:
|
||
|
|
if len(text) <= max_len:
|
||
|
|
return [text]
|
||
|
|
chunks = []
|
||
|
|
while text:
|
||
|
|
if len(text) <= max_len:
|
||
|
|
chunks.append(text)
|
||
|
|
break
|
||
|
|
split_at = text.rfind("\n", 0, max_len)
|
||
|
|
if split_at == -1 or split_at < max_len // 2:
|
||
|
|
split_at = max_len
|
||
|
|
chunks.append(text[:split_at])
|
||
|
|
text = text[split_at:].lstrip()
|
||
|
|
return chunks
|