49 lines
1.5 KiB
Python
49 lines
1.5 KiB
Python
|
import re
|
||
|
|
|
||
|
|
MAX_UTF8_LEN = 2048
|
||
|
|
|
||
|
|
|
||
|
|
def remove_markdown(text: str) -> str:
|
||
|
|
text = re.sub(r"```\w*\n?", "", text)
|
||
|
|
text = re.sub(r"`([^`]+)`", r"\1", text)
|
||
|
|
text = re.sub(r"\*\*([^*]+)\*\*", r"\1", text)
|
||
|
|
text = re.sub(r"\*([^*]+)\*", r"\1", text)
|
||
|
|
text = re.sub(r"__([^_]+)__", r"\1", text)
|
||
|
|
text = re.sub(r"_([^_]+)_", r"\1", text)
|
||
|
|
text = re.sub(r"(?m)^#{1,6}\s+", "", text)
|
||
|
|
text = re.sub(r"(?m)^[-*_]{3,}\s*$", "", text)
|
||
|
|
return text.strip()
|
||
|
|
|
||
|
|
|
||
|
|
def split_utf8(text: str, max_bytes: int = MAX_UTF8_LEN) -> list[str]:
|
||
|
|
encoded = text.encode("utf-8")
|
||
|
|
if len(encoded) <= max_bytes:
|
||
|
|
return [text]
|
||
|
|
result: list[str] = []
|
||
|
|
remaining = encoded
|
||
|
|
while remaining:
|
||
|
|
chunk = remaining[:max_bytes]
|
||
|
|
for cut in range(len(chunk), 0, -1):
|
||
|
|
try:
|
||
|
|
result.append(chunk[:cut].decode("utf-8"))
|
||
|
|
break
|
||
|
|
except UnicodeDecodeError:
|
||
|
|
continue
|
||
|
|
remaining = remaining[cut:]
|
||
|
|
return result
|
||
|
|
|
||
|
|
|
||
|
|
def split_utf8_safe(text: str, max_bytes: int = MAX_UTF8_LEN) -> tuple[str, str | None]:
|
||
|
|
encoded = text.encode("utf-8")
|
||
|
|
if len(encoded) <= max_bytes:
|
||
|
|
return text, None
|
||
|
|
hint = "\n【未完待续，回复任意文字以继续】"
|
||
|
|
hint_bytes = len(hint.encode("utf-8"))
|
||
|
|
limit = max_bytes - hint_bytes
|
||
|
|
try:
|
||
|
|
cut_point = len(encoded[:limit].decode("utf-8", errors="ignore"))
|
||
|
|
except Exception:
|
||
|
|
cut_point = limit
|
||
|
|
first_part = text[:cut_point] + hint
|
||
|
|
remaining = text[cut_point:]
|
||
|
|
return first_part, remaining
|