61 lines
1.6 KiB
Python
61 lines
1.6 KiB
Python
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import re
|
||
|
|
|
||
|
|
ZULIP_MAX_MESSAGE_LENGTH = 10000
|
||
|
|
|
||
|
|
|
||
|
|
def truncate_zulip_content(content: str, max_length: int = ZULIP_MAX_MESSAGE_LENGTH) -> str:
|
||
|
|
if len(content) <= max_length:
|
||
|
|
return content
|
||
|
|
suffix = "\n\n... *(content truncated)*"
|
||
|
|
return content[: max_length - len(suffix)] + suffix
|
||
|
|
|
||
|
|
|
||
|
|
def clean_mentions(content: str, bot_name: str | None = None) -> str:
|
||
|
|
result = content
|
||
|
|
if bot_name:
|
||
|
|
result = re.sub(rf"@(?:(?:\*\*)|(__)){re.escape(bot_name)}(?:(?:\*\*)|(__))", "", result)
|
||
|
|
return result.strip()
|
||
|
|
|
||
|
|
|
||
|
|
def safe_split_zulip_content(text: str, chunk_limit: int = ZULIP_MAX_MESSAGE_LENGTH) -> list[str]:
|
||
|
|
if len(text) <= chunk_limit:
|
||
|
|
return [text]
|
||
|
|
|
||
|
|
chunks: list[str] = []
|
||
|
|
remaining = text
|
||
|
|
while len(remaining) > chunk_limit:
|
||
|
|
split_at = _find_paragraph_break(remaining, chunk_limit)
|
||
|
|
chunks.append(remaining[:split_at].strip())
|
||
|
|
remaining = remaining[split_at:].strip()
|
||
|
|
if remaining:
|
||
|
|
chunks.append(remaining)
|
||
|
|
return chunks
|
||
|
|
|
||
|
|
|
||
|
|
def sanitize_zulip_text(text: str) -> str:
|
||
|
|
return text
|
||
|
|
|
||
|
|
|
||
|
|
def zulip_markdown_to_native(md_text: str) -> str:
|
||
|
|
return md_text
|
||
|
|
|
||
|
|
|
||
|
|
def zulip_native_to_markdown(native_content: str) -> str:
|
||
|
|
return native_content
|
||
|
|
|
||
|
|
|
||
|
|
def _find_paragraph_break(text: str, limit: int) -> int:
|
||
|
|
search_text = text[:limit]
|
||
|
|
for sep in ["\n\n", "\n"]:
|
||
|
|
idx = search_text.rfind(sep)
|
||
|
|
if idx > limit * 0.3:
|
||
|
|
return idx + len(sep)
|
||
|
|
|
||
|
|
last_space = search_text.rfind(" ")
|
||
|
|
if last_space > limit * 0.7:
|
||
|
|
return last_space + 1
|
||
|
|
|
||
|
|
return limit
|