124 lines
3.0 KiB
Python
124 lines
3.0 KiB
Python
|
from __future__ import annotations
|
||
|
|
|
||
|
|
|
||
|
|
def split_utf8(text: str, byte_limit: int) -> list[str]:
|
||
|
|
if byte_limit <= 0:
|
||
|
|
return [text]
|
||
|
|
|
||
|
|
if not text:
|
||
|
|
return [text]
|
||
|
|
|
||
|
|
chunks: list[str] = []
|
||
|
|
remaining = text
|
||
|
|
|
||
|
|
while remaining:
|
||
|
|
encoded = remaining.encode("utf-8")
|
||
|
|
if len(encoded) <= byte_limit:
|
||
|
|
chunks.append(remaining)
|
||
|
|
break
|
||
|
|
|
||
|
|
cut = _find_cut_point(encoded, byte_limit)
|
||
|
|
chunk_bytes = encoded[:cut]
|
||
|
|
chunk = chunk_bytes.decode("utf-8", errors="strict")
|
||
|
|
chunks.append(chunk)
|
||
|
|
remaining = remaining[len(chunk) :]
|
||
|
|
|
||
|
|
return chunks
|
||
|
|
|
||
|
|
|
||
|
|
def _utf8_char_len(byte: int) -> int:
|
||
|
|
if byte & 0x80 == 0:
|
||
|
|
return 1
|
||
|
|
if byte & 0xE0 == 0xC0:
|
||
|
|
return 2
|
||
|
|
if byte & 0xF0 == 0xE0:
|
||
|
|
return 3
|
||
|
|
if byte & 0xF8 == 0xF0:
|
||
|
|
return 4
|
||
|
|
return 1
|
||
|
|
|
||
|
|
|
||
|
|
def _find_cut_point(encoded: bytes, byte_limit: int) -> int:
|
||
|
|
cut = byte_limit
|
||
|
|
|
||
|
|
while cut > 0 and (encoded[cut - 1] & 0xC0) == 0x80:
|
||
|
|
cut -= 1
|
||
|
|
if cut == 0:
|
||
|
|
cut = max(1, byte_limit)
|
||
|
|
|
||
|
|
if cut > 0 and encoded[cut - 1] & 0x80:
|
||
|
|
char_len = _utf8_char_len(encoded[cut - 1])
|
||
|
|
if cut - 1 + char_len > byte_limit:
|
||
|
|
cut -= 1
|
||
|
|
|
||
|
|
return max(1, cut)
|
||
|
|
|
||
|
|
|
||
|
|
def calc_effective_limit(
|
||
|
|
target: str,
|
||
|
|
nick: str = "",
|
||
|
|
username: str = "",
|
||
|
|
server: str = "",
|
||
|
|
) -> int:
|
||
|
|
prefix = f":{nick}!{username}@{server} PRIVMSG {target} :"
|
||
|
|
overhead = len(prefix.encode("utf-8")) + 2
|
||
|
|
return max(64, 510 - overhead)
|
||
|
|
|
||
|
|
|
||
|
|
def split_markdown(text: str, byte_limit: int) -> list[str]:
|
||
|
|
if byte_limit <= 0:
|
||
|
|
return [text]
|
||
|
|
if not text:
|
||
|
|
return [text]
|
||
|
|
|
||
|
|
chunks: list[str] = []
|
||
|
|
paragraphs = _split_markdown_paragraphs(text)
|
||
|
|
|
||
|
|
for para in paragraphs:
|
||
|
|
if len(para.encode("utf-8")) <= byte_limit:
|
||
|
|
chunks.append(para)
|
||
|
|
else:
|
||
|
|
chunks.extend(_split_paragraph_on_boundaries(para, byte_limit))
|
||
|
|
|
||
|
|
return chunks
|
||
|
|
|
||
|
|
|
||
|
|
def _split_markdown_paragraphs(text: str) -> list[str]:
|
||
|
|
raw_paras = text.split("\n\n")
|
||
|
|
result: list[str] = []
|
||
|
|
for para in raw_paras:
|
||
|
|
stripped = para.strip()
|
||
|
|
if stripped:
|
||
|
|
result.append(stripped)
|
||
|
|
return result or [text]
|
||
|
|
|
||
|
|
|
||
|
|
def _split_paragraph_on_boundaries(text: str, byte_limit: int) -> list[str]:
|
||
|
|
chunks: list[str] = []
|
||
|
|
words = text.split(" ")
|
||
|
|
current = ""
|
||
|
|
|
||
|
|
for word in words:
|
||
|
|
candidate = f"{current} {word}" if current else word
|
||
|
|
if len(candidate.encode("utf-8")) <= byte_limit:
|
||
|
|
current = candidate
|
||
|
|
else:
|
||
|
|
if current:
|
||
|
|
chunks.append(current)
|
||
|
|
if len(word.encode("utf-8")) > byte_limit:
|
||
|
|
chunks.extend(split_utf8(word, byte_limit))
|
||
|
|
current = ""
|
||
|
|
else:
|
||
|
|
current = word
|
||
|
|
|
||
|
|
if current:
|
||
|
|
chunks.append(current)
|
||
|
|
return chunks
|
||
|
|
|
||
|
|
|
||
|
|
def split_irc_text(text: str, max_chars: int = 350) -> list[str]:
|
||
|
|
if max_chars <= 0 or not text:
|
||
|
|
return [text]
|
||
|
|
byte_limit = min(max_chars * 4, 1600)
|
||
|
|
return split_utf8(text, byte_limit)
|