75 lines
2.4 KiB
Python
75 lines
2.4 KiB
Python
|
from __future__ import annotations
|
|||
|
|
|
|||
|
|
import asyncio
|
|||
|
|
import time
|
|||
|
|
|
|||
|
|
|
|||
|
|
class BilibiliBlockChunker:
|
|||
|
|
def __init__(
|
|||
|
|
self,
|
|||
|
|
outbound,
|
|||
|
|
target_id: str,
|
|||
|
|
anti_risk,
|
|||
|
|
chunk_min_chars: int = 200,
|
|||
|
|
chunk_max_chars: int = 1200,
|
|||
|
|
coalesce_min_chars: int = 80,
|
|||
|
|
coalesce_max_chars: int = 400,
|
|||
|
|
coalesce_idle_ms: int = 500,
|
|||
|
|
):
|
|||
|
|
self._outbound = outbound
|
|||
|
|
self._target_id = target_id
|
|||
|
|
self._anti_risk = anti_risk
|
|||
|
|
self._chunk_min = chunk_min_chars
|
|||
|
|
self._chunk_max = chunk_max_chars
|
|||
|
|
self._coalesce_min = coalesce_min_chars
|
|||
|
|
self._coalesce_max = coalesce_max_chars
|
|||
|
|
self._coalesce_idle = coalesce_idle_ms / 1000.0
|
|||
|
|
self._buffer = ""
|
|||
|
|
self._last_flush = time.time()
|
|||
|
|
|
|||
|
|
async def append(self, text: str) -> None:
|
|||
|
|
self._buffer += text
|
|||
|
|
should_flush = len(self._buffer) >= self._chunk_max or (
|
|||
|
|
len(self._buffer) >= self._chunk_min and time.time() - self._last_flush > self._coalesce_idle
|
|||
|
|
)
|
|||
|
|
if should_flush:
|
|||
|
|
await self._flush()
|
|||
|
|
|
|||
|
|
async def finalize(self) -> None:
|
|||
|
|
if self._buffer:
|
|||
|
|
await self._flush()
|
|||
|
|
|
|||
|
|
async def _flush(self) -> None:
|
|||
|
|
chunks = self._split_chunks(self._buffer, self._chunk_max)
|
|||
|
|
for chunk in chunks:
|
|||
|
|
await self._outbound.send_text(to=self._target_id, text=chunk)
|
|||
|
|
await asyncio.sleep(0.5)
|
|||
|
|
self._buffer = ""
|
|||
|
|
self._last_flush = time.time()
|
|||
|
|
|
|||
|
|
def _split_chunks(self, text: str, limit: int) -> list[str]:
|
|||
|
|
if len(text) <= limit:
|
|||
|
|
return [text] if text else []
|
|||
|
|
|
|||
|
|
chunks = []
|
|||
|
|
remaining = text
|
|||
|
|
while len(remaining) > limit:
|
|||
|
|
window = remaining[:limit]
|
|||
|
|
last_para = window.rfind("\n\n")
|
|||
|
|
if last_para > limit * 0.5:
|
|||
|
|
chunks.append(window[:last_para].strip())
|
|||
|
|
remaining = remaining[last_para + 2 :]
|
|||
|
|
continue
|
|||
|
|
for sep in ["。\n", "。", "！", "？", "\n", " "]:
|
|||
|
|
idx = window.rfind(sep)
|
|||
|
|
if idx > limit * 0.3:
|
|||
|
|
chunks.append(window[: idx + len(sep)].strip())
|
|||
|
|
remaining = remaining[idx + len(sep) :]
|
|||
|
|
break
|
|||
|
|
else:
|
|||
|
|
chunks.append(window.strip())
|
|||
|
|
remaining = remaining[limit:]
|
|||
|
|
if remaining.strip():
|
|||
|
|
chunks.append(remaining.strip())
|
|||
|
|
return chunks
|