Kris
eb25707668
新增元宝（Yuanbao）渠道的完整适配器实现，包含以下核心模块： - 基础适配器与导出入口 - 协议编解码与WebSocket帧处理 - 会话管理与路由逻辑 - 事件队列与出站消息队列 - 消息格式转换与发送重试 - 安全审计与权限校验 - 配置映射与账户管理 - 视觉分析与工具函数 - 文档生成与设置向导
65 lines
1.8 KiB
Python
65 lines
1.8 KiB
Python
from __future__ import annotations
|
|
|
|
import re
|
|
|
|
_SENTENCE_BOUNDARY = re.compile(r"[。！？.!?\n]")
|
|
|
|
|
|
def chunk_text(text: str, limit: int = 20000) -> list[str]:
|
|
if len(text) <= limit:
|
|
return [text]
|
|
|
|
chunks: list[str] = []
|
|
paragraphs = text.split("\n\n")
|
|
current = ""
|
|
|
|
for para in paragraphs:
|
|
if len(current) + len(para) + 2 <= limit:
|
|
current = f"{current}\n\n{para}" if current else para
|
|
else:
|
|
if current:
|
|
chunks.append(current)
|
|
if len(para) > limit:
|
|
sub_chunks = _chunk_long_paragraph(para, limit)
|
|
if sub_chunks:
|
|
if sub_chunks[-1]:
|
|
current = sub_chunks.pop()
|
|
else:
|
|
sub_chunks.pop()
|
|
current = ""
|
|
chunks.extend(sub_chunks)
|
|
else:
|
|
current = ""
|
|
else:
|
|
current = para
|
|
|
|
if current:
|
|
chunks.append(current)
|
|
return chunks or [text[:limit]]
|
|
|
|
|
|
def _chunk_long_paragraph(text: str, limit: int) -> list[str]:
|
|
chunks: list[str] = []
|
|
while len(text) > limit:
|
|
split_at = _find_split_point(text, limit)
|
|
chunk = text[:split_at].rstrip()
|
|
if chunk:
|
|
chunks.append(chunk)
|
|
text = text[split_at:].lstrip()
|
|
if text:
|
|
chunks.append(text)
|
|
return chunks
|
|
|
|
|
|
def _find_split_point(text: str, limit: int) -> int:
|
|
candidates = [m.start() for m in _SENTENCE_BOUNDARY.finditer(text, limit // 2, limit)]
|
|
if candidates:
|
|
return candidates[-1] + 1
|
|
newline = text.rfind("\n", limit // 2, limit)
|
|
if newline != -1:
|
|
return newline + 1
|
|
space = text.rfind(" ", limit // 2, limit)
|
|
if space != -1:
|
|
return space + 1
|
|
return limit
|