Kris
7a972055b7
新增大量渠道适配器相关的协议、策略、工具类与基础设施代码，包括： 1. 多协议定义：认证、消息、配置、网关等核心接口 2. 策略模块：上下文、群聊、去重、防抖等业务策略 3. 工具集：重试、去重、文本分块、消息格式化等SDK工具 4. 基础设施：外部进程管理、事件广播、熔断机制等 5. 账户与管道系统：账户管理、消息处理管道实现 6. 运行时服务：状态收集、维护任务、日志等后台服务
107 lines
2.9 KiB
Python
107 lines
2.9 KiB
Python
from __future__ import annotations
|
|
|
|
import re
|
|
from dataclasses import dataclass
|
|
from typing import Literal
|
|
|
|
_SENTENCE_BOUNDARY = re.compile(r"[。！？.!?\n]")
|
|
|
|
|
|
@dataclass
|
|
class ChunkConfig:
|
|
limit: int = 4096
|
|
mode: Literal["text", "markdown"] = "text"
|
|
|
|
|
|
def chunk_message(content: str, config: ChunkConfig | None = None) -> list[str]:
|
|
cfg = config or ChunkConfig()
|
|
if len(content) <= cfg.limit:
|
|
return [content]
|
|
|
|
if cfg.mode == "markdown":
|
|
return _chunk_markdown(content, cfg.limit)
|
|
|
|
return _chunk_text(content, cfg.limit)
|
|
|
|
|
|
def _chunk_text(text: str, limit: int) -> list[str]:
|
|
chunks: list[str] = []
|
|
paragraphs = text.split("\n\n")
|
|
current = ""
|
|
|
|
for para in paragraphs:
|
|
if len(current) + len(para) + 2 <= limit:
|
|
current = f"{current}\n\n{para}" if current else para
|
|
else:
|
|
if current:
|
|
chunks.append(current)
|
|
if len(para) > limit:
|
|
sub = _chunk_long_text(para, limit)
|
|
if sub:
|
|
if sub[-1]:
|
|
current = sub.pop()
|
|
else:
|
|
sub.pop()
|
|
current = ""
|
|
chunks.extend(sub)
|
|
else:
|
|
current = ""
|
|
else:
|
|
current = para
|
|
|
|
if current:
|
|
chunks.append(current)
|
|
return chunks or [text]
|
|
|
|
|
|
def _chunk_long_text(text: str, limit: int) -> list[str]:
|
|
chunks: list[str] = []
|
|
while len(text) > limit:
|
|
split_at = _find_split_point(text, limit)
|
|
chunks.append(text[:split_at].rstrip())
|
|
text = text[split_at:].lstrip()
|
|
chunks.append(text)
|
|
return chunks
|
|
|
|
|
|
def _find_split_point(text: str, limit: int) -> int:
|
|
candidates = [m.start() for m in _SENTENCE_BOUNDARY.finditer(text, limit // 2, limit)]
|
|
if candidates:
|
|
return candidates[-1] + 1
|
|
newline = text.rfind("\n", limit // 2, limit)
|
|
if newline != -1:
|
|
return newline + 1
|
|
space = text.rfind(" ", limit // 2, limit)
|
|
if space != -1:
|
|
return space + 1
|
|
return limit
|
|
|
|
|
|
def _chunk_markdown(text: str, limit: int) -> list[str]:
|
|
lines = text.split("\n")
|
|
chunks: list[str] = []
|
|
current = ""
|
|
in_code_block = False
|
|
|
|
for line in lines:
|
|
if line.strip().startswith("```"):
|
|
if in_code_block:
|
|
in_code_block = False
|
|
else:
|
|
if current and len(current) + len(line) + 1 > limit:
|
|
chunks.append(current)
|
|
current = line
|
|
in_code_block = True
|
|
continue
|
|
in_code_block = True
|
|
|
|
if current and len(current) + len(line) + 1 > limit and not in_code_block:
|
|
chunks.append(current)
|
|
current = line
|
|
else:
|
|
current = f"{current}\n{line}" if current else line
|
|
|
|
if current:
|
|
chunks.append(current)
|
|
return chunks or [text]
|