Kris
5946478772
新增小红书、XMPP、元宝、Zalo 四个渠道扩展。 小红书渠道扩展主要模块：config, gateway, webhook, outbound, streaming, pairing, security, dedupe, media, status, window XMPP 渠道扩展主要模块：plugin, config, gateway, outbound, streaming, pairing, security, dedupe, accounts, commands, muc, rate_limiter, stanza_utils, status, monitor 元宝渠道扩展主要模块：plugin, client, config_schema, gateway, outbound(chunk/queue/transport), inbound(dispatcher), streaming, pairing, security, accounts, actions, commands, codec(biz/conn), session, shared, utils Zalo 渠道扩展主要模块：api, config, gateway, webhook, outbound, pairing, security, session, polling, monitor, status
120 lines
3.2 KiB
Python
120 lines
3.2 KiB
Python
from __future__ import annotations
|
|
|
|
|
|
def has_unclosed_fence(text: str) -> bool:
|
|
fence_patterns = ["```", "~~~"]
|
|
for pattern in fence_patterns:
|
|
count = text.count(pattern)
|
|
if count % 2 != 0:
|
|
return True
|
|
return False
|
|
|
|
|
|
def strip_outer_fence(text: str) -> str:
|
|
for pattern in ("```", "~~~"):
|
|
if text.startswith(pattern) and text.endswith(pattern):
|
|
inner = text[len(pattern):-len(pattern)]
|
|
if "\n" in inner:
|
|
return inner
|
|
return text
|
|
|
|
|
|
def merge_block_streaming(base: str, next_chunk: str) -> str:
|
|
if not base:
|
|
return next_chunk
|
|
if not next_chunk:
|
|
return base
|
|
|
|
if has_unclosed_fence(base):
|
|
return base + "\n" + next_chunk
|
|
|
|
if base.rstrip().endswith("|") and next_chunk.lstrip().startswith("|"):
|
|
return base + "\n" + next_chunk
|
|
|
|
base_end = base.rstrip()
|
|
next_start = next_chunk.lstrip()
|
|
|
|
if base_end.endswith((".", "!", "?", ":", ";", ")", "]")) and next_start[0].isalpha():
|
|
return base + " " + next_chunk
|
|
|
|
if base_end.endswith("\n") or next_start.startswith("\n"):
|
|
return base + next_chunk
|
|
|
|
return base + "\n" + next_chunk
|
|
|
|
|
|
def ends_with_table_row(text: str) -> bool:
|
|
lines = text.rstrip().split("\n")
|
|
if not lines:
|
|
return False
|
|
last = lines[-1].strip()
|
|
return last.startswith("|") and last.endswith("|") and "---" not in last
|
|
|
|
|
|
def infer_separator(base: str, next_chunk: str) -> str:
|
|
if has_unclosed_fence(base):
|
|
return "\n"
|
|
if ends_with_table_row(base):
|
|
return "\n"
|
|
return "\n"
|
|
|
|
|
|
def chunk_atomic_aware(text: str, max_chars: int, base_chunk: str = "") -> list[str]:
|
|
if len(text) <= max_chars:
|
|
return [text]
|
|
|
|
chunks = []
|
|
|
|
sections = text.split("\n\n")
|
|
for section in sections:
|
|
if len(section) > max_chars:
|
|
if has_unclosed_fence(section):
|
|
chunks.append(section)
|
|
else:
|
|
paragraphs = section.split("\n")
|
|
current = ""
|
|
for para in paragraphs:
|
|
if len(current) + len(para) + 1 > max_chars and current:
|
|
chunks.append(current)
|
|
current = para
|
|
else:
|
|
current = current + "\n" + para if current else para
|
|
if current:
|
|
chunks.append(current)
|
|
else:
|
|
chunks.append(section)
|
|
|
|
if not chunks:
|
|
return [text[:max_chars]]
|
|
|
|
return chunks
|
|
|
|
|
|
def drain_buffer(
|
|
buffer: str,
|
|
min_chars: int,
|
|
max_chars: int,
|
|
merge_base: str = "",
|
|
) -> tuple[list[str], str]:
|
|
if not buffer:
|
|
return [], buffer
|
|
|
|
if len(buffer) < min_chars:
|
|
if has_unclosed_fence(buffer):
|
|
merged = merge_block_streaming(merge_base, buffer) if merge_base else buffer
|
|
return [merged], ""
|
|
if ends_with_table_row(buffer):
|
|
return [], buffer
|
|
return [], buffer
|
|
|
|
merged = merge_block_streaming(merge_base, buffer) if merge_base else buffer
|
|
|
|
if len(merged) <= max_chars:
|
|
return [merged], ""
|
|
|
|
chunks = chunk_atomic_aware(merged, max_chars)
|
|
if len(chunks) == 1:
|
|
return [chunks[0]], ""
|
|
|
|
return chunks[:-1], chunks[-1]
|