93 lines
3.1 KiB
Python
93 lines
3.1 KiB
Python
|
from __future__ import annotations
|
|||
|
|
|
|||
|
|
import pytest
|
|||
|
|
|
|||
|
|
from yuxi.channels.adapters.wechat.chunker import WeChatChunker
|
|||
|
|
|
|||
|
|
|
|||
|
|
class TestWeChatChunker:
|
|||
|
|
def setup_method(self):
|
|||
|
|
self.chunker = WeChatChunker()
|
|||
|
|
|
|||
|
|
def test_chunk_text_within_limit(self):
|
|||
|
|
result = self.chunker.chunk_text("hello world", 2048)
|
|||
|
|
assert result == ["hello world"]
|
|||
|
|
|
|||
|
|
def test_chunk_text_empty(self):
|
|||
|
|
result = self.chunker.chunk_text("", 100)
|
|||
|
|
assert result == [""]
|
|||
|
|
|
|||
|
|
def test_chunk_text_exact_limit(self):
|
|||
|
|
text = "a" * 100
|
|||
|
|
result = self.chunker.chunk_text(text, 100)
|
|||
|
|
assert result == [text]
|
|||
|
|
|
|||
|
|
def test_chunk_text_splits_at_newline_double(self):
|
|||
|
|
para1 = "A" * 2000
|
|||
|
|
para2 = "B" * 2000
|
|||
|
|
text = f"{para1}\n\n{para2}"
|
|||
|
|
result = self.chunker.chunk_text(text, 3000)
|
|||
|
|
assert len(result) == 2
|
|||
|
|
assert para1 in result[0]
|
|||
|
|
|
|||
|
|
def test_chunk_text_splits_at_period(self):
|
|||
|
|
text = ("A" * 1000 + "。") * 5
|
|||
|
|
result = self.chunker.chunk_text(text, 2500)
|
|||
|
|
assert len(result) >= 2
|
|||
|
|
|
|||
|
|
def test_chunk_text_no_boundary_found_falls_back_to_limit(self):
|
|||
|
|
text = "A" * 3000
|
|||
|
|
result = self.chunker.chunk_text(text, 1000)
|
|||
|
|
assert len(result) == 3
|
|||
|
|
for chunk in result:
|
|||
|
|
assert len(chunk) <= 1000
|
|||
|
|
|
|||
|
|
def test_chunk_text_all_chunks_within_limit(self):
|
|||
|
|
text = "B" * 5000
|
|||
|
|
limit = 2048
|
|||
|
|
for chunk in self.chunker.chunk_text(text, limit):
|
|||
|
|
assert len(chunk) <= limit
|
|||
|
|
|
|||
|
|
def test_chunk_markdown_within_limit(self):
|
|||
|
|
result = self.chunker.chunk_markdown("hello world", 2048)
|
|||
|
|
assert result == ["hello world"]
|
|||
|
|
|
|||
|
|
def test_chunk_markdown_preserves_code_block(self):
|
|||
|
|
text = f"{'A' * 3000}```\ncode block\n```\n{'B' * 3000}"
|
|||
|
|
result = self.chunker.chunk_markdown(text, 2000)
|
|||
|
|
combined = "".join(result)
|
|||
|
|
assert "```" in combined
|
|||
|
|
assert "code block" in combined
|
|||
|
|
|
|||
|
|
def test_chunk_markdown_code_block_partial(self):
|
|||
|
|
text = f"{'A' * 1500}```\n{'C' * 1500}\n```\n{'B' * 1500}"
|
|||
|
|
result = self.chunker.chunk_markdown(text, 2000)
|
|||
|
|
combined = "".join(result)
|
|||
|
|
assert combined == text
|
|||
|
|
|
|||
|
|
def test_chunk_dispatch_text_mode(self):
|
|||
|
|
text = "hello"
|
|||
|
|
assert self.chunker.chunk(text, 100, "text") == ["hello"]
|
|||
|
|
|
|||
|
|
def test_chunk_dispatch_markdown_mode(self):
|
|||
|
|
text = "hello"
|
|||
|
|
assert self.chunker.chunk(text, 100, "markdown") == ["hello"]
|
|||
|
|
|
|||
|
|
def test_chunk_default_mode_is_text(self):
|
|||
|
|
text = "default_mode"
|
|||
|
|
assert self.chunker.chunk(text, 100) == ["default_mode"]
|
|||
|
|
|
|||
|
|
def test_find_boundary_prefers_largest_match(self):
|
|||
|
|
text = "hello\nworld。test！"
|
|||
|
|
boundary = WeChatChunker._find_boundary(text, len(text))
|
|||
|
|
assert boundary > 0
|
|||
|
|
|
|||
|
|
def test_find_boundary_no_match(self):
|
|||
|
|
text = "a" * 100
|
|||
|
|
boundary = WeChatChunker._find_boundary(text, 50)
|
|||
|
|
assert boundary >= 0
|
|||
|
|
|
|||
|
|
def test_find_boundary_at_start(self):
|
|||
|
|
text = "\n\nhello"
|
|||
|
|
boundary = WeChatChunker._find_boundary(text, 5)
|
|||
|
|
assert boundary >= 0
|