44 lines
1.2 KiB
Python
44 lines
1.2 KiB
Python
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import pytest
|
||
|
|
|
||
|
|
from yuxi.channels.adapters.telegram.chunking import chunk_text
|
||
|
|
|
||
|
|
|
||
|
|
class TestTextChunking:
|
||
|
|
def test_short_text_no_chunk(self):
|
||
|
|
result = chunk_text("hello world")
|
||
|
|
assert len(result) == 1
|
||
|
|
assert result[0] == "hello world"
|
||
|
|
|
||
|
|
def test_exact_limit(self):
|
||
|
|
text = "a" * 4096
|
||
|
|
result = chunk_text(text)
|
||
|
|
assert len(result) == 1
|
||
|
|
assert len(result[0]) == 4096
|
||
|
|
|
||
|
|
def test_over_limit_splits_at_paragraph(self):
|
||
|
|
para1 = "A" * 3000
|
||
|
|
para2 = "B" * 3000
|
||
|
|
text = f"{para1}\n\n{para2}"
|
||
|
|
result = chunk_text(text)
|
||
|
|
assert len(result) == 2
|
||
|
|
|
||
|
|
def test_over_limit_splits_at_sentence(self):
|
||
|
|
text = ("A" * 1000 + "\u3002") * 5
|
||
|
|
result = chunk_text(text)
|
||
|
|
assert len(result) == 2
|
||
|
|
|
||
|
|
def test_empty_text(self):
|
||
|
|
result = chunk_text("")
|
||
|
|
assert len(result) == 1
|
||
|
|
assert result[0] == ""
|
||
|
|
|
||
|
|
def test_source_lines_unchanged_below_80(self):
|
||
|
|
result = chunk_text("hello world")
|
||
|
|
assert result == ["hello world"]
|
||
|
|
|
||
|
|
def test_chunk_size_less_than_limit(self):
|
||
|
|
text = "B" * 5000
|
||
|
|
for ch in chunk_text(text):
|
||
|
|
assert len(ch) <= 4096, f"Chunk length {len(ch)} exceeds limit"
|