ForcePilot/backend/test/unit/backends/test_semantic_chunking.py

import pytest
import numpy as np
import os

# 现在可以安全地导入了，因为顶层不再有重型依赖
from yuxi.knowledge.chunking.ragflow_like.parsers import semantic
from yuxi.models import select_embedding_model

@pytest.fixture
def embed_fn():
    """使用真实的嵌入函数 (从 SiliconFlow 获取)"""
    model_id = "siliconflow/Qwen/Qwen3-Embedding-0.6B"
    try:
        model = select_embedding_model(model_id)
        def encode(sentences):
            if isinstance(sentences, str):
                sentences = [sentences]
            return model.encode(sentences)
        return encode
    except Exception as e:
        pytest.skip(f"无法初始化真实嵌入模型 {model_id}: {e}")

@pytest.fixture
def sample_markdown():
    with open("test/resource/test4.md", "r", encoding="utf-8") as f:
        return f.read()
def test_semantic_chunking_basic(embed_fn, sample_markdown):
    """测试基本的语义切分逻辑 (使用真实嵌入模型)"""
    
    # 配置切分参数
    parser_config = {
        "chunk_token_num": 200,  # 适中的 token 数以触发更多切分
        "overlapped_percent": 0
    }
    
    # 直接注入 embed_fn
    chunks = semantic.chunk_markdown(
        sample_markdown, 
        parser_config=parser_config, 
        embed_fn=embed_fn
    )
    
    # 验证结果
    assert isinstance(chunks, list)
    assert len(chunks) > 0
    
    print(f"\n成功切分为 {len(chunks)} 个片段:")
    for i, chunk in enumerate(chunks):
        print(f"--- 片段 {i+1} ---")
        print(chunk)
    
    # 验证标题路径增强是否生效 (检查是否包含 Part 或特殊元素标记)
    has_enhanced_title = any("|Part" in chunk or "|Math Block" in chunk for chunk in chunks)
    assert has_enhanced_title, "标题路径增强或分片标记失效"
    
    # 验证是否包含特殊的 LaTeX 公式或符号 (匹配输出中的格式)
    has_formula = any("f _ {0, 1}" in chunk or "\\sigma_ {y, d}" in chunk or "A _ {\\mathrm {s p}}" in chunk for chunk in chunks)
    assert has_formula, "LaTeX 公式/符号在切分中丢失"

    # 验证语义聚类是否大概生效 (钢制塔架和混凝土塔架内容应该分开)
    # 我们检查是否有钢制塔架相关的片段和混凝土塔架相关的片段
    # 使用 startswith 来更准确地定位章节开始，避免匹配到目次/前言中的文字
    has_steel = any(chunk.startswith("# 6 钢制塔架") for chunk in chunks)
    has_concrete = any(chunk.startswith("# 7 混凝土塔架") for chunk in chunks)
    
    assert has_steel and has_concrete, "语义内容丢失 (钢制或混凝土塔架章节开始部分)"
    
    # 验证它们是否在不同的片段中
    steel_chunks = [i for i, c in enumerate(chunks) if c.startswith("# 6 钢制塔架")]
    concrete_chunks = [i for i, c in enumerate(chunks) if c.startswith("# 7 混凝土塔架")]
    
    # 理论上语义聚类会将这些大章节分开
    if steel_chunks and concrete_chunks:
        assert not set(steel_chunks).intersection(set(concrete_chunks)), "钢制塔架和混凝土塔架大章节内容被错误地聚类在同一个 chunk 中了"

def test_heading_inference():
    """测试标题层级推断工具类"""
    from yuxi.knowledge.chunking.ragflow_like.utils.md_parser_utils import infer_heading_level
    
    assert infer_heading_level("1. 简介") == 1
    assert infer_heading_level("1.1 详细设计") == 2
    assert infer_heading_level("1.2.3 核心逻辑") == 3
    assert infer_heading_level("一、 背景") == 1
    assert infer_heading_level("普通文本") == 1