Merge pull request #652 from Flying-Henanese/feat/add-semantic-chunking
feat: 增加基于自然语义的文本切分功能
This commit is contained in:
Wenjie Zhang
GitHub
commit
743b5b03c4
@ -76,6 +76,12 @@ dependencies = [
|
||||
"typer>=0.16.0",
|
||||
"unstructured>=0.17.2",
|
||||
"wcmatch>=8.0.0",
|
||||
# Markdown 解析语义切分相关依赖
|
||||
"markdown-it-py>=3.0.0",
|
||||
"mdit-py-plugins>=0.4.0",
|
||||
"scikit-learn>=1.3.0",
|
||||
"nltk>=3.8.1",
|
||||
"beautifulsoup4>=4.12.0",
|
||||
]
|
||||
|
||||
[tool.setuptools]
|
||||
|
||||
@ -2,7 +2,7 @@ from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from yuxi.knowledge.chunking.ragflow_like.parsers import book, general, laws, qa
|
||||
from yuxi.knowledge.chunking.ragflow_like.parsers import book, general, laws, qa, semantic
|
||||
from yuxi.knowledge.chunking.ragflow_like.presets import map_to_internal_parser_id, normalize_chunk_preset_id
|
||||
|
||||
|
||||
@ -42,6 +42,8 @@ def _dispatch_markdown_parser(
|
||||
return book.chunk_markdown(markdown_content, parser_config)
|
||||
if parser_id == "laws":
|
||||
return laws.chunk_markdown(filename, markdown_content, parser_config)
|
||||
if parser_id == "semantic":
|
||||
return semantic.chunk_markdown(markdown_content, parser_config)
|
||||
|
||||
return general.chunk_markdown(markdown_content, parser_config)
|
||||
|
||||
|
||||
@ -0,0 +1,298 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from markdown_it import MarkdownIt
|
||||
from mdit_py_plugins.dollarmath import dollarmath_plugin
|
||||
|
||||
from yuxi.knowledge.chunking.ragflow_like.nlp import count_tokens
|
||||
from yuxi.utils.logging_config import logger
|
||||
|
||||
from ..utils.md_parser_utils import (
|
||||
extract_table_block,
|
||||
get_title_path,
|
||||
split_text_by_length_and_newline,
|
||||
)
|
||||
from ..utils.table_utils import html_table_to_key_value
|
||||
|
||||
|
||||
def _flush_content(
|
||||
result: list,
|
||||
current_content: list,
|
||||
title_stack: list,
|
||||
max_length: int,
|
||||
embed_fn: Any,
|
||||
special_element: str = None,
|
||||
allow_split: bool = False,
|
||||
) -> None:
|
||||
if not current_content:
|
||||
return
|
||||
|
||||
content = "\n".join(current_content).strip()
|
||||
if not content:
|
||||
current_content.clear()
|
||||
return
|
||||
|
||||
level = next((i + 1 for i in range(5, -1, -1) if title_stack[i]), 1)
|
||||
title_path = get_title_path(title_stack)
|
||||
|
||||
if special_element and not allow_split:
|
||||
header = f"{'#' * level} {title_path}|{special_element}" if title_path else f"{'#' * level} {special_element}"
|
||||
result.extend([header, content, "-" * 10])
|
||||
else:
|
||||
if count_tokens(content) > max_length:
|
||||
chunks = split_text_by_length_and_newline(
|
||||
content, max_length, embed_fn=embed_fn, token_count_fn=count_tokens
|
||||
)
|
||||
for idx, chunk in enumerate(chunks, 1):
|
||||
base_header = f"{'#' * level} {title_path}" if title_path else f"{'#' * level}"
|
||||
if special_element:
|
||||
header = f"{base_header}|{special_element}|Part {idx}"
|
||||
else:
|
||||
header = f"{base_header}|Part {idx}"
|
||||
result.extend([header, chunk, "-" * 10])
|
||||
else:
|
||||
base_header = f"{'#' * level} {title_path}" if title_path else f"{'#' * level}"
|
||||
if special_element:
|
||||
header = f"{base_header}|{special_element}"
|
||||
else:
|
||||
header = base_header
|
||||
|
||||
if header:
|
||||
result.append(header)
|
||||
result.append("")
|
||||
result.extend([content, "-" * 10])
|
||||
|
||||
current_content.clear()
|
||||
|
||||
|
||||
def _handle_image_caption(tokens, i, result, current_content, title_stack, max_length, embed_fn):
|
||||
token = tokens[i]
|
||||
if token.type != "paragraph_open":
|
||||
return False, i
|
||||
|
||||
inline_token = tokens[i + 1]
|
||||
if inline_token.type != "inline":
|
||||
return False, i
|
||||
|
||||
content = inline_token.content.strip()
|
||||
image_pattern = r"^!\[.*?\]\(.*?\)\s*$"
|
||||
caption_pattern = r"^(?:Figure|图|Fig\.|表|Table)\s*[\d\w\.]+"
|
||||
|
||||
img_match = re.search(r"^(!\[.*?\]\(.*?\))", content)
|
||||
if img_match:
|
||||
rest = content[img_match.end() :].strip()
|
||||
if rest and re.match(caption_pattern, rest, re.IGNORECASE):
|
||||
_flush_content(result, current_content, title_stack, max_length, embed_fn)
|
||||
current_content.append(content)
|
||||
caption_title = rest.split("\n")[0].strip()
|
||||
_flush_content(result, current_content, title_stack, max_length, embed_fn, special_element=caption_title)
|
||||
return True, i + 3
|
||||
|
||||
if re.match(image_pattern, content):
|
||||
next_p_idx = i + 3
|
||||
if next_p_idx + 1 < len(tokens) and tokens[next_p_idx].type == "paragraph_open":
|
||||
next_inline = tokens[next_p_idx + 1]
|
||||
if next_inline.type == "inline":
|
||||
next_content = next_inline.content.strip()
|
||||
if re.match(caption_pattern, next_content, re.IGNORECASE):
|
||||
_flush_content(result, current_content, title_stack, max_length, embed_fn)
|
||||
current_content.append(content)
|
||||
current_content.append(next_content)
|
||||
_flush_content(
|
||||
result, current_content, title_stack, max_length, embed_fn, special_element=next_content
|
||||
)
|
||||
return True, i + 6
|
||||
|
||||
if current_content and re.match(caption_pattern, content, re.IGNORECASE):
|
||||
last_item = current_content[-1].strip()
|
||||
if re.match(image_pattern, last_item):
|
||||
image_tag = current_content.pop()
|
||||
_flush_content(result, current_content, title_stack, max_length, embed_fn)
|
||||
current_content.append(image_tag)
|
||||
current_content.append(content)
|
||||
_flush_content(result, current_content, title_stack, max_length, embed_fn, special_element=content)
|
||||
return True, i + 3
|
||||
|
||||
return False, i
|
||||
|
||||
|
||||
def chunk_markdown(
|
||||
markdown_content: str, parser_config: dict[str, Any] | None = None, embed_fn: Any | None = None
|
||||
) -> list[str]:
|
||||
"""
|
||||
语义化切分 Markdown 内容。
|
||||
|
||||
Args:
|
||||
markdown_content: 待切分的 Markdown 文本
|
||||
parser_config: 切分参数，如 chunk_token_num
|
||||
embed_fn: 可选。传入用于生成向量的函数。如果不传，将从系统配置中加载模型。
|
||||
通过注入此参数可以避免在单元测试中加载重型资源。
|
||||
"""
|
||||
parser_config = parser_config or {}
|
||||
max_length = int(parser_config.get("chunk_token_num", 512))
|
||||
logger.info(f"语义切分开始: max_length={max_length}, content_length={len(markdown_content)}")
|
||||
|
||||
# 延迟加载重型资源，仅在没有注入 embed_fn 时触发
|
||||
if embed_fn is None:
|
||||
try:
|
||||
from yuxi.config.app import config
|
||||
from yuxi.models.embed import select_embedding_model
|
||||
|
||||
embed_model_id = parser_config.get("embed_model_id") or config.embed_model
|
||||
logger.info(f"语义切分加载Embedding模型: {embed_model_id}")
|
||||
embed_model = select_embedding_model(embed_model_id)
|
||||
embed_fn = embed_model.encode
|
||||
except Exception as e:
|
||||
logger.error(f"加载 Embedding 模型失败: {e}。将退化为简单切分。")
|
||||
embed_fn = None
|
||||
|
||||
md = MarkdownIt("commonmark").enable("table")
|
||||
md.use(dollarmath_plugin, allow_space=True, allow_digits=True)
|
||||
|
||||
tokens: list = md.parse(markdown_content)
|
||||
original_lines: list = markdown_content.split("\n")
|
||||
|
||||
result: list = []
|
||||
current_content: list = []
|
||||
title_stack: list = [""] * 6
|
||||
|
||||
i = 0
|
||||
while i < len(tokens):
|
||||
token = tokens[i]
|
||||
if token.type == "heading_open":
|
||||
_flush_content(result, current_content, title_stack, max_length, embed_fn)
|
||||
level = int(token.tag[1:]) if token.tag and len(token.tag) > 1 else 1
|
||||
inline_token = tokens[i + 1]
|
||||
if inline_token.type == "inline":
|
||||
full_title = inline_token.content.strip()
|
||||
title_stack[level - 1] = full_title
|
||||
for j in range(level, 6):
|
||||
title_stack[j] = ""
|
||||
i += 3
|
||||
continue
|
||||
elif token.type == "table_open":
|
||||
_flush_content(result, current_content, title_stack, max_length, embed_fn)
|
||||
j, table_content = extract_table_block(tokens, i, original_lines)
|
||||
current_content.append(table_content)
|
||||
_flush_content(result, current_content, title_stack, max_length, embed_fn, special_element="Table")
|
||||
i = j + 1 if j < len(tokens) else len(tokens)
|
||||
continue
|
||||
elif token.type == "paragraph_open":
|
||||
handled, new_i = _handle_image_caption(
|
||||
tokens, i, result, current_content, title_stack, max_length, embed_fn
|
||||
)
|
||||
if handled:
|
||||
i = new_i
|
||||
continue
|
||||
inline_token = tokens[i + 1]
|
||||
if inline_token.type == "inline":
|
||||
current_content.append(inline_token.content.strip())
|
||||
i += 3
|
||||
continue
|
||||
elif token.type == "fence":
|
||||
current_content.append(f"```\n{token.content}\n```")
|
||||
i += 1
|
||||
continue
|
||||
elif token.type == "ordered_list_open":
|
||||
# _flush_content(result, current_content, title_stack, max_length, embed_fn)
|
||||
list_content = []
|
||||
j = i + 1
|
||||
list_item_counter = 1
|
||||
while j < len(tokens) and tokens[j].type != "ordered_list_close":
|
||||
if tokens[j].type == "list_item_open":
|
||||
k = j + 1
|
||||
while k < len(tokens) and tokens[k].type != "list_item_close":
|
||||
if (
|
||||
tokens[k].type == "paragraph_open"
|
||||
and k + 1 < len(tokens)
|
||||
and tokens[k + 1].type == "inline"
|
||||
):
|
||||
list_content.append(f"{list_item_counter}. {tokens[k + 1].content.strip()}")
|
||||
list_item_counter += 1
|
||||
k += 1
|
||||
j += 1
|
||||
if list_content:
|
||||
current_content.extend(list_content)
|
||||
_flush_content(result, current_content, title_stack, max_length, embed_fn, special_element=token.type)
|
||||
i = j + 1
|
||||
continue
|
||||
elif token.type == "bullet_list_open":
|
||||
# _flush_content(result, current_content, title_stack, max_length, embed_fn)
|
||||
list_content = []
|
||||
j = i + 1
|
||||
while j < len(tokens) and tokens[j].type != "bullet_list_close":
|
||||
if tokens[j].type == "list_item_open":
|
||||
k = j + 1
|
||||
while k < len(tokens) and tokens[k].type != "list_item_close":
|
||||
if (
|
||||
tokens[k].type == "paragraph_open"
|
||||
and k + 1 < len(tokens)
|
||||
and tokens[k + 1].type == "inline"
|
||||
):
|
||||
list_content.append(f"- {tokens[k + 1].content.strip()}")
|
||||
k += 1
|
||||
j += 1
|
||||
if list_content:
|
||||
current_content.extend(list_content)
|
||||
_flush_content(result, current_content, title_stack, max_length, embed_fn, special_element=token.type)
|
||||
i = j + 1
|
||||
continue
|
||||
elif token.type == "html_block":
|
||||
_flush_content(result, current_content, title_stack, max_length, embed_fn)
|
||||
content = token.content.strip()
|
||||
is_converted_table = False
|
||||
if "<table" in content.lower():
|
||||
try:
|
||||
kv_list = html_table_to_key_value(content)
|
||||
if kv_list:
|
||||
content = "\n".join([f"- {item}" for item in kv_list])
|
||||
is_converted_table = True
|
||||
except Exception as e:
|
||||
logger.warning(f"HTML表格转KV失败: {e}")
|
||||
|
||||
current_content.append(content)
|
||||
if is_converted_table:
|
||||
_flush_content(
|
||||
result,
|
||||
current_content,
|
||||
title_stack,
|
||||
max_length,
|
||||
embed_fn,
|
||||
special_element="Table KV",
|
||||
allow_split=True,
|
||||
)
|
||||
else:
|
||||
_flush_content(result, current_content, title_stack, max_length, embed_fn, special_element=token.type)
|
||||
i += 1
|
||||
continue
|
||||
elif token.type in ["list_item_close", "ordered_list_close", "bullet_list_close", "list_item_open"]:
|
||||
i += 1
|
||||
continue
|
||||
elif token.type == "math_block":
|
||||
# _flush_content(result, current_content, title_stack, max_length, embed_fn)
|
||||
current_content.append(f"$ {token.content} $")
|
||||
_flush_content(result, current_content, title_stack, max_length, embed_fn, special_element="Math Block")
|
||||
i += 1
|
||||
continue
|
||||
else:
|
||||
i += 1
|
||||
|
||||
_flush_content(result, current_content, title_stack, max_length, embed_fn)
|
||||
|
||||
chunks = []
|
||||
current_chunk_parts = []
|
||||
for item in result:
|
||||
if item == "-" * 10:
|
||||
if current_chunk_parts:
|
||||
chunks.append("\n".join(current_chunk_parts).strip())
|
||||
current_chunk_parts = []
|
||||
else:
|
||||
current_chunk_parts.append(item)
|
||||
|
||||
if current_chunk_parts:
|
||||
chunks.append("\n".join(current_chunk_parts).strip())
|
||||
|
||||
logger.info(f"语义切分完成: chunks={len(chunks)}")
|
||||
return chunks
|
||||
@ -9,12 +9,14 @@ CHUNK_PRESET_GENERAL = "general"
|
||||
CHUNK_PRESET_QA = "qa"
|
||||
CHUNK_PRESET_BOOK = "book"
|
||||
CHUNK_PRESET_LAWS = "laws"
|
||||
CHUNK_PRESET_SEMANTIC = "semantic"
|
||||
|
||||
CHUNK_PRESET_IDS = {
|
||||
CHUNK_PRESET_GENERAL,
|
||||
CHUNK_PRESET_QA,
|
||||
CHUNK_PRESET_BOOK,
|
||||
CHUNK_PRESET_LAWS,
|
||||
CHUNK_PRESET_SEMANTIC,
|
||||
}
|
||||
|
||||
CHUNK_PRESET_DESCRIPTIONS: dict[str, str] = {
|
||||
@ -22,6 +24,7 @@ CHUNK_PRESET_DESCRIPTIONS: dict[str, str] = {
|
||||
CHUNK_PRESET_QA: "问答分块：优先抽取问题-回答结构，适合 FAQ、题库、问答手册。",
|
||||
CHUNK_PRESET_BOOK: "书籍分块：强化章节标题识别并做层级合并，适合教材、手册、长章节文档。",
|
||||
CHUNK_PRESET_LAWS: "法规分块：按法条层级组织与合并，适合法律法规、制度规范类文本。",
|
||||
CHUNK_PRESET_SEMANTIC: "语义分块：利用嵌入和聚类算法进行语义切分，并自动增强标题上下文。",
|
||||
}
|
||||
|
||||
CHUNK_ENGINE_VERSION = "ragflow_like_v1"
|
||||
@ -63,6 +66,14 @@ _PRESET_DEFAULTS: dict[str, dict[str, Any] | None] = {
|
||||
CHUNK_PRESET_QA: {"raptor": {"use_raptor": False}, "graphrag": {"use_graphrag": False}},
|
||||
CHUNK_PRESET_BOOK: {"raptor": {"use_raptor": False}, "graphrag": {"use_graphrag": False}},
|
||||
CHUNK_PRESET_LAWS: {"raptor": {"use_raptor": False}, "graphrag": {"use_graphrag": False}},
|
||||
CHUNK_PRESET_SEMANTIC: {
|
||||
"raptor": {"use_raptor": False},
|
||||
"graphrag": {
|
||||
"use_graphrag": True,
|
||||
"entity_types": ["organization", "person", "geo", "event", "category"],
|
||||
"method": "light",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@ -235,4 +246,9 @@ def get_chunk_preset_options() -> list[dict[str, str]]:
|
||||
"label": "Laws",
|
||||
"description": CHUNK_PRESET_DESCRIPTIONS[CHUNK_PRESET_LAWS],
|
||||
},
|
||||
{
|
||||
"value": CHUNK_PRESET_SEMANTIC,
|
||||
"label": "Semantic",
|
||||
"description": CHUNK_PRESET_DESCRIPTIONS[CHUNK_PRESET_SEMANTIC],
|
||||
},
|
||||
]
|
||||
|
||||
@ -0,0 +1,150 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from .semantic_utils import semantic_chunking_with_auto_clusters
|
||||
|
||||
|
||||
def infer_heading_level(title: str) -> int:
|
||||
"""
|
||||
根据标题文本推断其层级级别（1-6级）。
|
||||
|
||||
逻辑说明：
|
||||
1. 数字序号推断：
|
||||
- 匹配如 "1.", "1.1", "1.2.3" 等格式。
|
||||
- 根据点号分隔的数量确定层级，例如 "1.1" 为 2 级，"1.2.3" 为 3 级。
|
||||
- 层级限制在 1-6 之间。
|
||||
2. 中文序号推断：
|
||||
- 匹配如 "一、", "二." 等中文数字序号。
|
||||
- 统一归类为 1 级标题。
|
||||
3. 默认处理：
|
||||
- 若不匹配以上规则，默认返回 1 级。
|
||||
"""
|
||||
m = re.match(r"^\s*(\d+(?:\.\d+)*)[.)、]?\s*", title)
|
||||
if m:
|
||||
return max(1, min(len(m.group(1).split(".")), 6))
|
||||
m_zh = re.match(r"^\s*[一二三四五六七八九十百千]+[、.]\s*", title)
|
||||
if m_zh:
|
||||
return 1
|
||||
return 1
|
||||
|
||||
|
||||
def get_title_path(stack: list[str]) -> str:
|
||||
"""
|
||||
根据标题栈生成标题路径，用"|"分隔。
|
||||
"""
|
||||
return "|".join([t for t in stack if t])
|
||||
|
||||
|
||||
def extract_table_block(tokens: list[Any], i: int, original_lines: list[str]) -> tuple[int, str]:
|
||||
"""
|
||||
从token流和原始文本中提取完整的表格块。
|
||||
|
||||
逻辑说明：
|
||||
1. 定位起始：通过当前 token (i) 的 `map` 属性获取表格在原始行中的起始行号 `table_start`。
|
||||
2. 查找结束 token：遍历后续 tokens 直到找到 `table_close`。
|
||||
3. 确定结束行号 (`table_end`)：
|
||||
- 优先使用 `table_close` token 的 `map` 属性。
|
||||
- 若不存在，则尝试查找下一个带有 `map` 信息的 token 的起始行作为当前表格的结束。
|
||||
- 若上述均失败（如文件末尾或解析异常），则回退到基于文本内容的启发式扫描：
|
||||
从 `table_start` 开始向下扫描，直到遇到不符合 Markdown 表格特征（不以 '|' 开头且不含 '|'）的行为止。
|
||||
4. 返回结果：返回 `table_close` 的索引 `j` 以及拼接后的表格原始字符串。
|
||||
"""
|
||||
token = tokens[i]
|
||||
table_start = token.map[0] if token.map else 0
|
||||
j = i + 1
|
||||
while j < len(tokens) and tokens[j].type != "table_close":
|
||||
j += 1
|
||||
if j < len(tokens):
|
||||
end_token = tokens[j]
|
||||
if end_token.map and end_token.map[1] is not None:
|
||||
table_end = end_token.map[1]
|
||||
else:
|
||||
table_end = None
|
||||
for k in range(j + 1, len(tokens)):
|
||||
if tokens[k].map and tokens[k].map[0] is not None:
|
||||
table_end = tokens[k].map[0]
|
||||
break
|
||||
if table_end is None:
|
||||
table_end = table_start + 1
|
||||
for line_idx in range(table_start, len(original_lines)):
|
||||
line = original_lines[line_idx].strip()
|
||||
if not line or not (line.startswith("|") or "|" in line):
|
||||
table_end = line_idx
|
||||
break
|
||||
else:
|
||||
table_end = table_start + 1
|
||||
for line_idx in range(table_start, len(original_lines)):
|
||||
line = original_lines[line_idx].strip()
|
||||
if not line or not (line.startswith("|") or "|" in line):
|
||||
table_end = line_idx
|
||||
break
|
||||
return j, "\n".join(original_lines[table_start:table_end])
|
||||
|
||||
|
||||
def split_text_by_length_and_newline(
|
||||
text: str, max_length: int, embed_fn: Callable[[list[str]], Any] | None, token_count_fn: Callable[[str], int]
|
||||
) -> list[str]:
|
||||
"""
|
||||
层次化文本切分策略。
|
||||
"""
|
||||
chunks = []
|
||||
|
||||
paragraphs = text.split("\n\n")
|
||||
|
||||
for paragraph in paragraphs:
|
||||
paragraph = paragraph.strip()
|
||||
if not paragraph:
|
||||
continue
|
||||
|
||||
paragraph_token_count = token_count_fn(paragraph)
|
||||
|
||||
# 如果当前段落长度未超过最大 Token 数量，直接作为独立分块放入chunks
|
||||
# 否则继续尝试按行切分
|
||||
if paragraph_token_count <= max_length:
|
||||
chunks.append(paragraph)
|
||||
continue
|
||||
|
||||
# 把段落进一步使用换行符进行切分为行
|
||||
lines = paragraph.split("\n")
|
||||
current_chunk_lines = []
|
||||
current_chunk_tokens = 0
|
||||
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
if not line: # 跳过空行
|
||||
continue
|
||||
|
||||
line_token_count = token_count_fn(line) # 计算当前行的 Token 数量
|
||||
# 为了考虑行之间的空格，需要在计算 Token 数量时加 1（如果当前行不是第一行，需要添加一个换行符的Token数量）
|
||||
added_tokens = line_token_count + (1 if current_chunk_lines else 0)
|
||||
# 如果当前行的 Token 数量超过最大 Token 数量，直接作为独立分块放入chunks
|
||||
if line_token_count > max_length:
|
||||
if current_chunk_lines:
|
||||
chunks.append("\n".join(current_chunk_lines))
|
||||
current_chunk_lines = []
|
||||
current_chunk_tokens = 0
|
||||
|
||||
sub_chunks = semantic_chunking_with_auto_clusters(
|
||||
line, embed_fn=embed_fn, token_count_fn=token_count_fn, max_chunk_size=max_length
|
||||
)
|
||||
chunks.extend(sub_chunks)
|
||||
# 如果当前行的 Token 数量与当前分块的 Token 数量合并后超过最大 Token 数量，直接作为独立分块放入chunks
|
||||
elif current_chunk_tokens + added_tokens > max_length:
|
||||
# 把之前的分块内容放入chunks
|
||||
chunks.append("\n".join(current_chunk_lines))
|
||||
# 重置当前分块为当前行的内容
|
||||
current_chunk_lines = [line]
|
||||
# 更新当前分块的 Token 数量
|
||||
current_chunk_tokens = line_token_count
|
||||
# 如果当前行的内容加入当前分块后不会超过最大 Token 数量，直接加入当前分块
|
||||
else:
|
||||
current_chunk_lines.append(line)
|
||||
current_chunk_tokens += added_tokens # 更新当前分块的 Token 数量
|
||||
# 最后的收尾，把最后一行内容放入chunks
|
||||
if current_chunk_lines:
|
||||
chunks.append("\n".join(current_chunk_lines))
|
||||
|
||||
return chunks
|
||||
@ -0,0 +1,192 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
import nltk
|
||||
from nltk.tokenize import sent_tokenize
|
||||
from sklearn.cluster import AgglomerativeClustering
|
||||
from sklearn.metrics import silhouette_score
|
||||
|
||||
_punkt_checked = False
|
||||
|
||||
|
||||
def _ensure_punkt_tab() -> None:
|
||||
"""首次使用分句能力时检查 NLTK punkt_tab 资源，缺失时给出可操作错误。"""
|
||||
global _punkt_checked
|
||||
if _punkt_checked:
|
||||
return
|
||||
|
||||
try:
|
||||
nltk.data.find("tokenizers/punkt_tab")
|
||||
except LookupError as e:
|
||||
raise RuntimeError(
|
||||
"缺少 NLTK 资源 punkt_tab。请先执行: python -m nltk.downloader punkt_tab"
|
||||
) from e
|
||||
|
||||
_punkt_checked = True
|
||||
|
||||
|
||||
def split_sentences_chinese(text: str) -> list[str]:
|
||||
"""
|
||||
使用正则表达式将中文文本分割成句子。
|
||||
|
||||
逻辑：
|
||||
- 匹配中文句号、感叹号、问号（。！？）作为分隔点。
|
||||
- 使用正向/反向预查处理引号：确保如果标点后面紧跟引号（”’"），该引号会被保留在当前句子末尾，而不是被切分到下一句。
|
||||
- 返回去除两端空格且非空的句子列表。
|
||||
"""
|
||||
pattern = r'(?<=[。！？][”’"])|(?<=[。！？])(?![”’"])'
|
||||
sentences = re.split(pattern, text)
|
||||
return [s.strip() for s in sentences if s.strip()]
|
||||
|
||||
|
||||
def split_mixed_sentences(text: str) -> list[str]:
|
||||
"""
|
||||
处理中英文混合文本的分句逻辑，支持按物理段落分发不同的分句策略。
|
||||
|
||||
该函数采用“分而治之”的策略来处理复杂的混合文本：
|
||||
1. **物理分块**：首先按换行符 (`\\n+`) 将原始文本切分为多个物理段落（chunks），确保物理结构不被破坏。
|
||||
2. **语言检测与分发**：
|
||||
- **英文/混合路径**：若段落中包含英文字母 (`[A-Za-z]`)，则视为英文或混合文本，
|
||||
调用 NLTK 的 `sent_tokenize` 进行处理。NLTK 能更好地处理英文缩写、句点等复杂情况。
|
||||
- **中文路径**：若段落不含字母，则视为纯中文文本，调用 `split_sentences_chinese`。
|
||||
该方法通过正则精准匹配中文标点及后续引号。
|
||||
- **兜底方案**：若上述方法未产生结果，则使用简单的正则表达式按中文标点强制分割。
|
||||
3. **清洗与过滤**：汇总所有子句，去除两端空白字符，并过滤掉空字符串。
|
||||
|
||||
Args:
|
||||
text: 待分句的原始字符串。
|
||||
|
||||
Returns:
|
||||
List[str]: 分割后的句子列表。
|
||||
"""
|
||||
_ensure_punkt_tab()
|
||||
|
||||
chunks = re.split(r"(\n+)", text)
|
||||
sentences = []
|
||||
|
||||
for ch in chunks:
|
||||
if not ch.strip():
|
||||
continue
|
||||
if re.search(r"[A-Za-z]", ch):
|
||||
parts = sent_tokenize(ch)
|
||||
sentences.extend([p.strip() for p in parts if p.strip()])
|
||||
else:
|
||||
sents = split_sentences_chinese(ch)
|
||||
if sents:
|
||||
sentences.extend([s.strip() for s in sents if s.strip()])
|
||||
else:
|
||||
parts = re.split(r"(?<=[。！？])", ch)
|
||||
sentences.extend([p.strip() for p in parts if p.strip()])
|
||||
return sentences
|
||||
|
||||
|
||||
def find_best_num_clusters(embeddings: Any, min_clusters: int = 2, max_clusters: int = 10) -> int:
|
||||
"""
|
||||
使用轮廓系数选择最佳聚类数量。让每个分段语义集中，且分段之间界限分明
|
||||
|
||||
逻辑：
|
||||
- 遍历可能的聚类数量（从 min_clusters 到 max_clusters）。
|
||||
- 对每个聚类数量，使用 `AgglomerativeClustering` 进行聚类。
|
||||
- 计算轮廓系数（Silhouette Score）。
|
||||
- 选择轮廓系数最高的聚类数量作为最佳聚类数量。
|
||||
- 如果聚类数量为 1 或更少，直接返回 1。
|
||||
|
||||
Args:
|
||||
embeddings: 待聚类的向量数据（所有句子的嵌入向量列表）。
|
||||
min_clusters: 搜索的最佳聚类数量下限，默认为 2。
|
||||
max_clusters: 搜索的最佳聚类数量上限，默认为 10。
|
||||
|
||||
Returns:
|
||||
int: 轮廓系数表现最好的聚类数量。
|
||||
"""
|
||||
if len(embeddings) <= min_clusters:
|
||||
return len(embeddings)
|
||||
|
||||
best_score = -1
|
||||
best_k = min_clusters
|
||||
|
||||
limit_k = min(max_clusters, len(embeddings))
|
||||
for k in range(min_clusters, limit_k + 1):
|
||||
labels = AgglomerativeClustering(n_clusters=k, metric="cosine", linkage="average").fit_predict(embeddings)
|
||||
if len(set(labels)) <= 1:
|
||||
continue
|
||||
score = silhouette_score(embeddings, labels, metric="cosine")
|
||||
if score > best_score:
|
||||
best_score = score
|
||||
best_k = k
|
||||
|
||||
return best_k
|
||||
|
||||
|
||||
def semantic_chunking_with_auto_clusters(
|
||||
text: str, embed_fn: Callable[[list[str]], Any] | None, token_count_fn: Callable[[str], int], max_chunk_size: int = 512
|
||||
) -> list[str]:
|
||||
"""
|
||||
对传入的文本进行语义切分，过程中会自动选择最佳的聚集数量。
|
||||
|
||||
逻辑：
|
||||
- 先将文本中的句子按语言进行分发，英文/混合文本使用NLTK的sent_tokenize，中文文本使用split_sentences_chinese。
|
||||
- 对每个句子进行嵌入向量化。
|
||||
- 确定最佳的聚类数量（根据轮廓系数）。
|
||||
- 对句子进行聚类后，按原文顺序遍历：当聚类标签变化或达到长度上限时切分，形成连续分块。
|
||||
- 如果嵌入模型缺失，则退化为原始切分方式
|
||||
"""
|
||||
sentences = split_mixed_sentences(text)
|
||||
if len(sentences) < 2:
|
||||
return [text.strip()]
|
||||
|
||||
# 计算每个句子的token数量
|
||||
sentence_token_counts = [token_count_fn(s) for s in sentences]
|
||||
total_tokens = sum(sentence_token_counts)
|
||||
|
||||
# 如果没有提供向量化函数，或者整体未超长，则直接进行简单合并/返回
|
||||
if embed_fn is None or total_tokens <= max_chunk_size:
|
||||
chunks = []
|
||||
current_chunk = ""
|
||||
current_chunk_tokens = 0
|
||||
for s, cnt in zip(sentences, sentence_token_counts):
|
||||
if current_chunk_tokens + cnt > max_chunk_size and current_chunk:
|
||||
chunks.append(current_chunk.strip())
|
||||
current_chunk = s
|
||||
current_chunk_tokens = cnt
|
||||
else:
|
||||
current_chunk += s
|
||||
current_chunk_tokens += cnt
|
||||
if current_chunk:
|
||||
chunks.append(current_chunk.strip())
|
||||
return chunks
|
||||
|
||||
# 向量化每个句子, 得到他们的嵌入向量
|
||||
embeddings = embed_fn(sentences)
|
||||
|
||||
# 决定合适的聚集数量：超长时按上限向上取整，避免整除时多切一块
|
||||
best_k = (total_tokens + max_chunk_size - 1) // max_chunk_size
|
||||
best_k = min(best_k, len(sentences))
|
||||
|
||||
# 根据指定的聚集数量、相似度判断方式、联动方式，对句子进行聚类
|
||||
# labels 是每个句子的聚类标签列表（如 [0,0,1,2,2]），后续会按原文顺序在标签变化处切分连续分块
|
||||
labels = AgglomerativeClustering(n_clusters=best_k, metric="cosine", linkage="average").fit_predict(embeddings)
|
||||
|
||||
chunks = []
|
||||
current_chunk = ""
|
||||
current_chunk_tokens = 0
|
||||
current_label = labels[0]
|
||||
|
||||
for sentence, label, token_count in zip(sentences, labels, sentence_token_counts):
|
||||
if label != current_label or current_chunk_tokens + token_count > max_chunk_size:
|
||||
if current_chunk.strip():
|
||||
chunks.append(current_chunk.strip())
|
||||
current_chunk = sentence
|
||||
current_chunk_tokens = token_count
|
||||
current_label = label
|
||||
else:
|
||||
current_chunk += sentence
|
||||
current_chunk_tokens += token_count
|
||||
|
||||
if current_chunk.strip():
|
||||
chunks.append(current_chunk.strip())
|
||||
|
||||
return chunks
|
||||
@ -0,0 +1,147 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
|
||||
def html_table_to_markdown(html: str) -> str:
|
||||
"""
|
||||
将HTML表格转换为Markdown格式的具体实现
|
||||
"""
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
table = soup.find("table")
|
||||
if table is None:
|
||||
return ""
|
||||
|
||||
rows = table.find_all("tr")
|
||||
if not rows:
|
||||
return ""
|
||||
|
||||
grid = []
|
||||
|
||||
for r_idx, row in enumerate(rows):
|
||||
while len(grid) <= r_idx:
|
||||
grid.append([])
|
||||
|
||||
cells = row.find_all(["td", "th"])
|
||||
c_idx = 0
|
||||
|
||||
for cell in cells:
|
||||
while c_idx < len(grid[r_idx]) and grid[r_idx][c_idx] is not None:
|
||||
c_idx += 1
|
||||
|
||||
text = cell.get_text(strip=True)
|
||||
text = text.replace("\n", " ")
|
||||
|
||||
rowspan = int(cell.get("rowspan", 1))
|
||||
colspan = int(cell.get("colspan", 1))
|
||||
|
||||
for r in range(rowspan):
|
||||
target_r = r_idx + r
|
||||
while len(grid) <= target_r:
|
||||
grid.append([])
|
||||
|
||||
for c in range(colspan):
|
||||
target_c = c_idx + c
|
||||
while len(grid[target_r]) <= target_c:
|
||||
grid[target_r].append(None)
|
||||
|
||||
grid[target_r][target_c] = text
|
||||
|
||||
c_idx += colspan
|
||||
|
||||
if not grid:
|
||||
return ""
|
||||
|
||||
markdown_lines = []
|
||||
max_cols = max(len(r) for r in grid)
|
||||
for r in grid:
|
||||
while len(r) < max_cols:
|
||||
r.append("")
|
||||
|
||||
header = grid[0]
|
||||
header = [h if h is not None else "" for h in header]
|
||||
markdown_lines.append("| " + " | ".join(header) + " |")
|
||||
markdown_lines.append("|" + "|".join([" --- " for _ in range(max_cols)]) + "|")
|
||||
|
||||
for row in grid[1:]:
|
||||
row_clean = [cell if cell is not None else "" for cell in row]
|
||||
line = "| " + " | ".join(row_clean) + " |"
|
||||
markdown_lines.append(line)
|
||||
|
||||
return "\n".join(markdown_lines)
|
||||
|
||||
|
||||
def html_table_to_key_value(html: str) -> list[str]:
|
||||
"""
|
||||
将HTML表格转换为键值对格式的列表，为了应对过长的表格的切分问题。
|
||||
|
||||
处理逻辑：
|
||||
1. **网格重建**：由于 HTML 表格可能包含 `rowspan` 和 `colspan`（合并单元格），
|
||||
函数首先构建一个完整的二维网格（grid）。
|
||||
2. **单元格展开**：遍历 HTML 行和列，遇到合并单元格时，将其内容填充到网格中受影响的所有坐标点。
|
||||
这确保了原本被合并的区域在逻辑网格中每个点都有对应的值。
|
||||
3. **键值对转换**：
|
||||
- 将网格的第一行视为表头（Key）。
|
||||
- 从第二行开始，将每一行与表头对应，生成 "键：值" 形式的字符串。
|
||||
|
||||
例如：
|
||||
- 输入：HTML表格，包含姓名、年龄、性别三列。
|
||||
- 输出：['姓名：张三；年龄：25；性别：男', '姓名：李四；年龄：30；性别：女']
|
||||
"""
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
table = soup.find("table")
|
||||
if table is None:
|
||||
return []
|
||||
|
||||
rows = table.find_all("tr")
|
||||
if not rows:
|
||||
return []
|
||||
|
||||
grid = []
|
||||
|
||||
for r_idx, row in enumerate(rows):
|
||||
while len(grid) <= r_idx:
|
||||
grid.append([])
|
||||
|
||||
cells = row.find_all(["td", "th"])
|
||||
c_idx = 0
|
||||
|
||||
for cell in cells:
|
||||
while c_idx < len(grid[r_idx]) and grid[r_idx][c_idx] is not None:
|
||||
c_idx += 1
|
||||
|
||||
text = cell.get_text(strip=True)
|
||||
rowspan = int(cell.get("rowspan", 1))
|
||||
colspan = int(cell.get("colspan", 1))
|
||||
|
||||
for r in range(rowspan):
|
||||
target_r = r_idx + r
|
||||
while len(grid) <= target_r:
|
||||
grid.append([])
|
||||
|
||||
for c in range(colspan):
|
||||
target_c = c_idx + c
|
||||
while len(grid[target_r]) <= target_c:
|
||||
grid[target_r].append(None)
|
||||
grid[target_r][target_c] = text
|
||||
c_idx += colspan
|
||||
|
||||
if not grid:
|
||||
return []
|
||||
|
||||
headers = grid[0]
|
||||
headers = [h if h is not None else "" for h in headers]
|
||||
|
||||
kv_lines = []
|
||||
for row_values in grid[1:]:
|
||||
min_len = min(len(headers), len(row_values))
|
||||
row_parts = []
|
||||
for i in range(min_len):
|
||||
key = headers[i]
|
||||
val = row_values[i] if row_values[i] is not None else ""
|
||||
if key:
|
||||
row_parts.append(f"{key}：{val}")
|
||||
if row_parts:
|
||||
kv_lines.append("；".join(row_parts) + "；")
|
||||
|
||||
return kv_lines
|
||||
319
backend/test/unit/backends/test_semantic_chunking.py
Normal file
319
backend/test/unit/backends/test_semantic_chunking.py
Normal file
@ -0,0 +1,319 @@
|
||||
import pytest
|
||||
import numpy as np
|
||||
|
||||
# 现在可以安全地导入了，因为顶层不再有重型依赖
|
||||
from yuxi.knowledge.chunking.ragflow_like.parsers import semantic
|
||||
from yuxi.models import select_embedding_model
|
||||
|
||||
@pytest.fixture
|
||||
def embed_fn():
|
||||
"""使用真实的嵌入函数 (从 SiliconFlow 获取)"""
|
||||
model_id = "siliconflow/Qwen/Qwen3-Embedding-0.6B"
|
||||
try:
|
||||
model = select_embedding_model(model_id)
|
||||
def encode(sentences):
|
||||
if isinstance(sentences, str):
|
||||
sentences = [sentences]
|
||||
return model.encode(sentences)
|
||||
return encode
|
||||
except Exception as e:
|
||||
pytest.skip(f"无法初始化真实嵌入模型 {model_id}: {e}")
|
||||
|
||||
@pytest.fixture
|
||||
def sample_markdown():
|
||||
"""提供一个包含多章节、公式符号和复杂结构的临时 Markdown 样本"""
|
||||
return """
|
||||
## 痛点与挑战
|
||||
|
||||
在传统的RAG知识库前置处理过程中，我们面临着诸多挑战：
|
||||
|
||||
- **格式繁杂兼容难**：企业文档格式多样，传统工具（例如python-docx和pymupdf）难以同时高质量处理 Office 文档和复杂的 PDF 扫描件。
|
||||
- **语义割裂检索差**：简单的按字符切分导致上下文丢失，检索匹配度低，无法满足业务需求。
|
||||
- **解析效率瓶颈**：面对海量存量文档，缺乏高并发处理能力，响应速度慢，难以支撑大规模应用。
|
||||
|
||||
## 技术方案概览
|
||||
|
||||
为了解决上述问题，我们构建了一套高性能、可扩展的文档处理架构：
|
||||
|
||||
- **全格式支持**：全面覆盖 **Word、Excel、PDF、PPT和图片**等主流办公文档格式。
|
||||
- **高精度解析**：通过引入MinerU VLM模型实现准确的布局解析 and 元素提取。
|
||||
- **跨平台支持**：同时支持**英伟达CUDA和华为CANN框架。**
|
||||
- **知识库应用优化**：为了更好支持上层应用，解析后增加了智能切分和实体识别功能，增强后续的检索效果。
|
||||
- **高并发架构设计**：
|
||||
- 采用 **FastAPI** 作为服务入口，保障请求的高效接收与响应。
|
||||
- 引入 **Celery** 分布式任务队列，实现请求响应与处理过程的完全解耦，有效应对流量洪峰，保障系统稳定性和可用性。
|
||||
- **资源利用最大化**：通过创建多组 **Celery Worker**（执行CPU密集任务） + **vLLM Server**（执行GPU密集推理任务）的Pod单元，实现对多核CPU+多卡GPU的并行调用，大幅提升单机处理吞吐量。
|
||||
- 使用Docker Compose进行服务编排，不同功能划分为不同的进程，保证服务的并发量和性能。
|
||||
|
||||
|
||||
|
||||
## 架构设计
|
||||
|
||||
| 服务名称 | 容器名称 | 职责描述 |
|
||||
| --- | --- | --- |
|
||||
| **mineru-api** | `mineru-api` | **API 网关：**基于 FastAPI，提供对外 HTTP 接口 。负责接收请求、鉴权及任务分发。 |
|
||||
| **mineru-worker** | `mineru-worker` | **异步任务处理器：**基于 Celery，负责 PDF 解析、OCR、文档转换等耗时任务。 |
|
||||
| **vllm** | `mineru-vllm` | **推理后端：**运行 MinerU 视觉大模型 (VLM)，提供高并发的文档理解与提取能力。 |
|
||||
| **redis** | `mineru-redis` | **消息中间件：**作为 Celery 的 Broker 和 Backend，同时缓存部分运行时数据。 |
|
||||
|
||||
## 核心技术亮点
|
||||
|
||||
### 1. 多模态融合的高精度解析
|
||||
|
||||
针对不同类型的文档，我们采用了差异化的解析策略，以确保最佳效果：
|
||||
* **Office 文档**：利用 **Docling** ，快速、精准地从文件中提取文本与格式信息。
|
||||
* **复杂 PDF 文档**：引入 **MinerU 视觉大模型（VLM）**，该模型具备类似人类的视觉能力，能够深度理解文档排版，完美还原文档结构和准确提取页面上的公式和表格元素，并统一输出为标准的 Markdown 格式。
|
||||
|
||||
|
||||
|
||||
原始文件
|
||||
|
||||
|
||||
|
||||
解析后效果
|
||||
|
||||
### 2. 智能表格处理
|
||||
|
||||
针对文档中的表格数据，我们提供了两种灵活的转换方式，满足不同应用场景的需求：
|
||||
* **键值对转换**：将表格行列关系转换为自然语义表达，便于大模型理解和问答。并且**支持合并单元格的处理**，避免生成关系错乱的表格数据。例如，将”产品名称 | 价格 | 库存”的表格转换为”产品名称：A；价格：100元，库存：50件；产品名称：B；价格：150元，库存：30件”等键值对形式。
|
||||
* **Markdown表格转换**：保持表格的原始结构，转换为标准的Markdown表格格式，便于上层应用中可视化展示。
|
||||
|
||||
|
||||
|
||||
原始表格
|
||||
|
||||
|
||||
|
||||
markdown格式表格
|
||||
|
||||
|
||||
|
||||
key-value格式表格
|
||||
|
||||
此外，**针对过长的表格，支持按照长度切分，在不丢失信息的前提下**适配向量数据库的存储，保证后续的检索效果。
|
||||
|
||||
|
||||
|
||||
原始表格
|
||||
|
||||
|
||||
|
||||
表格切分后
|
||||
|
||||
### 3. 基于语义的智能切分
|
||||
|
||||
摒弃了传统机械的字数切分方式，我们引入了 **BGE Embedding 模型**。通过计算文本向量并利用相似度聚类算法，将含义紧密相关的段落自动聚合。这种方式确保了每一个数据切片都是一个完整的”语义单元”，大幅提升了后续检索的准确性。与此同时，模型规模为24M，资源占用率极小，对整体解析流程的影响微乎其微。
|
||||
|
||||
|
||||
|
||||
语义切分前1
|
||||
|
||||
|
||||
|
||||
语义切分后2
|
||||
|
||||
|
||||
|
||||
语义切分前2
|
||||
|
||||
|
||||
|
||||
语义切分后2
|
||||
|
||||
### 4. 上下文感知的标题聚合
|
||||
|
||||
针对企业长文档层级复杂的特点，我们开发了**标题聚合功能**。系统会自动将多级标题信息聚合到对应的段落中。无论文档如何切分，每个切片都能保留”父级标题-子标题”的完整路径，有效解决了碎片化导致的上下文丢失问题，从而提升了后续检索的准确性和相关性。
|
||||
|
||||
|
||||
|
||||
多级标题聚合
|
||||
|
||||
### 5. 关键信息自动提取
|
||||
|
||||
为了进一步提升检索效率，我们集成了中英文的 **NER（命名实体识别）模型**。在解析过程中，系统会自动抽取文档中的组织机构、人名、专有名词等关键实体，为文档打上”智能标签”，实现多维度的精准检索。
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
### 6. 内容检索
|
||||
|
||||
|
||||
|
||||
原始PDF页面
|
||||
|
||||
|
||||
|
||||
搜索关键字“叶片描述”
|
||||
|
||||
返回结果如下：
|
||||
|
||||
```json
|
||||
{
|
||||
"status": "success",
|
||||
"message": "搜索成功",
|
||||
"data": {
|
||||
"result": [
|
||||
{
|
||||
"page_idx": 14,
|
||||
"span_range": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"bbox": [
|
||||
70,
|
||||
586,
|
||||
525,
|
||||
614
|
||||
]
|
||||
},
|
||||
{
|
||||
"page_idx": 14,
|
||||
"span_range": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"bbox": [
|
||||
69,
|
||||
625,
|
||||
147,
|
||||
638
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
应用展示
|
||||
|
||||
|
||||
|
||||
### 7. 性能优化
|
||||
|
||||
通过将CPU密集型任务和GPU密集型任务切分为相互独立的进程，更好利用了高性能服务器的硬件资源，不仅可以进行多路解析，在单任务的处理上也有显著提升。在附加了实体识别和智能分段等附加功能，处理时间和使用MinerU CLI所需的时间几乎没有差别。在多路服务器上可以通过简单配置，使不同的任务使用服务器上独立硬件资源，成倍提速。
|
||||
|
||||
| 解析样例 | 原生MinerU | 本方案 |
|
||||
| --- | --- | --- |
|
||||
| PDF测试1（158页英文内容） | 79.26s | 79.661 |
|
||||
| PDF测试2（731页英文内容） | 402s | 393s |
|
||||
| PDF测试2（670页中文内容） | 163 | 116.53 |
|
||||
|
||||
测试环境：
|
||||
|
||||
- GPU: NVIDIA A100 80GB
|
||||
- CPU: Intel(R) Xeon(R) Gold 6348 CPU @ 2.60GHz
|
||||
|
||||
## 核心组件
|
||||
|
||||
| 依赖名称 | 版本范围 | 功能描述 |
|
||||
| --- | --- | --- |
|
||||
| **fastapi** | `>=0.115.12` | **Web 框架**：提供高性能的 API 服务接口，支持异步编程。 |
|
||||
| **mineru** | `>=2.5.0` | **核心引擎**：MinerU 核心库，提供 PDF 解析、布局分析和公式提取能力。 |
|
||||
| **celery** | `>=5.5.3` | **异步任务队列**：用于处理耗时的 PDF 解析和 NLP任务。 |
|
||||
| **vllm** | `>=0.11.0` | **大模型推理**：高性能 LLM/VLM 推理引擎，用于加速VLM视觉大模型的推理。 |
|
||||
| **transformers** | `>=4.53.0` | **NLP 模型库**：用于加载和运行各种预训练模型（如 NER、Embedding）。 |
|
||||
| **minio** | `>=7.2.15` | **对象存储客户端**：用于上传和下载 PDF 源文件及解析后的结果文件。 |
|
||||
| **sentence-transformers** | `>=5.1.0` | **文本向量化**：用于生成文本 Embedding，实现基于文本语义的切分。 |
|
||||
| **sqlalchemy** | `>=2.0.41` | **ORM 框架**：用于数据库操作 and 连接管理。 |
|
||||
| **redis** | (Implied) | **缓存与消息中间件**：作为 Celery 的 Broker 和应用缓存（通过 `redislite` 或外部服务）。 |
|
||||
|
||||
## 应用价值
|
||||
|
||||
- **提质**：将非结构化文档转化为高质量的 Markdown 数据，为大模型知识库建设提供了坚实的基础。
|
||||
- **增效**：通过自动化、并行化的处理流程，结合软件架构的优化，将文档处理效率提升数倍，实现了从“小时级”到“分钟级”的跨越。
|
||||
- **赋能**：该平台可广泛应用于政策文件检索、合同智能审核、技术文档问答等多种业务场景，切实助力企业提效减负。
|
||||
|
||||
| 对比维度 | 原生 MinerU | 本方案 | 优势 |
|
||||
| --- | --- | --- | --- |
|
||||
| **系统架构** | 单体应用，命令行工具 | 分布式微服务架构（FastAPI + Celery + vLLM + Redis） | 支持高并发、可水平扩展，适合企业级部署 |
|
||||
| **服务化能力** | 无服务接口，需本地调用 | 提供 RESTful API，支持 HTTP 调用 | 易于集成到现有系统，支持多语言调用 |
|
||||
| **并发处理** | 单任务串行处理 | 分布式任务队列，支持多 Worker 并行 | 处理效率提升数倍，支持海量文档批量处理 |
|
||||
| **资源利用** | CPU/GPU 资源利用率低 | CPU 密集任务与 GPU 推理任务分离，多卡并行 | 单机吞吐量大幅提升，资源利用最大化 |
|
||||
| **文档格式支持** | 主要支持 PDF | 全格式支持（Word、Excel、PDF、PPT、图片） | 一站式解决企业多格式文档处理需求 |
|
||||
| **表格处理** | 基础表格识别 | 智能表格处理（键值对转换 + Markdown 表格） | 满足问答 and 可视化展示双重场景需求 |
|
||||
| **文本切分** | 无切分功能 | 基于语义的智能切分（BGE Embedding + 聚类） | 保证语义完整性，提升检索准确性 |
|
||||
| **上下文管理** | 无标题聚合功能 | 上下文感知的标题聚合（多级标题路径保留） | 解决碎片化导致的上下文丢失问题 |
|
||||
| **信息提取** | 无实体识别功能 | 集成 NER 模型，自动提取关键实体 | 支持多维度精准检索，为文档打智能标签 |
|
||||
|
||||
"""
|
||||
def test_semantic_chunking_basic(embed_fn, sample_markdown):
|
||||
"""测试基本的语义切分逻辑 (使用真实嵌入模型)"""
|
||||
|
||||
# 配置切分参数
|
||||
parser_config = {
|
||||
"chunk_token_num": 1000, # 针对标准文档调整 token 数
|
||||
"overlapped_percent": 0.1
|
||||
}
|
||||
|
||||
# 执行语义切分
|
||||
chunks = semantic.chunk_markdown(
|
||||
sample_markdown,
|
||||
parser_config=parser_config,
|
||||
embed_fn=embed_fn
|
||||
)
|
||||
|
||||
# 1. 基础验证
|
||||
assert isinstance(chunks, list)
|
||||
assert len(chunks) >= 5, f"预期至少切分为 5 个片段，实际仅有 {len(chunks)} 个"
|
||||
|
||||
# 2. 验证上下文感知与标题聚合 (检查 RAGFlow 风格的层级路径)
|
||||
# 验证子章节片段是否保留了父级标题路径
|
||||
# 检查 "Docling" 相关的片段 (属于 "核心技术亮点" -> "多模态融合的高精度解析")
|
||||
docling_chunks = [c for c in chunks if "Docling" in c]
|
||||
assert docling_chunks, "未找到包含 'Docling' 的片段"
|
||||
for chunk in docling_chunks:
|
||||
assert "核心技术亮点" in chunk, "子章节片段丢失了父级标题 '核心技术亮点'"
|
||||
assert "多模态融合的高精度解析" in chunk, "子章节片段丢失了自身标题 '多模态融合的高精度解析'"
|
||||
# 验证层级分隔符
|
||||
path_part = chunk.split("\n")[0] if "\n" in chunk else chunk
|
||||
assert "|" in path_part, f"标题路径中缺失层级分隔符 '|': {path_part}"
|
||||
assert path_part.count("|") >= 1, f"标题路径层级深度不足: {path_part}"
|
||||
|
||||
# 检查顶级章节 (Level 2) 是否保持独立 (不应包含其他不相关的顶级标题)
|
||||
pain_point_chunks = [c for c in chunks if "格式繁杂兼容难" in c]
|
||||
for chunk in pain_point_chunks:
|
||||
path_part = chunk.split("\n")[0]
|
||||
assert "痛点与挑战" in path_part
|
||||
assert "技术方案概览" not in path_part, "顶级标题路径污染：包含了不相关的顶级标题"
|
||||
|
||||
# 3. 验证关键技术词汇识别
|
||||
# 重点检查文档中出现的核心组件词汇
|
||||
keywords_to_check = ["FastAPI", "Celery", "vLLM", "MinerU"]
|
||||
found_keywords = [s for s in keywords_to_check if any(s in chunk for chunk in chunks)]
|
||||
assert len(found_keywords) > 0, f"在切分结果中未找到关键技术词汇: {keywords_to_check}"
|
||||
|
||||
# 4. 验证核心章节内容是否存在
|
||||
has_arch_section = any("架构设计" in chunk for chunk in chunks)
|
||||
has_highlight_section = any("核心技术亮点" in chunk for chunk in chunks)
|
||||
assert has_arch_section, "未找到 '架构设计' 相关内容"
|
||||
assert has_highlight_section, "未找到 '核心技术亮点' 相关内容"
|
||||
|
||||
# 5. 验证语义聚类是否将不同主题分开
|
||||
# 架构设计和核心技术亮点是两个独立的大章节，语义聚类应该尽量避免将它们的核心内容混在一个 chunk 中
|
||||
arch_start_chunks = [i for i, c in enumerate(chunks) if "## 架构设计" in c]
|
||||
highlight_start_chunks = [i for i, c in enumerate(chunks) if "## 核心技术亮点" in c]
|
||||
|
||||
if arch_start_chunks and highlight_start_chunks:
|
||||
# 确保起始片段不重合
|
||||
assert not set(arch_start_chunks).intersection(set(highlight_start_chunks)), \
|
||||
"语义聚类错误：架构设计与核心技术亮点的章节头部被挤在了同一个 chunk 中"
|
||||
|
||||
print(f"\n[测试成功] 文档成功切分为 {len(chunks)} 个片段")
|
||||
print(f"识别到的关键技术词汇: {found_keywords}")
|
||||
|
||||
# 直接输出到控制台预览切分结果，避免文件副作用
|
||||
print("\n--- 语义切分结果开始 ---")
|
||||
for idx, chunk in enumerate(chunks, 1):
|
||||
print(f"\n[Chunk {idx}]\n{chunk}")
|
||||
print("\n--- 语义切分结果结束 ---")
|
||||
|
||||
def test_heading_inference():
|
||||
"""测试标题层级推断工具类"""
|
||||
from yuxi.knowledge.chunking.ragflow_like.utils.md_parser_utils import infer_heading_level
|
||||
|
||||
assert infer_heading_level("1. 简介") == 1
|
||||
assert infer_heading_level("1.1 详细设计") == 2
|
||||
assert infer_heading_level("1.2.3 核心逻辑") == 3
|
||||
assert infer_heading_level("一、 背景") == 1
|
||||
assert infer_heading_level("普通文本") == 1
|
||||
@ -7,8 +7,10 @@ sys.path.append(os.getcwd())
|
||||
|
||||
from yuxi.knowledge.chunking.ragflow_like.dispatcher import chunk_markdown
|
||||
from yuxi.knowledge.chunking.ragflow_like.nlp import bullets_category, count_tokens
|
||||
from yuxi.knowledge.chunking.ragflow_like.utils.semantic_utils import split_sentences_chinese
|
||||
from yuxi.knowledge.chunking.ragflow_like.presets import (
|
||||
CHUNK_ENGINE_VERSION,
|
||||
CHUNK_PRESET_IDS,
|
||||
get_chunk_preset_options,
|
||||
map_to_internal_parser_id,
|
||||
resolve_chunk_processing_params,
|
||||
@ -85,6 +87,13 @@ def test_book_chunking_hierarchical_merge() -> None:
|
||||
assert any("第一章" in ck["content"] for ck in chunks)
|
||||
|
||||
|
||||
def test_split_sentences_chinese_should_keep_quote_boundary() -> None:
|
||||
text = '他说：“你好。”然后问：“你在吗？”最后结束！'
|
||||
sentences = split_sentences_chinese(text)
|
||||
|
||||
assert sentences == ["他说：“你好。”", "然后问：“你在吗？”", "最后结束！"]
|
||||
|
||||
|
||||
def test_markdown_heading_has_higher_weight_in_bullet_category() -> None:
|
||||
sections = [
|
||||
"# 3.2 个人所得项目及计税、申报方式概括",
|
||||
@ -108,7 +117,7 @@ def test_mid_sentence_bullet_marker_should_not_be_treated_as_heading() -> None:
|
||||
|
||||
def test_chunk_preset_options_include_description() -> None:
|
||||
options = get_chunk_preset_options()
|
||||
assert len(options) == 4
|
||||
assert {option["value"] for option in options} == CHUNK_PRESET_IDS
|
||||
assert all(isinstance(option.get("description"), str) and option["description"] for option in options)
|
||||
|
||||
|
||||
|
||||
@ -2521,6 +2521,18 @@ wheels = [
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/9c/46/f6b4ad632c67ef35209a66127e4bddc95759649dd595f71f13fba11bdf9a/mcp-1.27.0-py3-none-any.whl", hash = "sha256:5ce1fa81614958e267b21fb2aa34e0aea8e2c6ede60d52aba45fd47246b4d741", size = 215967, upload-time = "2026-04-02T14:48:07.24Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mdit-py-plugins"
|
||||
version = "0.5.0"
|
||||
source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" }
|
||||
dependencies = [
|
||||
{ name = "markdown-it-py" },
|
||||
]
|
||||
sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/b2/fd/a756d36c0bfba5f6e39a1cdbdbfdd448dc02692467d83816dff4592a1ebc/mdit_py_plugins-0.5.0.tar.gz", hash = "sha256:f4918cb50119f50446560513a8e311d574ff6aaed72606ddae6d35716fe809c6", size = 44655, upload-time = "2025-08-11T07:25:49.083Z" }
|
||||
wheels = [
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/fb/86/dd6e5db36df29e76c7a7699123569a4a18c1623ce68d826ed96c62643cae/mdit_py_plugins-0.5.0-py3-none-any.whl", hash = "sha256:07a08422fc1936a5d26d146759e9155ea466e842f5ab2f7d2266dd084c8dab1f", size = 57205, upload-time = "2025-08-11T07:25:47.597Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mdurl"
|
||||
version = "0.1.2"
|
||||
@ -4337,6 +4349,38 @@ torch = [
|
||||
{ name = "torch", version = "2.8.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform != 'darwin'" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "scikit-learn"
|
||||
version = "1.8.0"
|
||||
source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" }
|
||||
dependencies = [
|
||||
{ name = "joblib" },
|
||||
{ name = "numpy" },
|
||||
{ name = "scipy" },
|
||||
{ name = "threadpoolctl" },
|
||||
]
|
||||
sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/0e/d4/40988bf3b8e34feec1d0e6a051446b1f66225f8529b9309becaeef62b6c4/scikit_learn-1.8.0.tar.gz", hash = "sha256:9bccbb3b40e3de10351f8f5068e105d0f4083b1a65fa07b6634fbc401a6287fd", size = 7335585, upload-time = "2025-12-10T07:08:53.618Z" }
|
||||
wheels = [
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/90/74/e6a7cc4b820e95cc38cf36cd74d5aa2b42e8ffc2d21fe5a9a9c45c1c7630/scikit_learn-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5fb63362b5a7ddab88e52b6dbb47dac3fd7dafeee740dc6c8d8a446ddedade8e", size = 8548242, upload-time = "2025-12-10T07:07:51.568Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/49/d8/9be608c6024d021041c7f0b3928d4749a706f4e2c3832bbede4fb4f58c95/scikit_learn-1.8.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:5025ce924beccb28298246e589c691fe1b8c1c96507e6d27d12c5fadd85bfd76", size = 8079075, upload-time = "2025-12-10T07:07:53.697Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/dd/47/f187b4636ff80cc63f21cd40b7b2d177134acaa10f6bb73746130ee8c2e5/scikit_learn-1.8.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4496bb2cf7a43ce1a2d7524a79e40bc5da45cf598dbf9545b7e8316ccba47bb4", size = 8660492, upload-time = "2025-12-10T07:07:55.574Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/97/74/b7a304feb2b49df9fafa9382d4d09061a96ee9a9449a7cbea7988dda0828/scikit_learn-1.8.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a0bcfe4d0d14aec44921545fd2af2338c7471de9cb701f1da4c9d85906ab847a", size = 8931904, upload-time = "2025-12-10T07:07:57.666Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/9f/c4/0ab22726a04ede56f689476b760f98f8f46607caecff993017ac1b64aa5d/scikit_learn-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:35c007dedb2ffe38fe3ee7d201ebac4a2deccd2408e8621d53067733e3c74809", size = 8019359, upload-time = "2025-12-10T07:07:59.838Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/24/90/344a67811cfd561d7335c1b96ca21455e7e472d281c3c279c4d3f2300236/scikit_learn-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:8c497fff237d7b4e07e9ef1a640887fa4fb765647f86fbe00f969ff6280ce2bb", size = 7641898, upload-time = "2025-12-10T07:08:01.36Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/03/aa/e22e0768512ce9255eba34775be2e85c2048da73da1193e841707f8f039c/scikit_learn-1.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0d6ae97234d5d7079dc0040990a6f7aeb97cb7fa7e8945f1999a429b23569e0a", size = 8513770, upload-time = "2025-12-10T07:08:03.251Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/58/37/31b83b2594105f61a381fc74ca19e8780ee923be2d496fcd8d2e1147bd99/scikit_learn-1.8.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:edec98c5e7c128328124a029bceb09eda2d526997780fef8d65e9a69eead963e", size = 8044458, upload-time = "2025-12-10T07:08:05.336Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/2d/5a/3f1caed8765f33eabb723596666da4ebbf43d11e96550fb18bdec42b467b/scikit_learn-1.8.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:74b66d8689d52ed04c271e1329f0c61635bcaf5b926db9b12d58914cdc01fe57", size = 8610341, upload-time = "2025-12-10T07:08:07.732Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/38/cf/06896db3f71c75902a8e9943b444a56e727418f6b4b4a90c98c934f51ed4/scikit_learn-1.8.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8fdf95767f989b0cfedb85f7ed8ca215d4be728031f56ff5a519ee1e3276dc2e", size = 8900022, upload-time = "2025-12-10T07:08:09.862Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/1c/f9/9b7563caf3ec8873e17a31401858efab6b39a882daf6c1bfa88879c0aa11/scikit_learn-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:2de443b9373b3b615aec1bb57f9baa6bb3a9bd093f1269ba95c17d870422b271", size = 7989409, upload-time = "2025-12-10T07:08:12.028Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/49/bd/1f4001503650e72c4f6009ac0c4413cb17d2d601cef6f71c0453da2732fc/scikit_learn-1.8.0-cp313-cp313-win_arm64.whl", hash = "sha256:eddde82a035681427cbedded4e6eff5e57fa59216c2e3e90b10b19ab1d0a65c3", size = 7619760, upload-time = "2025-12-10T07:08:13.688Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/d2/7d/a630359fc9dcc95496588c8d8e3245cc8fd81980251079bc09c70d41d951/scikit_learn-1.8.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:7cc267b6108f0a1499a734167282c00c4ebf61328566b55ef262d48e9849c735", size = 8826045, upload-time = "2025-12-10T07:08:15.215Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/cc/56/a0c86f6930cfcd1c7054a2bc417e26960bb88d32444fe7f71d5c2cfae891/scikit_learn-1.8.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:fe1c011a640a9f0791146011dfd3c7d9669785f9fed2b2a5f9e207536cf5c2fd", size = 8420324, upload-time = "2025-12-10T07:08:17.561Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/46/1e/05962ea1cebc1cf3876667ecb14c283ef755bf409993c5946ade3b77e303/scikit_learn-1.8.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:72358cce49465d140cc4e7792015bb1f0296a9742d5622c67e31399b75468b9e", size = 8680651, upload-time = "2025-12-10T07:08:19.952Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/fe/56/a85473cd75f200c9759e3a5f0bcab2d116c92a8a02ee08ccd73b870f8bb4/scikit_learn-1.8.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:80832434a6cc114f5219211eec13dcbc16c2bac0e31ef64c6d346cde3cf054cb", size = 8925045, upload-time = "2025-12-10T07:08:22.11Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/cc/b7/64d8cfa896c64435ae57f4917a548d7ac7a44762ff9802f75a79b77cb633/scikit_learn-1.8.0-cp313-cp313t-win_amd64.whl", hash = "sha256:ee787491dbfe082d9c3013f01f5991658b0f38aa8177e4cd4bf434c58f551702", size = 8507994, upload-time = "2025-12-10T07:08:23.943Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/5e/37/e192ea709551799379958b4c4771ec507347027bb7c942662c7fbeba31cb/scikit_learn-1.8.0-cp313-cp313t-win_arm64.whl", hash = "sha256:bf97c10a3f5a7543f9b88cbf488d33d175e9146115a451ae34568597ba33dcde", size = 7869518, upload-time = "2025-12-10T07:08:25.71Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "scipy"
|
||||
version = "1.17.1"
|
||||
@ -4756,6 +4800,15 @@ wheels = [
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/f5/15/a11f7bb3cbc97dfecf32a90552f5a8f8a5c99316a99c6c17bdabf5baf256/thinc-8.3.13-cp313-cp313-win_arm64.whl", hash = "sha256:723949cab11d1925c15447928513a718276316cec6e0de28337cca0a62be0521", size = 1644606, upload-time = "2026-03-23T07:22:21.339Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "threadpoolctl"
|
||||
version = "3.6.0"
|
||||
source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" }
|
||||
sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/b7/4d/08c89e34946fce2aec4fbb45c9016efd5f4d7f24af8e5d93296e935631d8/threadpoolctl-3.6.0.tar.gz", hash = "sha256:8ab8b4aa3491d812b623328249fab5302a68d2d71745c8a4c719a2fcaba9f44e", size = 21274, upload-time = "2025-03-13T13:49:23.031Z" }
|
||||
wheels = [
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl", hash = "sha256:43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb", size = 18638, upload-time = "2025-03-13T13:49:21.846Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tiktoken"
|
||||
version = "0.12.0"
|
||||
@ -4878,9 +4931,9 @@ dependencies = [
|
||||
{ name = "typing-extensions", marker = "sys_platform == 'darwin'" },
|
||||
]
|
||||
wheels = [
|
||||
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:a47b7986bee3f61ad217d8a8ce24605809ab425baf349f97de758815edd2ef54" },
|
||||
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:fbe2e149c5174ef90d29a5f84a554dfaf28e003cb4f61fa2c8c024c17ec7ca58" },
|
||||
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0-cp313-none-macosx_11_0_arm64.whl", hash = "sha256:057efd30a6778d2ee5e2374cd63a63f63311aa6f33321e627c655df60abdd390" },
|
||||
{ url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:a47b7986bee3f61ad217d8a8ce24605809ab425baf349f97de758815edd2ef54" },
|
||||
{ url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:fbe2e149c5174ef90d29a5f84a554dfaf28e003cb4f61fa2c8c024c17ec7ca58" },
|
||||
{ url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0-cp313-none-macosx_11_0_arm64.whl", hash = "sha256:057efd30a6778d2ee5e2374cd63a63f63311aa6f33321e627c655df60abdd390" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -4903,19 +4956,19 @@ dependencies = [
|
||||
{ name = "typing-extensions", marker = "sys_platform != 'darwin'" },
|
||||
]
|
||||
wheels = [
|
||||
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-linux_s390x.whl", hash = "sha256:0e34e276722ab7dd0dffa9e12fe2135a9b34a0e300c456ed7ad6430229404eb5" },
|
||||
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:610f600c102386e581327d5efc18c0d6edecb9820b4140d26163354a99cd800d" },
|
||||
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:cb9a8ba8137ab24e36bf1742cb79a1294bd374db570f09fc15a5e1318160db4e" },
|
||||
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-win_amd64.whl", hash = "sha256:2be20b2c05a0cce10430cc25f32b689259640d273232b2de357c35729132256d" },
|
||||
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-win_arm64.whl", hash = "sha256:99fc421a5d234580e45957a7b02effbf3e1c884a5dd077afc85352c77bf41434" },
|
||||
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313-linux_s390x.whl", hash = "sha256:8b5882276633cf91fe3d2d7246c743b94d44a7e660b27f1308007fdb1bb89f7d" },
|
||||
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:a5064b5e23772c8d164068cc7c12e01a75faf7b948ecd95a0d4007d7487e5f25" },
|
||||
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:8f81dedb4c6076ec325acc3b47525f9c550e5284a18eae1d9061c543f7b6e7de" },
|
||||
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313-win_amd64.whl", hash = "sha256:e1ee1b2346ade3ea90306dfbec7e8ff17bc220d344109d189ae09078333b0856" },
|
||||
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313-win_arm64.whl", hash = "sha256:64c187345509f2b1bb334feed4666e2c781ca381874bde589182f81247e61f88" },
|
||||
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:af81283ac671f434b1b25c95ba295f270e72db1fad48831eb5e4748ff9840041" },
|
||||
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:a9dbb6f64f63258bc811e2c0c99640a81e5af93c531ad96e95c5ec777ea46dab" },
|
||||
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313t-win_amd64.whl", hash = "sha256:6d93a7165419bc4b2b907e859ccab0dea5deeab261448ae9a5ec5431f14c0e64" },
|
||||
{ url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-linux_s390x.whl", hash = "sha256:0e34e276722ab7dd0dffa9e12fe2135a9b34a0e300c456ed7ad6430229404eb5" },
|
||||
{ url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:610f600c102386e581327d5efc18c0d6edecb9820b4140d26163354a99cd800d" },
|
||||
{ url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:cb9a8ba8137ab24e36bf1742cb79a1294bd374db570f09fc15a5e1318160db4e" },
|
||||
{ url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-win_amd64.whl", hash = "sha256:2be20b2c05a0cce10430cc25f32b689259640d273232b2de357c35729132256d" },
|
||||
{ url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-win_arm64.whl", hash = "sha256:99fc421a5d234580e45957a7b02effbf3e1c884a5dd077afc85352c77bf41434" },
|
||||
{ url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313-linux_s390x.whl", hash = "sha256:8b5882276633cf91fe3d2d7246c743b94d44a7e660b27f1308007fdb1bb89f7d" },
|
||||
{ url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:a5064b5e23772c8d164068cc7c12e01a75faf7b948ecd95a0d4007d7487e5f25" },
|
||||
{ url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:8f81dedb4c6076ec325acc3b47525f9c550e5284a18eae1d9061c543f7b6e7de" },
|
||||
{ url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313-win_amd64.whl", hash = "sha256:e1ee1b2346ade3ea90306dfbec7e8ff17bc220d344109d189ae09078333b0856" },
|
||||
{ url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313-win_arm64.whl", hash = "sha256:64c187345509f2b1bb334feed4666e2c781ca381874bde589182f81247e61f88" },
|
||||
{ url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:af81283ac671f434b1b25c95ba295f270e72db1fad48831eb5e4748ff9840041" },
|
||||
{ url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:a9dbb6f64f63258bc811e2c0c99640a81e5af93c531ad96e95c5ec777ea46dab" },
|
||||
{ url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313t-win_amd64.whl", hash = "sha256:6d93a7165419bc4b2b907e859ccab0dea5deeab261448ae9a5ec5431f14c0e64" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -5616,6 +5669,7 @@ dependencies = [
|
||||
{ name = "aiosqlite" },
|
||||
{ name = "argon2-cffi" },
|
||||
{ name = "asyncpg" },
|
||||
{ name = "beautifulsoup4" },
|
||||
{ name = "chardet" },
|
||||
{ name = "colorlog" },
|
||||
{ name = "dashscope" },
|
||||
@ -5642,11 +5696,14 @@ dependencies = [
|
||||
{ name = "llama-index" },
|
||||
{ name = "llama-index-readers-file" },
|
||||
{ name = "loguru" },
|
||||
{ name = "markdown-it-py" },
|
||||
{ name = "markdownify" },
|
||||
{ name = "mcp" },
|
||||
{ name = "mdit-py-plugins" },
|
||||
{ name = "minio" },
|
||||
{ name = "neo4j" },
|
||||
{ name = "networkx" },
|
||||
{ name = "nltk" },
|
||||
{ name = "openai" },
|
||||
{ name = "opencv-python-headless" },
|
||||
{ name = "pillow" },
|
||||
@ -5664,6 +5721,7 @@ dependencies = [
|
||||
{ name = "readability-lxml" },
|
||||
{ name = "redis" },
|
||||
{ name = "rich" },
|
||||
{ name = "scikit-learn" },
|
||||
{ name = "sqlalchemy", extra = ["asyncio"] },
|
||||
{ name = "tabulate" },
|
||||
{ name = "tavily-python" },
|
||||
@ -5689,6 +5747,7 @@ requires-dist = [
|
||||
{ name = "aiosqlite", specifier = ">=0.20.0" },
|
||||
{ name = "argon2-cffi", specifier = ">=25.1.0" },
|
||||
{ name = "asyncpg", specifier = ">=0.30.0" },
|
||||
{ name = "beautifulsoup4", specifier = ">=4.12.0" },
|
||||
{ name = "chardet", specifier = ">=5.0.0" },
|
||||
{ name = "colorlog", specifier = ">=6.9.0" },
|
||||
{ name = "dashscope", specifier = ">=1.23.2" },
|
||||
@ -5715,11 +5774,14 @@ requires-dist = [
|
||||
{ name = "llama-index", specifier = ">=0.14" },
|
||||
{ name = "llama-index-readers-file", specifier = ">=0.4.7" },
|
||||
{ name = "loguru", specifier = ">=0.7.3" },
|
||||
{ name = "markdown-it-py", specifier = ">=3.0.0" },
|
||||
{ name = "markdownify", specifier = ">=1.1.0" },
|
||||
{ name = "mcp", specifier = ">=1.20" },
|
||||
{ name = "mdit-py-plugins", specifier = ">=0.4.0" },
|
||||
{ name = "minio", specifier = ">=7.2.7" },
|
||||
{ name = "neo4j", specifier = ">=5.28.1" },
|
||||
{ name = "networkx", specifier = ">=3.5" },
|
||||
{ name = "nltk", specifier = ">=3.8.1" },
|
||||
{ name = "openai", specifier = ">=1.109" },
|
||||
{ name = "opencv-python-headless", specifier = ">=4.11.0.86" },
|
||||
{ name = "pillow", specifier = ">=10.5.0" },
|
||||
@ -5737,6 +5799,7 @@ requires-dist = [
|
||||
{ name = "readability-lxml", specifier = ">=0.8.1" },
|
||||
{ name = "redis", specifier = ">=5.2.0" },
|
||||
{ name = "rich", specifier = ">=13.7.1" },
|
||||
{ name = "scikit-learn", specifier = ">=1.3.0" },
|
||||
{ name = "sqlalchemy", extras = ["asyncio"], specifier = ">=2.0.0" },
|
||||
{ name = "tabulate", specifier = ">=0.9.0" },
|
||||
{ name = "tavily-python", specifier = ">=0.7.0" },
|
||||
|
||||
@ -52,6 +52,7 @@ services:
|
||||
- ./.env:/app/.env
|
||||
- ${MODEL_DIR:-./models}:/models # 使用默认值处理未定义的环境变量
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
- nltk_data:/root/nltk_data # 使用命名卷持久化NLTK数据，避免路径展开差异
|
||||
ports:
|
||||
- "5050:5050"
|
||||
networks:
|
||||
@ -430,6 +431,9 @@ services:
|
||||
- app-network
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
nltk_data:
|
||||
|
||||
networks:
|
||||
app-network:
|
||||
driver: bridge
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
"name": "yuxi-web",
|
||||
"version": "0.6.1",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
"server": "vite serve --host",
|
||||
|
||||
@ -18,6 +18,11 @@ export const CHUNK_PRESET_OPTIONS = [
|
||||
value: 'laws',
|
||||
label: 'Laws',
|
||||
description: '法规分块：按法条层级组织与合并，适合法律法规、制度规范类文本。'
|
||||
},
|
||||
{
|
||||
value: 'semantic',
|
||||
label: 'Semantic',
|
||||
description: '语义分块：利用嵌入和聚类算法进行语义切分，并自动增强标题上下文。'
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user