docs: 为知识分块工具函数添加详细注释

- 为 html_table_to_key_value 函数添加处理逻辑说明,解释如何应对过长表格的切分问题
- 为 infer_heading_level 函数添加层级推断规则说明,包括数字序号和中文序号的处理
- 为 extract_table_block 函数添加表格提取算法的详细步骤说明
- 为 split_text_by_length_and_newline 函数添加分块逻辑的详细注释,解释段落和行的处理流程
- 为语义分块相关函数添加中文文档,包括句子分割、聚类算法和整体流程说明
- 更新测试用例,使用更清晰的测试文档并添加结果保存功能
This commit is contained in:
zhou shujian 2026-04-16 17:00:59 +08:00
parent a2ed3d9a10
commit edf66bbe38
16 changed files with 2720 additions and 294 deletions

View File

@ -2,100 +2,107 @@ from __future__ import annotations
import re import re
from typing import Any from typing import Any
from markdown_it import MarkdownIt from markdown_it import MarkdownIt
from mdit_py_plugins.dollarmath import dollarmath_plugin from mdit_py_plugins.dollarmath import dollarmath_plugin
from yuxi.utils.logging_config import logger
from yuxi.knowledge.chunking.ragflow_like.nlp import count_tokens from yuxi.knowledge.chunking.ragflow_like.nlp import count_tokens
from yuxi.utils.logging_config import logger
from ..utils.md_parser_utils import ( from ..utils.md_parser_utils import (
infer_heading_level, extract_table_block,
get_title_path, get_title_path,
extract_table_block, split_text_by_length_and_newline,
split_text_by_length_and_newline
) )
from ..utils.table_utils import html_table_to_key_value from ..utils.table_utils import html_table_to_key_value
def _flush_content( def _flush_content(
result: list, result: list,
current_content: list, current_content: list,
title_stack: list, title_stack: list,
max_length: int, max_length: int,
embed_fn: Any, embed_fn: Any,
special_element: str = None, special_element: str = None,
allow_split: bool = False allow_split: bool = False,
) -> None: ) -> None:
if not current_content: if not current_content:
return return
content = '\n'.join(current_content).strip() content = "\n".join(current_content).strip()
if not content: if not content:
current_content.clear() current_content.clear()
return return
level = next((i + 1 for i in range(5, -1, -1) if title_stack[i]), 1) level = next((i + 1 for i in range(5, -1, -1) if title_stack[i]), 1)
title_path = get_title_path(title_stack) title_path = get_title_path(title_stack)
if special_element and not allow_split: if special_element and not allow_split:
header = f"{'#' * level} {title_path}|{special_element}" if title_path else f"{'#' * level} {special_element}" header = f"{'#' * level} {title_path}|{special_element}" if title_path else f"{'#' * level} {special_element}"
result.extend([header, content, '-' * 10]) result.extend([header, content, "-" * 10])
else: else:
if count_tokens(content) > max_length: if count_tokens(content) > max_length:
chunks = split_text_by_length_and_newline(content, max_length, embed_fn=embed_fn, token_count_fn=count_tokens) chunks = split_text_by_length_and_newline(
content, max_length, embed_fn=embed_fn, token_count_fn=count_tokens
)
for idx, chunk in enumerate(chunks, 1): for idx, chunk in enumerate(chunks, 1):
base_header = f"{'#' * level} {title_path}" if title_path else f"{'#' * level}" base_header = f"{'#' * level} {title_path}" if title_path else f"{'#' * level}"
if special_element: if special_element:
header = f"{base_header}|{special_element}|Part {idx}" header = f"{base_header}|{special_element}|Part {idx}"
else: else:
header = f"{base_header}|Part {idx}" header = f"{base_header}|Part {idx}"
result.extend([header, chunk, '-' * 10]) result.extend([header, chunk, "-" * 10])
else: else:
base_header = f"{'#' * level} {title_path}" if title_path else f"{'#' * level}" base_header = f"{'#' * level} {title_path}" if title_path else f"{'#' * level}"
if special_element: if special_element:
header = f"{base_header}|{special_element}" header = f"{base_header}|{special_element}"
else: else:
header = base_header header = base_header
if header: if header:
result.append(header) result.append(header)
result.append("") result.append("")
result.extend([content, '-' * 10]) result.extend([content, "-" * 10])
current_content.clear() current_content.clear()
def _handle_image_caption(tokens, i, result, current_content, title_stack, max_length, embed_fn): def _handle_image_caption(tokens, i, result, current_content, title_stack, max_length, embed_fn):
token = tokens[i] token = tokens[i]
if token.type != 'paragraph_open': if token.type != "paragraph_open":
return False, i return False, i
inline_token = tokens[i + 1]
if inline_token.type != 'inline':
return False, i
content = inline_token.content.strip()
image_pattern = r'^!\[.*?\]\(.*?\)\s*$'
caption_pattern = r'^(?:Figure|图|Fig\.|表|Table)\s*[\d\w\.]+'
img_match = re.search(r'^(!\[.*?\]\(.*?\))', content) inline_token = tokens[i + 1]
if inline_token.type != "inline":
return False, i
content = inline_token.content.strip()
image_pattern = r"^!\[.*?\]\(.*?\)\s*$"
caption_pattern = r"^(?:Figure|图|Fig\.|表|Table)\s*[\d\w\.]+"
img_match = re.search(r"^(!\[.*?\]\(.*?\))", content)
if img_match: if img_match:
rest = content[img_match.end():].strip() rest = content[img_match.end() :].strip()
if rest and re.match(caption_pattern, rest, re.IGNORECASE): if rest and re.match(caption_pattern, rest, re.IGNORECASE):
_flush_content(result, current_content, title_stack, max_length, embed_fn) _flush_content(result, current_content, title_stack, max_length, embed_fn)
current_content.append(content) current_content.append(content)
caption_title = rest.split('\n')[0].strip() caption_title = rest.split("\n")[0].strip()
_flush_content(result, current_content, title_stack, max_length, embed_fn, special_element=caption_title) _flush_content(result, current_content, title_stack, max_length, embed_fn, special_element=caption_title)
return True, i + 3 return True, i + 3
if re.match(image_pattern, content): if re.match(image_pattern, content):
next_p_idx = i + 3 next_p_idx = i + 3
if next_p_idx + 1 < len(tokens) and tokens[next_p_idx].type == 'paragraph_open': if next_p_idx + 1 < len(tokens) and tokens[next_p_idx].type == "paragraph_open":
next_inline = tokens[next_p_idx + 1] next_inline = tokens[next_p_idx + 1]
if next_inline.type == 'inline': if next_inline.type == "inline":
next_content = next_inline.content.strip() next_content = next_inline.content.strip()
if re.match(caption_pattern, next_content, re.IGNORECASE): if re.match(caption_pattern, next_content, re.IGNORECASE):
_flush_content(result, current_content, title_stack, max_length, embed_fn) _flush_content(result, current_content, title_stack, max_length, embed_fn)
current_content.append(content) current_content.append(content)
current_content.append(next_content) current_content.append(next_content)
_flush_content(result, current_content, title_stack, max_length, embed_fn, special_element=next_content) _flush_content(
result, current_content, title_stack, max_length, embed_fn, special_element=next_content
)
return True, i + 6 return True, i + 6
if current_content and re.match(caption_pattern, content, re.IGNORECASE): if current_content and re.match(caption_pattern, content, re.IGNORECASE):
@ -110,14 +117,13 @@ def _handle_image_caption(tokens, i, result, current_content, title_stack, max_l
return False, i return False, i
def chunk_markdown( def chunk_markdown(
markdown_content: str, markdown_content: str, parser_config: dict[str, Any] | None = None, embed_fn: Any | None = None
parser_config: dict[str, Any] | None = None,
embed_fn: Any | None = None
) -> list[str]: ) -> list[str]:
""" """
语义化切分 Markdown 内容 语义化切分 Markdown 内容
Args: Args:
markdown_content: 待切分的 Markdown 文本 markdown_content: 待切分的 Markdown 文本
parser_config: 切分参数 chunk_token_num parser_config: 切分参数 chunk_token_num
@ -127,7 +133,7 @@ def chunk_markdown(
parser_config = parser_config or {} parser_config = parser_config or {}
max_length = int(parser_config.get("chunk_token_num", 512)) max_length = int(parser_config.get("chunk_token_num", 512))
logger.info(f"语义切分开始: max_length={max_length}, content_length={len(markdown_content)}") logger.info(f"语义切分开始: max_length={max_length}, content_length={len(markdown_content)}")
# 延迟加载重型资源,仅在没有注入 embed_fn 时触发 # 延迟加载重型资源,仅在没有注入 embed_fn 时触发
if embed_fn is None: if embed_fn is None:
try: try:
@ -142,61 +148,67 @@ def chunk_markdown(
logger.error(f"加载 Embedding 模型失败: {e}。将退化为简单切分。") logger.error(f"加载 Embedding 模型失败: {e}。将退化为简单切分。")
embed_fn = None embed_fn = None
md = MarkdownIt('commonmark').enable('table') md = MarkdownIt("commonmark").enable("table")
md.use(dollarmath_plugin, allow_space=True, allow_digits=True) md.use(dollarmath_plugin, allow_space=True, allow_digits=True)
tokens: list = md.parse(markdown_content) tokens: list = md.parse(markdown_content)
original_lines: list = markdown_content.split('\n') original_lines: list = markdown_content.split("\n")
result: list = [] result: list = []
current_content: list = [] current_content: list = []
title_stack: list = [''] * 6 title_stack: list = [""] * 6
i = 0 i = 0
while i < len(tokens): while i < len(tokens):
token = tokens[i] token = tokens[i]
if token.type == 'heading_open': if token.type == "heading_open":
_flush_content(result, current_content, title_stack, max_length, embed_fn) _flush_content(result, current_content, title_stack, max_length, embed_fn)
level = int(token.tag[1:]) if token.tag and len(token.tag) > 1 else 1 level = int(token.tag[1:]) if token.tag and len(token.tag) > 1 else 1
inline_token = tokens[i + 1] inline_token = tokens[i + 1]
if inline_token.type == 'inline': if inline_token.type == "inline":
full_title = inline_token.content.strip() full_title = inline_token.content.strip()
title_stack[level - 1] = full_title title_stack[level - 1] = full_title
for j in range(level, 6): for j in range(level, 6):
title_stack[j] = '' title_stack[j] = ""
i += 3 i += 3
continue continue
elif token.type == 'table_open': elif token.type == "table_open":
_flush_content(result, current_content, title_stack, max_length, embed_fn) _flush_content(result, current_content, title_stack, max_length, embed_fn)
j, table_content = extract_table_block(tokens, i, original_lines) j, table_content = extract_table_block(tokens, i, original_lines)
current_content.append(table_content) current_content.append(table_content)
_flush_content(result, current_content, title_stack, max_length, embed_fn, special_element='Table') _flush_content(result, current_content, title_stack, max_length, embed_fn, special_element="Table")
i = j + 1 if j < len(tokens) else len(tokens) i = j + 1 if j < len(tokens) else len(tokens)
continue continue
elif token.type == 'paragraph_open': elif token.type == "paragraph_open":
handled, new_i = _handle_image_caption(tokens, i, result, current_content, title_stack, max_length, embed_fn) handled, new_i = _handle_image_caption(
tokens, i, result, current_content, title_stack, max_length, embed_fn
)
if handled: if handled:
i = new_i i = new_i
continue continue
inline_token = tokens[i + 1] inline_token = tokens[i + 1]
if inline_token.type == 'inline': if inline_token.type == "inline":
current_content.append(inline_token.content.strip()) current_content.append(inline_token.content.strip())
i += 3 i += 3
continue continue
elif token.type == 'fence': elif token.type == "fence":
current_content.append(f"```\n{token.content}\n```") current_content.append(f"```\n{token.content}\n```")
i += 1 i += 1
continue continue
elif token.type == 'ordered_list_open': elif token.type == "ordered_list_open":
_flush_content(result, current_content, title_stack, max_length, embed_fn) _flush_content(result, current_content, title_stack, max_length, embed_fn)
list_content = [] list_content = []
j = i + 1 j = i + 1
list_item_counter = 1 list_item_counter = 1
while j < len(tokens) and tokens[j].type != 'ordered_list_close': while j < len(tokens) and tokens[j].type != "ordered_list_close":
if tokens[j].type == 'list_item_open': if tokens[j].type == "list_item_open":
k = j + 1 k = j + 1
while k < len(tokens) and tokens[k].type != 'list_item_close': while k < len(tokens) and tokens[k].type != "list_item_close":
if tokens[k].type == 'paragraph_open' and k + 1 < len(tokens) and tokens[k + 1].type == 'inline': if (
tokens[k].type == "paragraph_open"
and k + 1 < len(tokens)
and tokens[k + 1].type == "inline"
):
list_content.append(f"{list_item_counter}. {tokens[k + 1].content.strip()}") list_content.append(f"{list_item_counter}. {tokens[k + 1].content.strip()}")
list_item_counter += 1 list_item_counter += 1
k += 1 k += 1
@ -206,15 +218,19 @@ def chunk_markdown(
_flush_content(result, current_content, title_stack, max_length, embed_fn, special_element=token.type) _flush_content(result, current_content, title_stack, max_length, embed_fn, special_element=token.type)
i = j + 1 i = j + 1
continue continue
elif token.type == 'bullet_list_open': elif token.type == "bullet_list_open":
_flush_content(result, current_content, title_stack, max_length, embed_fn) _flush_content(result, current_content, title_stack, max_length, embed_fn)
list_content = [] list_content = []
j = i + 1 j = i + 1
while j < len(tokens) and tokens[j].type != 'bullet_list_close': while j < len(tokens) and tokens[j].type != "bullet_list_close":
if tokens[j].type == 'list_item_open': if tokens[j].type == "list_item_open":
k = j + 1 k = j + 1
while k < len(tokens) and tokens[k].type != 'list_item_close': while k < len(tokens) and tokens[k].type != "list_item_close":
if tokens[k].type == 'paragraph_open' and k + 1 < len(tokens) and tokens[k + 1].type == 'inline': if (
tokens[k].type == "paragraph_open"
and k + 1 < len(tokens)
and tokens[k + 1].type == "inline"
):
list_content.append(f"- {tokens[k + 1].content.strip()}") list_content.append(f"- {tokens[k + 1].content.strip()}")
k += 1 k += 1
j += 1 j += 1
@ -223,33 +239,41 @@ def chunk_markdown(
_flush_content(result, current_content, title_stack, max_length, embed_fn, special_element=token.type) _flush_content(result, current_content, title_stack, max_length, embed_fn, special_element=token.type)
i = j + 1 i = j + 1
continue continue
elif token.type == 'html_block': elif token.type == "html_block":
_flush_content(result, current_content, title_stack, max_length, embed_fn) _flush_content(result, current_content, title_stack, max_length, embed_fn)
content = token.content.strip() content = token.content.strip()
is_converted_table = False is_converted_table = False
if '<table' in content.lower(): if "<table" in content.lower():
try: try:
kv_list = html_table_to_key_value(content) kv_list = html_table_to_key_value(content)
if kv_list: if kv_list:
content = '\n'.join([f"- {item}" for item in kv_list]) content = "\n".join([f"- {item}" for item in kv_list])
is_converted_table = True is_converted_table = True
except Exception as e: except Exception as e:
logger.warning(f"HTML表格转KV失败: {e}") logger.warning(f"HTML表格转KV失败: {e}")
current_content.append(content) current_content.append(content)
if is_converted_table: if is_converted_table:
_flush_content(result, current_content, title_stack, max_length, embed_fn, special_element='Table KV', allow_split=True) _flush_content(
result,
current_content,
title_stack,
max_length,
embed_fn,
special_element="Table KV",
allow_split=True,
)
else: else:
_flush_content(result, current_content, title_stack, max_length, embed_fn, special_element=token.type) _flush_content(result, current_content, title_stack, max_length, embed_fn, special_element=token.type)
i += 1 i += 1
continue continue
elif token.type in ['list_item_close', 'ordered_list_close', 'bullet_list_close', 'list_item_open']: elif token.type in ["list_item_close", "ordered_list_close", "bullet_list_close", "list_item_open"]:
i += 1 i += 1
continue continue
elif token.type == 'math_block': elif token.type == "math_block":
_flush_content(result, current_content, title_stack, max_length, embed_fn) _flush_content(result, current_content, title_stack, max_length, embed_fn)
current_content.append(f"$ {token.content} $") current_content.append(f"$ {token.content} $")
_flush_content(result, current_content, title_stack, max_length, embed_fn, special_element='Math Block') _flush_content(result, current_content, title_stack, max_length, embed_fn, special_element="Math Block")
i += 1 i += 1
continue continue
else: else:
@ -260,15 +284,15 @@ def chunk_markdown(
chunks = [] chunks = []
current_chunk_parts = [] current_chunk_parts = []
for item in result: for item in result:
if item == '-' * 10: if item == "-" * 10:
if current_chunk_parts: if current_chunk_parts:
chunks.append('\n'.join(current_chunk_parts).strip()) chunks.append("\n".join(current_chunk_parts).strip())
current_chunk_parts = [] current_chunk_parts = []
else: else:
current_chunk_parts.append(item) current_chunk_parts.append(item)
if current_chunk_parts: if current_chunk_parts:
chunks.append('\n'.join(current_chunk_parts).strip()) chunks.append("\n".join(current_chunk_parts).strip())
logger.info(f"语义切分完成: chunks={len(chunks)}") logger.info(f"语义切分完成: chunks={len(chunks)}")
return chunks return chunks

View File

@ -1,35 +1,61 @@
from __future__ import annotations from __future__ import annotations
import re import re
from typing import List, Callable, Any from collections.abc import Callable
from typing import Any
from .semantic_utils import semantic_chunking_with_auto_clusters from .semantic_utils import semantic_chunking_with_auto_clusters
def infer_heading_level(title: str) -> int: def infer_heading_level(title: str) -> int:
""" """
根据标题文本推断其层级级别1-6 根据标题文本推断其层级级别1-6
逻辑说明
1. 数字序号推断
- 匹配如 "1.", "1.1", "1.2.3" 等格式
- 根据点号分隔的数量确定层级例如 "1.1" 2 "1.2.3" 3
- 层级限制在 1-6 之间
2. 中文序号推断
- 匹配如 "一、", "二." 等中文数字序号
- 统一归类为 1 级标题
3. 默认处理
- 若不匹配以上规则默认返回 1
""" """
m = re.match(r'^\s*(\d+(?:\.\d+)*)[.)、]?\s*', title) m = re.match(r"^\s*(\d+(?:\.\d+)*)[.)、]?\s*", title)
if m: if m:
return max(1, min(len(m.group(1).split('.')), 6)) return max(1, min(len(m.group(1).split(".")), 6))
m_zh = re.match(r'^\s*[一二三四五六七八九十百千]+[、.]\s*', title) m_zh = re.match(r"^\s*[一二三四五六七八九十百千]+[、.]\s*", title)
if m_zh: if m_zh:
return 1 return 1
return 1 return 1
def get_title_path(stack: List[str]) -> str:
def get_title_path(stack: list[str]) -> str:
""" """
根据标题栈生成标题路径"|"分隔 根据标题栈生成标题路径"|"分隔
""" """
return '|'.join([t for t in stack if t]) return "|".join([t for t in stack if t])
def extract_table_block(tokens: List[Any], i: int, original_lines: List[str]) -> tuple[int, str]:
def extract_table_block(tokens: list[Any], i: int, original_lines: list[str]) -> tuple[int, str]:
""" """
从token流和原始文本中提取完整的表格块 从token流和原始文本中提取完整的表格块
逻辑说明
1. 定位起始通过当前 token (i) `map` 属性获取表格在原始行中的起始行号 `table_start`
2. 查找结束 token遍历后续 tokens 直到找到 `table_close`
3. 确定结束行号 (`table_end`)
- 优先使用 `table_close` token `map` 属性
- 若不存在则尝试查找下一个带有 `map` 信息的 token 的起始行作为当前表格的结束
- 若上述均失败如文件末尾或解析异常则回退到基于文本内容的启发式扫描
`table_start` 开始向下扫描直到遇到不符合 Markdown 表格特征不以 '|' 开头且不含 '|'的行为止
4. 返回结果返回 `table_close` 的索引 `j` 以及拼接后的表格原始字符串
""" """
token = tokens[i] token = tokens[i]
table_start = token.map[0] if token.map else 0 table_start = token.map[0] if token.map else 0
j = i + 1 j = i + 1
while j < len(tokens) and tokens[j].type != 'table_close': while j < len(tokens) and tokens[j].type != "table_close":
j += 1 j += 1
if j < len(tokens): if j < len(tokens):
end_token = tokens[j] end_token = tokens[j]
@ -45,78 +71,80 @@ def extract_table_block(tokens: List[Any], i: int, original_lines: List[str]) ->
table_end = table_start + 1 table_end = table_start + 1
for line_idx in range(table_start, len(original_lines)): for line_idx in range(table_start, len(original_lines)):
line = original_lines[line_idx].strip() line = original_lines[line_idx].strip()
if not line or not (line.startswith('|') or '|' in line): if not line or not (line.startswith("|") or "|" in line):
table_end = line_idx table_end = line_idx
break break
else: else:
table_end = table_start + 1 table_end = table_start + 1
for line_idx in range(table_start, len(original_lines)): for line_idx in range(table_start, len(original_lines)):
line = original_lines[line_idx].strip() line = original_lines[line_idx].strip()
if not line or not (line.startswith('|') or '|' in line): if not line or not (line.startswith("|") or "|" in line):
table_end = line_idx table_end = line_idx
break break
return j, '\n'.join(original_lines[table_start:table_end]) return j, "\n".join(original_lines[table_start:table_end])
def split_text_by_length_and_newline( def split_text_by_length_and_newline(
text: str, text: str, max_length: int, embed_fn: Callable[[list[str]], Any], token_count_fn: Callable[[str], int]
max_length: int, ) -> list[str]:
embed_fn: Callable[[List[str]], Any],
token_count_fn: Callable[[str], int]
) -> List[str]:
""" """
层次化文本切分策略 层次化文本切分策略
""" """
chunks = [] chunks = []
paragraphs = text.split('\n\n') paragraphs = text.split("\n\n")
for paragraph in paragraphs: for paragraph in paragraphs:
paragraph = paragraph.strip() paragraph = paragraph.strip()
if not paragraph: if not paragraph:
continue continue
paragraph_token_count = token_count_fn(paragraph) paragraph_token_count = token_count_fn(paragraph)
# 如果当前段落长度未超过最大 Token 数量直接作为独立分块放入chunks
# 否则继续尝试按行切分
if paragraph_token_count <= max_length: if paragraph_token_count <= max_length:
chunks.append(paragraph) chunks.append(paragraph)
continue continue
lines = paragraph.split('\n') # 把段落进一步使用换行符进行切分为行
lines = paragraph.split("\n")
current_chunk_lines = [] current_chunk_lines = []
current_chunk_tokens = 0 current_chunk_tokens = 0
for line in lines: for line in lines:
line = line.strip() line = line.strip()
if not line: if not line: # 跳过空行
continue continue
line_token_count = token_count_fn(line) line_token_count = token_count_fn(line) # 计算当前行的 Token 数量
# 为了考虑行之间的空格,需要在计算 Token 数量时加 1如果当前行不是第一行需要添加一个换行符的Token数量
added_tokens = line_token_count + (1 if current_chunk_lines else 0) added_tokens = line_token_count + (1 if current_chunk_lines else 0)
# 如果当前行的 Token 数量超过最大 Token 数量直接作为独立分块放入chunks
if line_token_count > max_length: if line_token_count > max_length:
if current_chunk_lines: if current_chunk_lines:
chunks.append('\n'.join(current_chunk_lines)) chunks.append("\n".join(current_chunk_lines))
current_chunk_lines = [] current_chunk_lines = []
current_chunk_tokens = 0 current_chunk_tokens = 0
sub_chunks = semantic_chunking_with_auto_clusters( sub_chunks = semantic_chunking_with_auto_clusters(
line, line, embed_fn=embed_fn, token_count_fn=token_count_fn, max_chunk_size=max_length
embed_fn=embed_fn,
token_count_fn=token_count_fn,
max_chunk_size=max_length
) )
chunks.extend(sub_chunks) chunks.extend(sub_chunks)
# 如果当前行的 Token 数量与当前分块的 Token 数量合并后超过最大 Token 数量直接作为独立分块放入chunks
elif current_chunk_tokens + added_tokens > max_length: elif current_chunk_tokens + added_tokens > max_length:
chunks.append('\n'.join(current_chunk_lines)) # 把之前的分块内容放入chunks
chunks.append("\n".join(current_chunk_lines))
# 重置当前分块为当前行的内容
current_chunk_lines = [line] current_chunk_lines = [line]
# 更新当前分块的 Token 数量
current_chunk_tokens = line_token_count current_chunk_tokens = line_token_count
# 如果当前行的内容加入当前分块后不会超过最大 Token 数量,直接加入当前分块
else: else:
current_chunk_lines.append(line) current_chunk_lines.append(line)
current_chunk_tokens += added_tokens current_chunk_tokens += added_tokens # 更新当前分块的 Token 数量
# 最后的收尾把最后一行内容放入chunks
if current_chunk_lines: if current_chunk_lines:
chunks.append('\n'.join(current_chunk_lines)) chunks.append("\n".join(current_chunk_lines))
return chunks return chunks

View File

@ -1,37 +1,62 @@
from __future__ import annotations from __future__ import annotations
import re import re
from collections.abc import Callable
from typing import Any
import nltk import nltk
from nltk.tokenize import sent_tokenize from nltk.tokenize import sent_tokenize
from sklearn.cluster import AgglomerativeClustering from sklearn.cluster import AgglomerativeClustering
from sklearn.metrics import silhouette_score from sklearn.metrics import silhouette_score
from typing import Callable, List, Any
# Ensure punkt_tab is available # 模块加载的时候先检查是否已经下载了punkt_tab模型用于识别句子的边界
try: try:
nltk.data.find('tokenizers/punkt_tab') nltk.data.find("tokenizers/punkt_tab")
except LookupError: except LookupError:
nltk.download('punkt_tab') nltk.download("punkt_tab")
def split_sentences_chinese(text: str) -> List[str]:
def split_sentences_chinese(text: str) -> list[str]:
""" """
Split sentences by Chinese punctuation while keeping the punctuation. 使用正则表达式将中文文本分割成句子
逻辑
- 匹配中文句号感叹号问号作为分隔点
- 使用正向/反向预查处理引号确保如果标点后面紧跟引号"),该引号会被保留在当前句子末尾,而不是被切分到下一句。
- 返回去除两端空格且非空的句子列表
""" """
pattern = r'(?<=[。!?])(?![”’"])|(?<=[。!?][”’"])' pattern = r'(?<=[。!?])(?![”’"])|(?<=[。!?])(?![”’"])|(?<=[。!?][”’"])'
sentences = re.split(pattern, text) sentences = re.split(pattern, text)
return [s.strip() for s in sentences if s.strip()] return [s.strip() for s in sentences if s.strip()]
def split_mixed_sentences(text: str) -> List[str]:
def split_mixed_sentences(text: str) -> list[str]:
""" """
Handle both Chinese and English sentence splitting. 处理中英文混合文本的分句逻辑支持按物理段落分发不同的分句策略
该函数采用分而治之的策略来处理复杂的混合文本
1. **物理分块**首先按换行符 (`\\n+`) 将原始文本切分为多个物理段落chunks确保物理结构不被破坏
2. **语言检测与分发**
- **英文/混合路径**若段落中包含英文字母 (`[A-Za-z]`)则视为英文或混合文本
调用 NLTK `sent_tokenize` 进行处理NLTK 能更好地处理英文缩写句点等复杂情况
- **中文路径**若段落不含字母则视为纯中文文本调用 `split_sentences_chinese`
该方法通过正则精准匹配中文标点及后续引号
- **兜底方案**若上述方法未产生结果则使用简单的正则表达式按中文标点强制分割
3. **清洗与过滤**汇总所有子句去除两端空白字符并过滤掉空字符串
Args:
text: 待分句的原始字符串
Returns:
List[str]: 分割后的句子列表
""" """
chunks = re.split(r'(\n+)', text) chunks = re.split(r"(\n+)", text)
sentences = [] sentences = []
for ch in chunks: for ch in chunks:
if not ch.strip(): if not ch.strip():
continue continue
if re.search(r'[A-Za-z]', ch): if re.search(r"[A-Za-z]", ch):
parts = sent_tokenize(ch) parts = sent_tokenize(ch)
sentences.extend([p.strip() for p in parts if p.strip()]) sentences.extend([p.strip() for p in parts if p.strip()])
else: else:
@ -39,13 +64,29 @@ def split_mixed_sentences(text: str) -> List[str]:
if sents: if sents:
sentences.extend([s.strip() for s in sents if s.strip()]) sentences.extend([s.strip() for s in sents if s.strip()])
else: else:
parts = re.split(r'(?<=[。!?])', ch) parts = re.split(r"(?<=[。!?])", ch)
sentences.extend([p.strip() for p in parts if p.strip()]) sentences.extend([p.strip() for p in parts if p.strip()])
return sentences return sentences
def find_best_num_clusters(embeddings: Any, min_clusters: int = 2, max_clusters: int = 10) -> int: def find_best_num_clusters(embeddings: Any, min_clusters: int = 2, max_clusters: int = 10) -> int:
""" """
Select best number of clusters using silhouette score. 使用轮廓系数选择最佳聚类数量让每个分段语义集中且分段之间界限分明
逻辑
- 遍历可能的聚类数量 min_clusters max_clusters
- 对每个聚类数量使用 `AgglomerativeClustering` 进行聚类
- 计算轮廓系数Silhouette Score
- 选择轮廓系数最高的聚类数量作为最佳聚类数量
- 如果聚类数量为 1 或更少直接返回 1
Args:
embeddings: 待聚类的向量数据所有句子的嵌入向量列表
min_clusters: 搜索的最佳聚类数量下限默认为 2
max_clusters: 搜索的最佳聚类数量上限默认为 10
Returns:
int: 轮廓系数表现最好的聚类数量
""" """
if len(embeddings) <= min_clusters: if len(embeddings) <= min_clusters:
return len(embeddings) return len(embeddings)
@ -55,42 +96,48 @@ def find_best_num_clusters(embeddings: Any, min_clusters: int = 2, max_clusters:
limit_k = min(max_clusters, len(embeddings)) limit_k = min(max_clusters, len(embeddings))
for k in range(min_clusters, limit_k + 1): for k in range(min_clusters, limit_k + 1):
labels = AgglomerativeClustering(n_clusters=k, metric='cosine', linkage='average').fit_predict(embeddings) labels = AgglomerativeClustering(n_clusters=k, metric="cosine", linkage="average").fit_predict(embeddings)
if len(set(labels)) <= 1: if len(set(labels)) <= 1:
continue continue
score = silhouette_score(embeddings, labels, metric='cosine') score = silhouette_score(embeddings, labels, metric="cosine")
if score > best_score: if score > best_score:
best_score = score best_score = score
best_k = k best_k = k
return best_k return best_k
def semantic_chunking_with_auto_clusters( def semantic_chunking_with_auto_clusters(
text: str, text: str, embed_fn: Callable[[list[str]], Any], token_count_fn: Callable[[str], int], max_chunk_size: int = 512
embed_fn: Callable[[List[str]], Any], ) -> list[str]:
token_count_fn: Callable[[str], int],
max_chunk_size: int = 512
) -> List[str]:
""" """
Semantic chunking with automatic cluster number selection. 对传入的文本进行语义切分过程中会自动选择最佳的聚集数量
逻辑
- 先将文本中的句子按语言进行分发英文/混合文本使用NLTK的sent_tokenize中文文本使用split_sentences_chinese
- 对每个句子进行嵌入向量化
- 确定最佳的聚类数量根据轮廓系数
- 对句子进行聚类将每个聚类中的句子连接起来形成一个分块
""" """
sentences = split_mixed_sentences(text) sentences = split_mixed_sentences(text)
if len(sentences) < 2: if len(sentences) < 2:
return [text.strip()] return [text.strip()]
# Vectorization # 向量化每个句子, 得到他们的嵌入向量
embeddings = embed_fn(sentences) embeddings = embed_fn(sentences)
# Pre-calculate token counts # 计算每个句子的token数量
sentence_token_counts = [token_count_fn(s) for s in sentences] sentence_token_counts = [token_count_fn(s) for s in sentences]
total_tokens = sum(sentence_token_counts) total_tokens = sum(sentence_token_counts)
# Determine number of clusters # 决定合适的聚集数量需要保证每个分块的token数量都不超过max_chunk_size
best_k = max(total_tokens // max_chunk_size, 1) + 1 best_k = max(total_tokens // max_chunk_size, 1) + 1
best_k = min(best_k, len(sentences)) best_k = min(best_k, len(sentences))
# Clustering # 根据指定的聚集数量、相似度判断方式、联动方式,对句子进行聚类
labels = AgglomerativeClustering(n_clusters=best_k, metric='cosine', linkage='average').fit_predict(embeddings) # 这里返回的labels是一个每个句子的聚类标签列表例如[0,0,1,2,2]相同ID的句子被聚类到同一个分块中
labels = AgglomerativeClustering(n_clusters=best_k, metric="cosine", linkage="average").fit_predict(embeddings)
chunks = [] chunks = []
current_chunk = "" current_chunk = ""

View File

@ -1,57 +1,56 @@
from __future__ import annotations from __future__ import annotations
from typing import List
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from yuxi.utils.logging_config import logger
def html_table_to_markdown(html: str) -> str: def html_table_to_markdown(html: str) -> str:
""" """
将HTML表格转换为Markdown格式的具体实现 将HTML表格转换为Markdown格式的具体实现
""" """
soup = BeautifulSoup(html, 'html.parser') soup = BeautifulSoup(html, "html.parser")
table = soup.find('table') table = soup.find("table")
if table is None: if table is None:
return '' return ""
rows = table.find_all('tr') rows = table.find_all("tr")
if not rows: if not rows:
return '' return ""
grid = [] grid = []
for r_idx, row in enumerate(rows): for r_idx, row in enumerate(rows):
while len(grid) <= r_idx: while len(grid) <= r_idx:
grid.append([]) grid.append([])
cells = row.find_all(['td', 'th']) cells = row.find_all(["td", "th"])
c_idx = 0 c_idx = 0
for cell in cells: for cell in cells:
while c_idx < len(grid[r_idx]) and grid[r_idx][c_idx] is not None: while c_idx < len(grid[r_idx]) and grid[r_idx][c_idx] is not None:
c_idx += 1 c_idx += 1
text = cell.get_text(strip=True) text = cell.get_text(strip=True)
text = text.replace('\n', ' ') text = text.replace("\n", " ")
rowspan = int(cell.get('rowspan', 1)) rowspan = int(cell.get("rowspan", 1))
colspan = int(cell.get('colspan', 1)) colspan = int(cell.get("colspan", 1))
for r in range(rowspan): for r in range(rowspan):
target_r = r_idx + r target_r = r_idx + r
while len(grid) <= target_r: while len(grid) <= target_r:
grid.append([]) grid.append([])
for c in range(colspan): for c in range(colspan):
target_c = c_idx + c target_c = c_idx + c
while len(grid[target_r]) <= target_c: while len(grid[target_r]) <= target_c:
grid[target_r].append(None) grid[target_r].append(None)
grid[target_r][target_c] = text grid[target_r][target_c] = text
c_idx += colspan c_idx += colspan
if not grid: if not grid:
return '' return ""
markdown_lines = [] markdown_lines = []
max_cols = max(len(r) for r in grid) max_cols = max(len(r) for r in grid)
@ -61,52 +60,65 @@ def html_table_to_markdown(html: str) -> str:
header = grid[0] header = grid[0]
header = [h if h is not None else "" for h in header] header = [h if h is not None else "" for h in header]
markdown_lines.append('| ' + ' | '.join(header) + ' |') markdown_lines.append("| " + " | ".join(header) + " |")
markdown_lines.append('|' + '|'.join([' --- ' for _ in range(max_cols)]) + '|') markdown_lines.append("|" + "|".join([" --- " for _ in range(max_cols)]) + "|")
for row in grid[1:]: for row in grid[1:]:
row_clean = [cell if cell is not None else "" for cell in row] row_clean = [cell if cell is not None else "" for cell in row]
line = '| ' + ' | '.join(row_clean) + ' |' line = "| " + " | ".join(row_clean) + " |"
markdown_lines.append(line) markdown_lines.append(line)
return '\n'.join(markdown_lines) return "\n".join(markdown_lines)
def html_table_to_key_value(html: str) -> List[str]: def html_table_to_key_value(html: str) -> list[str]:
""" """
将HTML表格转换为键值对格式的列表 将HTML表格转换为键值对格式的列表为了应对过长的表格的切分问题
处理逻辑
1. **网格重建**由于 HTML 表格可能包含 `rowspan` `colspan`合并单元格
函数首先构建一个完整的二维网格grid
2. **单元格展开**遍历 HTML 行和列遇到合并单元格时将其内容填充到网格中受影响的所有坐标点
这确保了原本被合并的区域在逻辑网格中每个点都有对应的值
3. **键值对转换**
- 将网格的第一行视为表头Key
- 从第二行开始将每一行与表头对应生成 "键:值" 形式的字符串
例如
- 输入HTML表格包含姓名年龄性别三列
- 输出['姓名张三年龄25性别', '姓名李四年龄30性别']
""" """
soup = BeautifulSoup(html, 'html.parser') soup = BeautifulSoup(html, "html.parser")
table = soup.find('table') table = soup.find("table")
if table is None: if table is None:
return [] return []
rows = table.find_all('tr') rows = table.find_all("tr")
if not rows: if not rows:
return [] return []
grid = [] grid = []
for r_idx, row in enumerate(rows): for r_idx, row in enumerate(rows):
while len(grid) <= r_idx: while len(grid) <= r_idx:
grid.append([]) grid.append([])
cells = row.find_all(['td', 'th']) cells = row.find_all(["td", "th"])
c_idx = 0 c_idx = 0
for cell in cells: for cell in cells:
while c_idx < len(grid[r_idx]) and grid[r_idx][c_idx] is not None: while c_idx < len(grid[r_idx]) and grid[r_idx][c_idx] is not None:
c_idx += 1 c_idx += 1
text = cell.get_text(strip=True) text = cell.get_text(strip=True)
rowspan = int(cell.get('rowspan', 1)) rowspan = int(cell.get("rowspan", 1))
colspan = int(cell.get('colspan', 1)) colspan = int(cell.get("colspan", 1))
for r in range(rowspan): for r in range(rowspan):
target_r = r_idx + r target_r = r_idx + r
while len(grid) <= target_r: while len(grid) <= target_r:
grid.append([]) grid.append([])
for c in range(colspan): for c in range(colspan):
target_c = c_idx + c target_c = c_idx + c
while len(grid[target_r]) <= target_c: while len(grid[target_r]) <= target_c:
@ -116,10 +128,10 @@ def html_table_to_key_value(html: str) -> List[str]:
if not grid: if not grid:
return [] return []
headers = grid[0] headers = grid[0]
headers = [h if h is not None else "" for h in headers] headers = [h if h is not None else "" for h in headers]
kv_lines = [] kv_lines = []
for row_values in grid[1:]: for row_values in grid[1:]:
min_len = min(len(headers), len(row_values)) min_len = min(len(headers), len(row_values))
@ -131,5 +143,5 @@ def html_table_to_key_value(html: str) -> List[str]:
row_parts.append(f"{key}{val}") row_parts.append(f"{key}{val}")
if row_parts: if row_parts:
kv_lines.append("".join(row_parts) + "") kv_lines.append("".join(row_parts) + "")
return kv_lines return kv_lines

View File

@ -475,10 +475,7 @@ async def find_user_by_oidc_sub(db, sub: str) -> User | None:
# 方法1: 检查是否有用户的 user_id 直接等于 "oidc:{sub}"(标准 OIDC 用户) # 方法1: 检查是否有用户的 user_id 直接等于 "oidc:{sub}"(标准 OIDC 用户)
standard_oidc_user_id = f"oidc:{sub}" standard_oidc_user_id = f"oidc:{sub}"
# 占位绑定记录会被标记为 is_deleted=1但我们仍需要查询它们来获取绑定关系 # 占位绑定记录会被标记为 is_deleted=1但我们仍需要查询它们来获取绑定关系
result = await db.execute(select(User).filter( result = await db.execute(select(User).filter(User.user_id == standard_oidc_user_id, User.is_deleted == 0))
User.user_id == standard_oidc_user_id,
User.is_deleted == 0
))
user = result.scalar_one_or_none() user = result.scalar_one_or_none()
if user: if user:
return user return user
@ -486,10 +483,9 @@ async def find_user_by_oidc_sub(db, sub: str) -> User | None:
# 方法2: 检查是否有绑定占位用户格式: "oidc:{sub}:{target_user_id}"use_raw_username 绑定记录) # 方法2: 检查是否有绑定占位用户格式: "oidc:{sub}:{target_user_id}"use_raw_username 绑定记录)
# 绑定占位用户被标记为 is_deleted=1需要包括deleted来查询 # 绑定占位用户被标记为 is_deleted=1需要包括deleted来查询
legacy_result = await db.execute( legacy_result = await db.execute(
select(User).filter( select(User)
User.user_id.like(f"{standard_oidc_user_id}:%"), .filter(User.user_id.like(f"{standard_oidc_user_id}:%"), User.is_deleted.in_([0, 1]))
User.is_deleted.in_([0, 1]) .order_by(User.id.asc())
).order_by(User.id.asc())
) )
legacy_users = list(legacy_result.scalars().all()) legacy_users = list(legacy_result.scalars().all())
if legacy_users: if legacy_users:
@ -528,10 +524,7 @@ async def find_deleted_oidc_user_by_sub(db, sub: str) -> User | None:
# 检查绑定占位格式 oidc:{sub}:{target_user_id}占位本身是deleted需要查询目标用户 # 检查绑定占位格式 oidc:{sub}:{target_user_id}占位本身是deleted需要查询目标用户
legacy_result = await db.execute( legacy_result = await db.execute(
select(User).filter( select(User).filter(User.user_id.like(f"{oidc_user_id}:%"), User.is_deleted == 1).order_by(User.id.asc())
User.user_id.like(f"{oidc_user_id}:%"),
User.is_deleted == 1
).order_by(User.id.asc())
) )
legacy_users = list(legacy_result.scalars().all()) legacy_users = list(legacy_result.scalars().all())
if legacy_users: if legacy_users:
@ -585,6 +578,7 @@ async def _create_oidc_binding_placeholder(db, sub: str, target_user: User) -> N
# username 使用 oidc-binding-{sub_hash} 避免冲突sub_hash 基于完整 sub 生成 # username 使用 oidc-binding-{sub_hash} 避免冲突sub_hash 基于完整 sub 生成
import hashlib import hashlib
sub_hash = hashlib.sha256(sub.encode()).hexdigest()[:8] sub_hash = hashlib.sha256(sub.encode()).hexdigest()[:8]
username = f"oidc-binding-{sub_hash}" username = f"oidc-binding-{sub_hash}"
@ -604,8 +598,7 @@ async def _create_oidc_binding_placeholder(db, sub: str, target_user: User) -> N
db.add(placeholder_user) db.add(placeholder_user)
await db.commit() await db.commit()
logger.info( logger.info(
f"Created OIDC binding placeholder (deleted) for sub {sub} -> " f"Created OIDC binding placeholder (deleted) for sub {sub} -> user {target_user.id} ({target_user.user_id})"
f"user {target_user.id} ({target_user.user_id})"
) )
except IntegrityError: except IntegrityError:
# 并发创建冲突,回滚后忽略 # 并发创建冲突,回滚后忽略
@ -661,8 +654,7 @@ async def create_oidc_user(db, user_info: dict, department_id: int | None = None
if user_by_sub and user_by_sub.id == existing_user.id: if user_by_sub and user_by_sub.id == existing_user.id:
# sub 已经正确绑定到该用户,允许返回 # sub 已经正确绑定到该用户,允许返回
logger.info( logger.info(
f"User with raw username {user_id} already exists and " f"User with raw username {user_id} already exists and bound to sub {sub}, returning existing user"
f"bound to sub {sub}, returning existing user"
) )
return existing_user return existing_user
elif user_by_sub is None: elif user_by_sub is None:

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,7 @@
[
"## 风力发电机组 塔架设计规范|1 概述\n\n本标准规定了风力发电机组塔架的设计、制造、运输和安装要求。",
"### 风力发电机组 塔架设计规范|6 钢制塔架|6.1 一般要求\n\n钢制塔架的设计应考虑极端载荷和疲劳载荷。在计算连接强度时需要用到螺纹截面积 Asp 以及承载力设计值 Rd。\n此外结构应力 σ _ {y, d} 的计算必须符合相关标准要求。",
"### 风力发电机组 塔架设计规范|6 钢制塔架|6.2 疲劳极限状态\n\n疲劳计算应基于 Miner 线性累积损伤理论。",
"### 风力发电机组 塔架设计规范|7 混凝土塔架|7.1 材料特性\n\n混凝土强度等级不应低于 C50。",
"### 风力发电机组 塔架设计规范|7 混凝土塔架|7.2 施工工艺\n\n混凝土塔架可采用现浇或预制拼装方式。对于预制片段应严格控制拼装精度。"
]

View File

@ -1,6 +1,7 @@
import pytest import pytest
import numpy as np import numpy as np
import os import os
import json
# 现在可以安全地导入了,因为顶层不再有重型依赖 # 现在可以安全地导入了,因为顶层不再有重型依赖
from yuxi.knowledge.chunking.ragflow_like.parsers import semantic from yuxi.knowledge.chunking.ragflow_like.parsers import semantic
@ -22,57 +23,87 @@ def embed_fn():
@pytest.fixture @pytest.fixture
def sample_markdown(): def sample_markdown():
with open("test/resource/test4.md", "r", encoding="utf-8") as f: """提供一个包含多章节、公式符号和复杂结构的临时 Markdown 样本"""
return f.read() return """
# 风力发电机组 塔架设计规范
## 1 概述
本标准规定了风力发电机组塔架的设计制造运输和安装要求
## 6 钢制塔架
### 6.1 一般要求
钢制塔架的设计应考虑极端载荷和疲劳载荷在计算连接强度时需要用到螺纹截面积 Asp 以及承载力设计值 Rd
此外结构应力 σ _ {y, d} 的计算必须符合相关标准要求
### 6.2 疲劳极限状态
疲劳计算应基于 Miner 线性累积损伤理论
## 7 混凝土塔架
### 7.1 材料特性
混凝土强度等级不应低于 C50
### 7.2 施工工艺
混凝土塔架可采用现浇或预制拼装方式对于预制片段应严格控制拼装精度
"""
def test_semantic_chunking_basic(embed_fn, sample_markdown): def test_semantic_chunking_basic(embed_fn, sample_markdown):
"""测试基本的语义切分逻辑 (使用真实嵌入模型)""" """测试基本的语义切分逻辑 (使用真实嵌入模型)"""
# 配置切分参数 # 配置切分参数
parser_config = { parser_config = {
"chunk_token_num": 200, # 适中的 token 数以触发更多切分 "chunk_token_num": 300, # 针对标准文档调整 token 数
"overlapped_percent": 0 "overlapped_percent": 0.1
} }
# 直接注入 embed_fn # 执行语义切分
chunks = semantic.chunk_markdown( chunks = semantic.chunk_markdown(
sample_markdown, sample_markdown,
parser_config=parser_config, parser_config=parser_config,
embed_fn=embed_fn embed_fn=embed_fn
) )
# 验证结果 # 1. 基础验证
assert isinstance(chunks, list) assert isinstance(chunks, list)
assert len(chunks) > 0 assert len(chunks) >= 5, f"预期至少切分为 5 个片段,实际仅有 {len(chunks)}"
print(f"\n成功切分为 {len(chunks)} 个片段:")
for i, chunk in enumerate(chunks):
print(f"--- 片段 {i+1} ---")
print(chunk)
# 验证标题路径增强是否生效 (检查是否包含 Part 或特殊元素标记)
has_enhanced_title = any("|Part" in chunk or "|Math Block" in chunk for chunk in chunks)
assert has_enhanced_title, "标题路径增强或分片标记失效"
# 验证是否包含特殊的 LaTeX 公式或符号 (匹配输出中的格式)
has_formula = any("f _ {0, 1}" in chunk or "\\sigma_ {y, d}" in chunk or "A _ {\\mathrm {s p}}" in chunk for chunk in chunks)
assert has_formula, "LaTeX 公式/符号在切分中丢失"
# 验证语义聚类是否大概生效 (钢制塔架和混凝土塔架内容应该分开) # 2. 验证标题路径增强 (检查 RAGFlow 风格的层级标记)
# 我们检查是否有钢制塔架相关的片段和混凝土塔架相关的片段 # 标准文档通常会被赋予类似 "风力发电机组 塔架 | 6 钢制塔架" 的标题路径
# 使用 startswith 来更准确地定位章节开始,避免匹配到目次/前言中的文字 has_enhanced_title = any("|" in chunk and ("钢制塔架" in chunk or "混凝土塔架" in chunk) for chunk in chunks)
has_steel = any(chunk.startswith("# 6 钢制塔架") for chunk in chunks) assert has_enhanced_title, "标题路径增强失效:未在 chunk 中发现层级分隔符 '|' 或核心章节标题"
has_concrete = any(chunk.startswith("# 7 混凝土塔架") for chunk in chunks)
assert has_steel and has_concrete, "语义内容丢失 (钢制或混凝土塔架章节开始部分)"
# 验证它们是否在不同的片段中
steel_chunks = [i for i, c in enumerate(chunks) if c.startswith("# 6 钢制塔架")]
concrete_chunks = [i for i, c in enumerate(chunks) if c.startswith("# 7 混凝土塔架")]
# 理论上语义聚类会将这些大章节分开
if steel_chunks and concrete_chunks:
assert not set(steel_chunks).intersection(set(concrete_chunks)), "钢制塔架和混凝土塔架大章节内容被错误地聚类在同一个 chunk 中了"
# 3. 验证公式与符号识别 (匹配 test4.md 4.1 节中的符号)
# 重点检查文档中出现的符号Asp (螺纹截面积), σ _ {y, d} (结构应力), Rd (承载力设计值)
symbols_to_check = ["Asp", "Rd", "σ _ {y, d}"]
found_symbols = [s for s in symbols_to_check if any(s in chunk for chunk in chunks)]
assert len(found_symbols) > 0, f"在切分结果中未找到关键符号: {symbols_to_check}"
# 4. 验证核心章节内容是否存在
has_steel_section = any("6 钢制塔架" in chunk for chunk in chunks)
has_concrete_section = any("7 混凝土塔架" in chunk for chunk in chunks)
assert has_steel_section, "未找到第 6 章 '钢制塔架' 相关内容"
assert has_concrete_section, "未找到第 7 章 '混凝土塔架' 相关内容"
# 5. 验证语义聚类是否将不同主题分开
# 钢制塔架和混凝土塔架是两个独立的大章节,语义聚类应该避免将它们的核心内容混在一个 chunk 中
steel_start_chunks = [i for i, c in enumerate(chunks) if "# 6 钢制塔架" in c]
concrete_start_chunks = [i for i, c in enumerate(chunks) if "# 7 混凝土塔架" in c]
if steel_start_chunks and concrete_start_chunks:
# 确保起始片段不重合
assert not set(steel_start_chunks).intersection(set(concrete_start_chunks)), \
"语义聚类错误:钢制塔架与混凝土塔架的章节头部被挤在了同一个 chunk 中"
print(f"\n[测试成功] 文档成功切分为 {len(chunks)} 个片段")
print(f"识别到的关键符号: {found_symbols}")
# 将切分后的内容写入到 resource 目录
output_dir = os.path.join(os.path.dirname(__file__), "resource")
os.makedirs(output_dir, exist_ok=True)
output_path = os.path.join(output_dir, "semantic_chunks.json")
with open(output_path, "w", encoding="utf-8") as f:
json.dump(chunks, f, ensure_ascii=False, indent=2)
print(f"切分内容已保存至: {output_path}")
def test_heading_inference(): def test_heading_inference():
"""测试标题层级推断工具类""" """测试标题层级推断工具类"""
from yuxi.knowledge.chunking.ragflow_like.utils.md_parser_utils import infer_heading_level from yuxi.knowledge.chunking.ragflow_like.utils.md_parser_utils import infer_heading_level

View File

@ -2521,6 +2521,18 @@ wheels = [
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/9c/46/f6b4ad632c67ef35209a66127e4bddc95759649dd595f71f13fba11bdf9a/mcp-1.27.0-py3-none-any.whl", hash = "sha256:5ce1fa81614958e267b21fb2aa34e0aea8e2c6ede60d52aba45fd47246b4d741", size = 215967, upload-time = "2026-04-02T14:48:07.24Z" }, { url = "https://pypi.tuna.tsinghua.edu.cn/packages/9c/46/f6b4ad632c67ef35209a66127e4bddc95759649dd595f71f13fba11bdf9a/mcp-1.27.0-py3-none-any.whl", hash = "sha256:5ce1fa81614958e267b21fb2aa34e0aea8e2c6ede60d52aba45fd47246b4d741", size = 215967, upload-time = "2026-04-02T14:48:07.24Z" },
] ]
[[package]]
name = "mdit-py-plugins"
version = "0.5.0"
source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" }
dependencies = [
{ name = "markdown-it-py" },
]
sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/b2/fd/a756d36c0bfba5f6e39a1cdbdbfdd448dc02692467d83816dff4592a1ebc/mdit_py_plugins-0.5.0.tar.gz", hash = "sha256:f4918cb50119f50446560513a8e311d574ff6aaed72606ddae6d35716fe809c6", size = 44655, upload-time = "2025-08-11T07:25:49.083Z" }
wheels = [
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/fb/86/dd6e5db36df29e76c7a7699123569a4a18c1623ce68d826ed96c62643cae/mdit_py_plugins-0.5.0-py3-none-any.whl", hash = "sha256:07a08422fc1936a5d26d146759e9155ea466e842f5ab2f7d2266dd084c8dab1f", size = 57205, upload-time = "2025-08-11T07:25:47.597Z" },
]
[[package]] [[package]]
name = "mdurl" name = "mdurl"
version = "0.1.2" version = "0.1.2"
@ -4337,6 +4349,38 @@ torch = [
{ name = "torch", version = "2.8.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform != 'darwin'" }, { name = "torch", version = "2.8.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform != 'darwin'" },
] ]
[[package]]
name = "scikit-learn"
version = "1.8.0"
source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" }
dependencies = [
{ name = "joblib" },
{ name = "numpy" },
{ name = "scipy" },
{ name = "threadpoolctl" },
]
sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/0e/d4/40988bf3b8e34feec1d0e6a051446b1f66225f8529b9309becaeef62b6c4/scikit_learn-1.8.0.tar.gz", hash = "sha256:9bccbb3b40e3de10351f8f5068e105d0f4083b1a65fa07b6634fbc401a6287fd", size = 7335585, upload-time = "2025-12-10T07:08:53.618Z" }
wheels = [
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/90/74/e6a7cc4b820e95cc38cf36cd74d5aa2b42e8ffc2d21fe5a9a9c45c1c7630/scikit_learn-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5fb63362b5a7ddab88e52b6dbb47dac3fd7dafeee740dc6c8d8a446ddedade8e", size = 8548242, upload-time = "2025-12-10T07:07:51.568Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/49/d8/9be608c6024d021041c7f0b3928d4749a706f4e2c3832bbede4fb4f58c95/scikit_learn-1.8.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:5025ce924beccb28298246e589c691fe1b8c1c96507e6d27d12c5fadd85bfd76", size = 8079075, upload-time = "2025-12-10T07:07:53.697Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/dd/47/f187b4636ff80cc63f21cd40b7b2d177134acaa10f6bb73746130ee8c2e5/scikit_learn-1.8.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4496bb2cf7a43ce1a2d7524a79e40bc5da45cf598dbf9545b7e8316ccba47bb4", size = 8660492, upload-time = "2025-12-10T07:07:55.574Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/97/74/b7a304feb2b49df9fafa9382d4d09061a96ee9a9449a7cbea7988dda0828/scikit_learn-1.8.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a0bcfe4d0d14aec44921545fd2af2338c7471de9cb701f1da4c9d85906ab847a", size = 8931904, upload-time = "2025-12-10T07:07:57.666Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/9f/c4/0ab22726a04ede56f689476b760f98f8f46607caecff993017ac1b64aa5d/scikit_learn-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:35c007dedb2ffe38fe3ee7d201ebac4a2deccd2408e8621d53067733e3c74809", size = 8019359, upload-time = "2025-12-10T07:07:59.838Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/24/90/344a67811cfd561d7335c1b96ca21455e7e472d281c3c279c4d3f2300236/scikit_learn-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:8c497fff237d7b4e07e9ef1a640887fa4fb765647f86fbe00f969ff6280ce2bb", size = 7641898, upload-time = "2025-12-10T07:08:01.36Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/03/aa/e22e0768512ce9255eba34775be2e85c2048da73da1193e841707f8f039c/scikit_learn-1.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0d6ae97234d5d7079dc0040990a6f7aeb97cb7fa7e8945f1999a429b23569e0a", size = 8513770, upload-time = "2025-12-10T07:08:03.251Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/58/37/31b83b2594105f61a381fc74ca19e8780ee923be2d496fcd8d2e1147bd99/scikit_learn-1.8.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:edec98c5e7c128328124a029bceb09eda2d526997780fef8d65e9a69eead963e", size = 8044458, upload-time = "2025-12-10T07:08:05.336Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/2d/5a/3f1caed8765f33eabb723596666da4ebbf43d11e96550fb18bdec42b467b/scikit_learn-1.8.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:74b66d8689d52ed04c271e1329f0c61635bcaf5b926db9b12d58914cdc01fe57", size = 8610341, upload-time = "2025-12-10T07:08:07.732Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/38/cf/06896db3f71c75902a8e9943b444a56e727418f6b4b4a90c98c934f51ed4/scikit_learn-1.8.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8fdf95767f989b0cfedb85f7ed8ca215d4be728031f56ff5a519ee1e3276dc2e", size = 8900022, upload-time = "2025-12-10T07:08:09.862Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/1c/f9/9b7563caf3ec8873e17a31401858efab6b39a882daf6c1bfa88879c0aa11/scikit_learn-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:2de443b9373b3b615aec1bb57f9baa6bb3a9bd093f1269ba95c17d870422b271", size = 7989409, upload-time = "2025-12-10T07:08:12.028Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/49/bd/1f4001503650e72c4f6009ac0c4413cb17d2d601cef6f71c0453da2732fc/scikit_learn-1.8.0-cp313-cp313-win_arm64.whl", hash = "sha256:eddde82a035681427cbedded4e6eff5e57fa59216c2e3e90b10b19ab1d0a65c3", size = 7619760, upload-time = "2025-12-10T07:08:13.688Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/d2/7d/a630359fc9dcc95496588c8d8e3245cc8fd81980251079bc09c70d41d951/scikit_learn-1.8.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:7cc267b6108f0a1499a734167282c00c4ebf61328566b55ef262d48e9849c735", size = 8826045, upload-time = "2025-12-10T07:08:15.215Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/cc/56/a0c86f6930cfcd1c7054a2bc417e26960bb88d32444fe7f71d5c2cfae891/scikit_learn-1.8.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:fe1c011a640a9f0791146011dfd3c7d9669785f9fed2b2a5f9e207536cf5c2fd", size = 8420324, upload-time = "2025-12-10T07:08:17.561Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/46/1e/05962ea1cebc1cf3876667ecb14c283ef755bf409993c5946ade3b77e303/scikit_learn-1.8.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:72358cce49465d140cc4e7792015bb1f0296a9742d5622c67e31399b75468b9e", size = 8680651, upload-time = "2025-12-10T07:08:19.952Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/fe/56/a85473cd75f200c9759e3a5f0bcab2d116c92a8a02ee08ccd73b870f8bb4/scikit_learn-1.8.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:80832434a6cc114f5219211eec13dcbc16c2bac0e31ef64c6d346cde3cf054cb", size = 8925045, upload-time = "2025-12-10T07:08:22.11Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/cc/b7/64d8cfa896c64435ae57f4917a548d7ac7a44762ff9802f75a79b77cb633/scikit_learn-1.8.0-cp313-cp313t-win_amd64.whl", hash = "sha256:ee787491dbfe082d9c3013f01f5991658b0f38aa8177e4cd4bf434c58f551702", size = 8507994, upload-time = "2025-12-10T07:08:23.943Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/5e/37/e192ea709551799379958b4c4771ec507347027bb7c942662c7fbeba31cb/scikit_learn-1.8.0-cp313-cp313t-win_arm64.whl", hash = "sha256:bf97c10a3f5a7543f9b88cbf488d33d175e9146115a451ae34568597ba33dcde", size = 7869518, upload-time = "2025-12-10T07:08:25.71Z" },
]
[[package]] [[package]]
name = "scipy" name = "scipy"
version = "1.17.1" version = "1.17.1"
@ -4756,6 +4800,15 @@ wheels = [
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/f5/15/a11f7bb3cbc97dfecf32a90552f5a8f8a5c99316a99c6c17bdabf5baf256/thinc-8.3.13-cp313-cp313-win_arm64.whl", hash = "sha256:723949cab11d1925c15447928513a718276316cec6e0de28337cca0a62be0521", size = 1644606, upload-time = "2026-03-23T07:22:21.339Z" }, { url = "https://pypi.tuna.tsinghua.edu.cn/packages/f5/15/a11f7bb3cbc97dfecf32a90552f5a8f8a5c99316a99c6c17bdabf5baf256/thinc-8.3.13-cp313-cp313-win_arm64.whl", hash = "sha256:723949cab11d1925c15447928513a718276316cec6e0de28337cca0a62be0521", size = 1644606, upload-time = "2026-03-23T07:22:21.339Z" },
] ]
[[package]]
name = "threadpoolctl"
version = "3.6.0"
source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" }
sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/b7/4d/08c89e34946fce2aec4fbb45c9016efd5f4d7f24af8e5d93296e935631d8/threadpoolctl-3.6.0.tar.gz", hash = "sha256:8ab8b4aa3491d812b623328249fab5302a68d2d71745c8a4c719a2fcaba9f44e", size = 21274, upload-time = "2025-03-13T13:49:23.031Z" }
wheels = [
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl", hash = "sha256:43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb", size = 18638, upload-time = "2025-03-13T13:49:21.846Z" },
]
[[package]] [[package]]
name = "tiktoken" name = "tiktoken"
version = "0.12.0" version = "0.12.0"
@ -4878,9 +4931,9 @@ dependencies = [
{ name = "typing-extensions", marker = "sys_platform == 'darwin'" }, { name = "typing-extensions", marker = "sys_platform == 'darwin'" },
] ]
wheels = [ wheels = [
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:a47b7986bee3f61ad217d8a8ce24605809ab425baf349f97de758815edd2ef54" }, { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:a47b7986bee3f61ad217d8a8ce24605809ab425baf349f97de758815edd2ef54" },
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:fbe2e149c5174ef90d29a5f84a554dfaf28e003cb4f61fa2c8c024c17ec7ca58" }, { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:fbe2e149c5174ef90d29a5f84a554dfaf28e003cb4f61fa2c8c024c17ec7ca58" },
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0-cp313-none-macosx_11_0_arm64.whl", hash = "sha256:057efd30a6778d2ee5e2374cd63a63f63311aa6f33321e627c655df60abdd390" }, { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0-cp313-none-macosx_11_0_arm64.whl", hash = "sha256:057efd30a6778d2ee5e2374cd63a63f63311aa6f33321e627c655df60abdd390" },
] ]
[[package]] [[package]]
@ -4903,19 +4956,19 @@ dependencies = [
{ name = "typing-extensions", marker = "sys_platform != 'darwin'" }, { name = "typing-extensions", marker = "sys_platform != 'darwin'" },
] ]
wheels = [ wheels = [
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-linux_s390x.whl", hash = "sha256:0e34e276722ab7dd0dffa9e12fe2135a9b34a0e300c456ed7ad6430229404eb5" }, { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-linux_s390x.whl", hash = "sha256:0e34e276722ab7dd0dffa9e12fe2135a9b34a0e300c456ed7ad6430229404eb5" },
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:610f600c102386e581327d5efc18c0d6edecb9820b4140d26163354a99cd800d" }, { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:610f600c102386e581327d5efc18c0d6edecb9820b4140d26163354a99cd800d" },
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:cb9a8ba8137ab24e36bf1742cb79a1294bd374db570f09fc15a5e1318160db4e" }, { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:cb9a8ba8137ab24e36bf1742cb79a1294bd374db570f09fc15a5e1318160db4e" },
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-win_amd64.whl", hash = "sha256:2be20b2c05a0cce10430cc25f32b689259640d273232b2de357c35729132256d" }, { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-win_amd64.whl", hash = "sha256:2be20b2c05a0cce10430cc25f32b689259640d273232b2de357c35729132256d" },
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-win_arm64.whl", hash = "sha256:99fc421a5d234580e45957a7b02effbf3e1c884a5dd077afc85352c77bf41434" }, { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-win_arm64.whl", hash = "sha256:99fc421a5d234580e45957a7b02effbf3e1c884a5dd077afc85352c77bf41434" },
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313-linux_s390x.whl", hash = "sha256:8b5882276633cf91fe3d2d7246c743b94d44a7e660b27f1308007fdb1bb89f7d" }, { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313-linux_s390x.whl", hash = "sha256:8b5882276633cf91fe3d2d7246c743b94d44a7e660b27f1308007fdb1bb89f7d" },
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:a5064b5e23772c8d164068cc7c12e01a75faf7b948ecd95a0d4007d7487e5f25" }, { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:a5064b5e23772c8d164068cc7c12e01a75faf7b948ecd95a0d4007d7487e5f25" },
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:8f81dedb4c6076ec325acc3b47525f9c550e5284a18eae1d9061c543f7b6e7de" }, { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:8f81dedb4c6076ec325acc3b47525f9c550e5284a18eae1d9061c543f7b6e7de" },
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313-win_amd64.whl", hash = "sha256:e1ee1b2346ade3ea90306dfbec7e8ff17bc220d344109d189ae09078333b0856" }, { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313-win_amd64.whl", hash = "sha256:e1ee1b2346ade3ea90306dfbec7e8ff17bc220d344109d189ae09078333b0856" },
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313-win_arm64.whl", hash = "sha256:64c187345509f2b1bb334feed4666e2c781ca381874bde589182f81247e61f88" }, { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313-win_arm64.whl", hash = "sha256:64c187345509f2b1bb334feed4666e2c781ca381874bde589182f81247e61f88" },
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:af81283ac671f434b1b25c95ba295f270e72db1fad48831eb5e4748ff9840041" }, { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:af81283ac671f434b1b25c95ba295f270e72db1fad48831eb5e4748ff9840041" },
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:a9dbb6f64f63258bc811e2c0c99640a81e5af93c531ad96e95c5ec777ea46dab" }, { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:a9dbb6f64f63258bc811e2c0c99640a81e5af93c531ad96e95c5ec777ea46dab" },
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313t-win_amd64.whl", hash = "sha256:6d93a7165419bc4b2b907e859ccab0dea5deeab261448ae9a5ec5431f14c0e64" }, { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313t-win_amd64.whl", hash = "sha256:6d93a7165419bc4b2b907e859ccab0dea5deeab261448ae9a5ec5431f14c0e64" },
] ]
[[package]] [[package]]
@ -5616,6 +5669,7 @@ dependencies = [
{ name = "aiosqlite" }, { name = "aiosqlite" },
{ name = "argon2-cffi" }, { name = "argon2-cffi" },
{ name = "asyncpg" }, { name = "asyncpg" },
{ name = "beautifulsoup4" },
{ name = "chardet" }, { name = "chardet" },
{ name = "colorlog" }, { name = "colorlog" },
{ name = "dashscope" }, { name = "dashscope" },
@ -5642,11 +5696,14 @@ dependencies = [
{ name = "llama-index" }, { name = "llama-index" },
{ name = "llama-index-readers-file" }, { name = "llama-index-readers-file" },
{ name = "loguru" }, { name = "loguru" },
{ name = "markdown-it-py" },
{ name = "markdownify" }, { name = "markdownify" },
{ name = "mcp" }, { name = "mcp" },
{ name = "mdit-py-plugins" },
{ name = "minio" }, { name = "minio" },
{ name = "neo4j" }, { name = "neo4j" },
{ name = "networkx" }, { name = "networkx" },
{ name = "nltk" },
{ name = "openai" }, { name = "openai" },
{ name = "opencv-python-headless" }, { name = "opencv-python-headless" },
{ name = "pillow" }, { name = "pillow" },
@ -5664,6 +5721,7 @@ dependencies = [
{ name = "readability-lxml" }, { name = "readability-lxml" },
{ name = "redis" }, { name = "redis" },
{ name = "rich" }, { name = "rich" },
{ name = "scikit-learn" },
{ name = "sqlalchemy", extra = ["asyncio"] }, { name = "sqlalchemy", extra = ["asyncio"] },
{ name = "tabulate" }, { name = "tabulate" },
{ name = "tavily-python" }, { name = "tavily-python" },
@ -5689,6 +5747,7 @@ requires-dist = [
{ name = "aiosqlite", specifier = ">=0.20.0" }, { name = "aiosqlite", specifier = ">=0.20.0" },
{ name = "argon2-cffi", specifier = ">=25.1.0" }, { name = "argon2-cffi", specifier = ">=25.1.0" },
{ name = "asyncpg", specifier = ">=0.30.0" }, { name = "asyncpg", specifier = ">=0.30.0" },
{ name = "beautifulsoup4", specifier = ">=4.12.0" },
{ name = "chardet", specifier = ">=5.0.0" }, { name = "chardet", specifier = ">=5.0.0" },
{ name = "colorlog", specifier = ">=6.9.0" }, { name = "colorlog", specifier = ">=6.9.0" },
{ name = "dashscope", specifier = ">=1.23.2" }, { name = "dashscope", specifier = ">=1.23.2" },
@ -5715,11 +5774,14 @@ requires-dist = [
{ name = "llama-index", specifier = ">=0.14" }, { name = "llama-index", specifier = ">=0.14" },
{ name = "llama-index-readers-file", specifier = ">=0.4.7" }, { name = "llama-index-readers-file", specifier = ">=0.4.7" },
{ name = "loguru", specifier = ">=0.7.3" }, { name = "loguru", specifier = ">=0.7.3" },
{ name = "markdown-it-py", specifier = ">=3.0.0" },
{ name = "markdownify", specifier = ">=1.1.0" }, { name = "markdownify", specifier = ">=1.1.0" },
{ name = "mcp", specifier = ">=1.20" }, { name = "mcp", specifier = ">=1.20" },
{ name = "mdit-py-plugins", specifier = ">=0.4.0" },
{ name = "minio", specifier = ">=7.2.7" }, { name = "minio", specifier = ">=7.2.7" },
{ name = "neo4j", specifier = ">=5.28.1" }, { name = "neo4j", specifier = ">=5.28.1" },
{ name = "networkx", specifier = ">=3.5" }, { name = "networkx", specifier = ">=3.5" },
{ name = "nltk", specifier = ">=3.8.1" },
{ name = "openai", specifier = ">=1.109" }, { name = "openai", specifier = ">=1.109" },
{ name = "opencv-python-headless", specifier = ">=4.11.0.86" }, { name = "opencv-python-headless", specifier = ">=4.11.0.86" },
{ name = "pillow", specifier = ">=10.5.0" }, { name = "pillow", specifier = ">=10.5.0" },
@ -5737,6 +5799,7 @@ requires-dist = [
{ name = "readability-lxml", specifier = ">=0.8.1" }, { name = "readability-lxml", specifier = ">=0.8.1" },
{ name = "redis", specifier = ">=5.2.0" }, { name = "redis", specifier = ">=5.2.0" },
{ name = "rich", specifier = ">=13.7.1" }, { name = "rich", specifier = ">=13.7.1" },
{ name = "scikit-learn", specifier = ">=1.3.0" },
{ name = "sqlalchemy", extras = ["asyncio"], specifier = ">=2.0.0" }, { name = "sqlalchemy", extras = ["asyncio"], specifier = ">=2.0.0" },
{ name = "tabulate", specifier = ">=0.9.0" }, { name = "tabulate", specifier = ">=0.9.0" },
{ name = "tavily-python", specifier = ">=0.7.0" }, { name = "tavily-python", specifier = ">=0.7.0" },

View File

@ -2,6 +2,7 @@
"name": "yuxi-web", "name": "yuxi-web",
"version": "0.6.0", "version": "0.6.0",
"private": true, "private": true,
"type": "module",
"scripts": { "scripts": {
"dev": "vite", "dev": "vite",
"server": "vite serve --host", "server": "vite serve --host",

View File

@ -183,8 +183,13 @@
size="16" size="16"
class="agent-switcher-menu-icon" class="agent-switcher-menu-icon"
/> />
<span class="agent-switcher-menu-text">{{ agent.name || 'Unknown' }}</span> <span class="agent-switcher-menu-text">{{
<span v-if="agent.id === currentAgentId" class="agent-switcher-menu-badge"> agent.name || 'Unknown'
}}</span>
<span
v-if="agent.id === currentAgentId"
class="agent-switcher-menu-badge"
>
当前 当前
</span> </span>
</div> </div>
@ -605,7 +610,10 @@ const conversationRows = computed(() => {
if (currentThreadConfigNotice.value) { if (currentThreadConfigNotice.value) {
const insertAfterCount = Math.max( const insertAfterCount = Math.max(
0, 0,
Math.min(Number(currentThreadConfigNotice.value.insertAfterConversationCount) || 0, rows.length) Math.min(
Number(currentThreadConfigNotice.value.insertAfterConversationCount) || 0,
rows.length
)
) )
rows.splice(insertAfterCount, 0, { rows.splice(insertAfterCount, 0, {
type: 'notice', type: 'notice',
@ -646,7 +654,10 @@ const showStartAgentSelector = computed(() => {
}) })
const showStartAgentDropdown = computed(() => { const showStartAgentDropdown = computed(() => {
return showStartAgentSelector.value && (startAgents.value.length >= 4 || localUIState.chatMainWidth < 380) return (
showStartAgentSelector.value &&
(startAgents.value.length >= 4 || localUIState.chatMainWidth < 380)
)
}) })
const showStartAgentSegment = computed(() => { const showStartAgentSegment = computed(() => {
@ -859,7 +870,11 @@ const queuePendingThreadConfigNotice = (threadId) => {
} }
const flushPendingThreadConfigNotice = (threadId) => { const flushPendingThreadConfigNotice = (threadId) => {
if (!threadId || !currentThreadHasHistory.value || !threadPendingConfigNoticeMap.value[threadId]) { if (
!threadId ||
!currentThreadHasHistory.value ||
!threadPendingConfigNoticeMap.value[threadId]
) {
return return
} }
@ -1419,7 +1434,11 @@ const selectChat = async (chatId) => {
// 线 // 线
chatState.currentThreadId = chatId chatState.currentThreadId = chatId
if (!props.singleMode && targetChat?.agent_id && targetChat.agent_id !== currentAgentId.value) { if (
!props.singleMode &&
targetChat?.agent_id &&
targetChat.agent_id !== currentAgentId.value
) {
await agentStore.selectAgent(targetChat.agent_id) await agentStore.selectAgent(targetChat.agent_id)
} }
@ -1912,10 +1931,10 @@ const hasVisibleAssistantBody = (message) => {
const { content, reasoningContent } = extractAssistantMessageBody(message) const { content, reasoningContent } = extractAssistantMessageBody(message)
return Boolean( return Boolean(
content || content ||
reasoningContent || reasoningContent ||
message.error_type || message.error_type ||
message.extra_metadata?.error_type || message.extra_metadata?.error_type ||
message.isStoppedByUser message.isStoppedByUser
) )
} }
@ -1997,9 +2016,7 @@ const isDisplayMessageProcessing = (conv, displayItem) => {
const isToolGroupActive = (conv, itemIndex, displayItems) => { const isToolGroupActive = (conv, itemIndex, displayItems) => {
return ( return (
isReplyLoading.value && isReplyLoading.value && conv?.status === 'streaming' && itemIndex === displayItems.length - 1
conv?.status === 'streaming' &&
itemIndex === displayItems.length - 1
) )
} }

View File

@ -57,11 +57,7 @@
/> />
<!-- 统一显示所有配置项 --> <!-- 统一显示所有配置项 -->
<template v-for="(value, key) in filteredConfigurableItems" :key="key"> <template v-for="(value, key) in filteredConfigurableItems" :key="key">
<a-form-item <a-form-item :label="getConfigLabel(key, value)" :name="key" class="config-item">
:label="getConfigLabel(key, value)"
:name="key"
class="config-item"
>
<p v-if="value.description" class="config-description">{{ value.description }}</p> <p v-if="value.description" class="config-description">{{ value.description }}</p>
<!-- <div>{{ value }}</div> --> <!-- <div>{{ value }}</div> -->

View File

@ -77,7 +77,7 @@
</div> </div>
<!-- Result Slot --> <!-- Result Slot -->
<div class="tool-result" style="opacity: 0.8;" v-if="hasResult"> <div class="tool-result" style="opacity: 0.8" v-if="hasResult">
<slot name="result" :tool-call="toolCall" :result-content="resultContent"> <slot name="result" :tool-call="toolCall" :result-content="resultContent">
<div class="tool-result-content" :data-tool-call-id="toolCall.id"> <div class="tool-result-content" :data-tool-call-id="toolCall.id">
<!-- Default rendering --> <!-- Default rendering -->
@ -234,7 +234,7 @@ const formatResultData = (data) => {
background-color: var(--gray-25); background-color: var(--gray-25);
} }
&>span { & > span {
display: flex; display: flex;
align-items: center; align-items: center;
} }

View File

@ -13,8 +13,12 @@
</span> </span>
<span class="summary-content"> <span class="summary-content">
<span class="summary-title">{{ toolCallsSummaryTitle }}</span> <span class="summary-title">{{ toolCallsSummaryTitle }}</span>
<span class="summary-separator" v-if="normalizedToolCalls.length > 1 && toolCallsNamesMeta">·</span> <span class="summary-separator" v-if="normalizedToolCalls.length > 1 && toolCallsNamesMeta"
<span class="summary-meta" v-if="normalizedToolCalls.length > 1 && toolCallsNamesMeta">{{ toolCallsNamesMeta }}</span> >·</span
>
<span class="summary-meta" v-if="normalizedToolCalls.length > 1 && toolCallsNamesMeta">{{
toolCallsNamesMeta
}}</span>
<span class="summary-status-tag" v-if="statusSummary">{{ statusSummary }}</span> <span class="summary-status-tag" v-if="statusSummary">{{ statusSummary }}</span>
</span> </span>
<span class="summary-trailing"> <span class="summary-trailing">
@ -123,8 +127,9 @@ const statusSummary = computed(() => {
(toolCall) => (toolCall) =>
toolCall.status !== 'success' && toolCall.status !== 'error' && !toolCall.tool_call_result toolCall.status !== 'success' && toolCall.status !== 'error' && !toolCall.tool_call_result
).length ).length
const errorCount = normalizedToolCalls.value.filter((toolCall) => toolCall.status === 'error') const errorCount = normalizedToolCalls.value.filter(
.length (toolCall) => toolCall.status === 'error'
).length
const parts = [] const parts = []
if (successCount > 0 && successCount === normalizedToolCalls.value.length) { if (successCount > 0 && successCount === normalizedToolCalls.value.length) {

View File

@ -208,10 +208,7 @@
<!-- 部门选择器仅超级管理员可见 --> <!-- 部门选择器仅超级管理员可见 -->
<a-form-item v-if="userStore.isSuperAdmin" label="部门" class="form-item"> <a-form-item v-if="userStore.isSuperAdmin" label="部门" class="form-item">
<a-select <a-select v-model:value="userManagement.form.departmentId" placeholder="请选择部门">
v-model:value="userManagement.form.departmentId"
placeholder="请选择部门"
>
<a-select-option <a-select-option
v-for="dept in departmentManagement.departments" v-for="dept in departmentManagement.departments"
:key="dept.id" :key="dept.id"

View File

@ -167,13 +167,8 @@ const route = useRoute()
const router = useRouter() const router = useRouter()
// agentStore // agentStore
const { const { selectedAgentId, defaultAgentId, selectedAgentConfigId, agentConfigs, isLoadingConfig } =
selectedAgentId, storeToRefs(agentStore)
defaultAgentId,
selectedAgentConfigId,
agentConfigs,
isLoadingConfig
} = storeToRefs(agentStore)
const syncingRouteThread = ref(false) const syncingRouteThread = ref(false)