docs: 为知识分块工具函数添加详细注释
- 为 html_table_to_key_value 函数添加处理逻辑说明，解释如何应对过长表格的切分问题 - 为 infer_heading_level 函数添加层级推断规则说明，包括数字序号和中文序号的处理 - 为 extract_table_block 函数添加表格提取算法的详细步骤说明 - 为 split_text_by_length_and_newline 函数添加分块逻辑的详细注释，解释段落和行的处理流程 - 为语义分块相关函数添加中文文档，包括句子分割、聚类算法和整体流程说明 - 更新测试用例，使用更清晰的测试文档并添加结果保存功能
This commit is contained in:
zhou shujian
parent
a2ed3d9a10
commit
edf66bbe38
@ -2,100 +2,107 @@ from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from markdown_it import MarkdownIt
|
||||
from mdit_py_plugins.dollarmath import dollarmath_plugin
|
||||
from yuxi.utils.logging_config import logger
|
||||
|
||||
from yuxi.knowledge.chunking.ragflow_like.nlp import count_tokens
|
||||
from yuxi.utils.logging_config import logger
|
||||
|
||||
from ..utils.md_parser_utils import (
|
||||
infer_heading_level,
|
||||
get_title_path,
|
||||
extract_table_block,
|
||||
split_text_by_length_and_newline
|
||||
extract_table_block,
|
||||
get_title_path,
|
||||
split_text_by_length_and_newline,
|
||||
)
|
||||
from ..utils.table_utils import html_table_to_key_value
|
||||
|
||||
|
||||
def _flush_content(
|
||||
result: list,
|
||||
current_content: list,
|
||||
title_stack: list,
|
||||
max_length: int,
|
||||
result: list,
|
||||
current_content: list,
|
||||
title_stack: list,
|
||||
max_length: int,
|
||||
embed_fn: Any,
|
||||
special_element: str = None,
|
||||
allow_split: bool = False
|
||||
) -> None:
|
||||
special_element: str = None,
|
||||
allow_split: bool = False,
|
||||
) -> None:
|
||||
if not current_content:
|
||||
return
|
||||
|
||||
content = '\n'.join(current_content).strip()
|
||||
|
||||
content = "\n".join(current_content).strip()
|
||||
if not content:
|
||||
current_content.clear()
|
||||
return
|
||||
|
||||
level = next((i + 1 for i in range(5, -1, -1) if title_stack[i]), 1)
|
||||
title_path = get_title_path(title_stack)
|
||||
|
||||
|
||||
if special_element and not allow_split:
|
||||
header = f"{'#' * level} {title_path}|{special_element}" if title_path else f"{'#' * level} {special_element}"
|
||||
result.extend([header, content, '-' * 10])
|
||||
result.extend([header, content, "-" * 10])
|
||||
else:
|
||||
if count_tokens(content) > max_length:
|
||||
chunks = split_text_by_length_and_newline(content, max_length, embed_fn=embed_fn, token_count_fn=count_tokens)
|
||||
chunks = split_text_by_length_and_newline(
|
||||
content, max_length, embed_fn=embed_fn, token_count_fn=count_tokens
|
||||
)
|
||||
for idx, chunk in enumerate(chunks, 1):
|
||||
base_header = f"{'#' * level} {title_path}" if title_path else f"{'#' * level}"
|
||||
if special_element:
|
||||
header = f"{base_header}|{special_element}|Part {idx}"
|
||||
else:
|
||||
header = f"{base_header}|Part {idx}"
|
||||
result.extend([header, chunk, '-' * 10])
|
||||
result.extend([header, chunk, "-" * 10])
|
||||
else:
|
||||
base_header = f"{'#' * level} {title_path}" if title_path else f"{'#' * level}"
|
||||
if special_element:
|
||||
header = f"{base_header}|{special_element}"
|
||||
else:
|
||||
header = base_header
|
||||
|
||||
|
||||
if header:
|
||||
result.append(header)
|
||||
result.append("")
|
||||
result.extend([content, '-' * 10])
|
||||
|
||||
result.extend([content, "-" * 10])
|
||||
|
||||
current_content.clear()
|
||||
|
||||
|
||||
def _handle_image_caption(tokens, i, result, current_content, title_stack, max_length, embed_fn):
|
||||
token = tokens[i]
|
||||
if token.type != 'paragraph_open':
|
||||
if token.type != "paragraph_open":
|
||||
return False, i
|
||||
|
||||
inline_token = tokens[i + 1]
|
||||
if inline_token.type != 'inline':
|
||||
return False, i
|
||||
|
||||
content = inline_token.content.strip()
|
||||
image_pattern = r'^!\[.*?\]\(.*?\)\s*$'
|
||||
caption_pattern = r'^(?:Figure|图|Fig\.|表|Table)\s*[\d\w\.]+'
|
||||
|
||||
img_match = re.search(r'^(!\[.*?\]\(.*?\))', content)
|
||||
inline_token = tokens[i + 1]
|
||||
if inline_token.type != "inline":
|
||||
return False, i
|
||||
|
||||
content = inline_token.content.strip()
|
||||
image_pattern = r"^!\[.*?\]\(.*?\)\s*$"
|
||||
caption_pattern = r"^(?:Figure|图|Fig\.|表|Table)\s*[\d\w\.]+"
|
||||
|
||||
img_match = re.search(r"^(!\[.*?\]\(.*?\))", content)
|
||||
if img_match:
|
||||
rest = content[img_match.end():].strip()
|
||||
rest = content[img_match.end() :].strip()
|
||||
if rest and re.match(caption_pattern, rest, re.IGNORECASE):
|
||||
_flush_content(result, current_content, title_stack, max_length, embed_fn)
|
||||
current_content.append(content)
|
||||
caption_title = rest.split('\n')[0].strip()
|
||||
caption_title = rest.split("\n")[0].strip()
|
||||
_flush_content(result, current_content, title_stack, max_length, embed_fn, special_element=caption_title)
|
||||
return True, i + 3
|
||||
|
||||
if re.match(image_pattern, content):
|
||||
next_p_idx = i + 3
|
||||
if next_p_idx + 1 < len(tokens) and tokens[next_p_idx].type == 'paragraph_open':
|
||||
if next_p_idx + 1 < len(tokens) and tokens[next_p_idx].type == "paragraph_open":
|
||||
next_inline = tokens[next_p_idx + 1]
|
||||
if next_inline.type == 'inline':
|
||||
if next_inline.type == "inline":
|
||||
next_content = next_inline.content.strip()
|
||||
if re.match(caption_pattern, next_content, re.IGNORECASE):
|
||||
_flush_content(result, current_content, title_stack, max_length, embed_fn)
|
||||
current_content.append(content)
|
||||
current_content.append(next_content)
|
||||
_flush_content(result, current_content, title_stack, max_length, embed_fn, special_element=next_content)
|
||||
_flush_content(
|
||||
result, current_content, title_stack, max_length, embed_fn, special_element=next_content
|
||||
)
|
||||
return True, i + 6
|
||||
|
||||
if current_content and re.match(caption_pattern, content, re.IGNORECASE):
|
||||
@ -110,14 +117,13 @@ def _handle_image_caption(tokens, i, result, current_content, title_stack, max_l
|
||||
|
||||
return False, i
|
||||
|
||||
|
||||
def chunk_markdown(
|
||||
markdown_content: str,
|
||||
parser_config: dict[str, Any] | None = None,
|
||||
embed_fn: Any | None = None
|
||||
markdown_content: str, parser_config: dict[str, Any] | None = None, embed_fn: Any | None = None
|
||||
) -> list[str]:
|
||||
"""
|
||||
语义化切分 Markdown 内容。
|
||||
|
||||
|
||||
Args:
|
||||
markdown_content: 待切分的 Markdown 文本
|
||||
parser_config: 切分参数，如 chunk_token_num
|
||||
@ -127,7 +133,7 @@ def chunk_markdown(
|
||||
parser_config = parser_config or {}
|
||||
max_length = int(parser_config.get("chunk_token_num", 512))
|
||||
logger.info(f"语义切分开始: max_length={max_length}, content_length={len(markdown_content)}")
|
||||
|
||||
|
||||
# 延迟加载重型资源，仅在没有注入 embed_fn 时触发
|
||||
if embed_fn is None:
|
||||
try:
|
||||
@ -142,61 +148,67 @@ def chunk_markdown(
|
||||
logger.error(f"加载 Embedding 模型失败: {e}。将退化为简单切分。")
|
||||
embed_fn = None
|
||||
|
||||
md = MarkdownIt('commonmark').enable('table')
|
||||
md = MarkdownIt("commonmark").enable("table")
|
||||
md.use(dollarmath_plugin, allow_space=True, allow_digits=True)
|
||||
|
||||
|
||||
tokens: list = md.parse(markdown_content)
|
||||
original_lines: list = markdown_content.split('\n')
|
||||
|
||||
original_lines: list = markdown_content.split("\n")
|
||||
|
||||
result: list = []
|
||||
current_content: list = []
|
||||
title_stack: list = [''] * 6
|
||||
title_stack: list = [""] * 6
|
||||
|
||||
i = 0
|
||||
while i < len(tokens):
|
||||
token = tokens[i]
|
||||
if token.type == 'heading_open':
|
||||
if token.type == "heading_open":
|
||||
_flush_content(result, current_content, title_stack, max_length, embed_fn)
|
||||
level = int(token.tag[1:]) if token.tag and len(token.tag) > 1 else 1
|
||||
inline_token = tokens[i + 1]
|
||||
if inline_token.type == 'inline':
|
||||
if inline_token.type == "inline":
|
||||
full_title = inline_token.content.strip()
|
||||
title_stack[level - 1] = full_title
|
||||
for j in range(level, 6):
|
||||
title_stack[j] = ''
|
||||
title_stack[j] = ""
|
||||
i += 3
|
||||
continue
|
||||
elif token.type == 'table_open':
|
||||
elif token.type == "table_open":
|
||||
_flush_content(result, current_content, title_stack, max_length, embed_fn)
|
||||
j, table_content = extract_table_block(tokens, i, original_lines)
|
||||
current_content.append(table_content)
|
||||
_flush_content(result, current_content, title_stack, max_length, embed_fn, special_element='Table')
|
||||
_flush_content(result, current_content, title_stack, max_length, embed_fn, special_element="Table")
|
||||
i = j + 1 if j < len(tokens) else len(tokens)
|
||||
continue
|
||||
elif token.type == 'paragraph_open':
|
||||
handled, new_i = _handle_image_caption(tokens, i, result, current_content, title_stack, max_length, embed_fn)
|
||||
elif token.type == "paragraph_open":
|
||||
handled, new_i = _handle_image_caption(
|
||||
tokens, i, result, current_content, title_stack, max_length, embed_fn
|
||||
)
|
||||
if handled:
|
||||
i = new_i
|
||||
continue
|
||||
inline_token = tokens[i + 1]
|
||||
if inline_token.type == 'inline':
|
||||
if inline_token.type == "inline":
|
||||
current_content.append(inline_token.content.strip())
|
||||
i += 3
|
||||
continue
|
||||
elif token.type == 'fence':
|
||||
elif token.type == "fence":
|
||||
current_content.append(f"```\n{token.content}\n```")
|
||||
i += 1
|
||||
continue
|
||||
elif token.type == 'ordered_list_open':
|
||||
elif token.type == "ordered_list_open":
|
||||
_flush_content(result, current_content, title_stack, max_length, embed_fn)
|
||||
list_content = []
|
||||
j = i + 1
|
||||
list_item_counter = 1
|
||||
while j < len(tokens) and tokens[j].type != 'ordered_list_close':
|
||||
if tokens[j].type == 'list_item_open':
|
||||
while j < len(tokens) and tokens[j].type != "ordered_list_close":
|
||||
if tokens[j].type == "list_item_open":
|
||||
k = j + 1
|
||||
while k < len(tokens) and tokens[k].type != 'list_item_close':
|
||||
if tokens[k].type == 'paragraph_open' and k + 1 < len(tokens) and tokens[k + 1].type == 'inline':
|
||||
while k < len(tokens) and tokens[k].type != "list_item_close":
|
||||
if (
|
||||
tokens[k].type == "paragraph_open"
|
||||
and k + 1 < len(tokens)
|
||||
and tokens[k + 1].type == "inline"
|
||||
):
|
||||
list_content.append(f"{list_item_counter}. {tokens[k + 1].content.strip()}")
|
||||
list_item_counter += 1
|
||||
k += 1
|
||||
@ -206,15 +218,19 @@ def chunk_markdown(
|
||||
_flush_content(result, current_content, title_stack, max_length, embed_fn, special_element=token.type)
|
||||
i = j + 1
|
||||
continue
|
||||
elif token.type == 'bullet_list_open':
|
||||
elif token.type == "bullet_list_open":
|
||||
_flush_content(result, current_content, title_stack, max_length, embed_fn)
|
||||
list_content = []
|
||||
j = i + 1
|
||||
while j < len(tokens) and tokens[j].type != 'bullet_list_close':
|
||||
if tokens[j].type == 'list_item_open':
|
||||
while j < len(tokens) and tokens[j].type != "bullet_list_close":
|
||||
if tokens[j].type == "list_item_open":
|
||||
k = j + 1
|
||||
while k < len(tokens) and tokens[k].type != 'list_item_close':
|
||||
if tokens[k].type == 'paragraph_open' and k + 1 < len(tokens) and tokens[k + 1].type == 'inline':
|
||||
while k < len(tokens) and tokens[k].type != "list_item_close":
|
||||
if (
|
||||
tokens[k].type == "paragraph_open"
|
||||
and k + 1 < len(tokens)
|
||||
and tokens[k + 1].type == "inline"
|
||||
):
|
||||
list_content.append(f"- {tokens[k + 1].content.strip()}")
|
||||
k += 1
|
||||
j += 1
|
||||
@ -223,33 +239,41 @@ def chunk_markdown(
|
||||
_flush_content(result, current_content, title_stack, max_length, embed_fn, special_element=token.type)
|
||||
i = j + 1
|
||||
continue
|
||||
elif token.type == 'html_block':
|
||||
elif token.type == "html_block":
|
||||
_flush_content(result, current_content, title_stack, max_length, embed_fn)
|
||||
content = token.content.strip()
|
||||
is_converted_table = False
|
||||
if '<table' in content.lower():
|
||||
if "<table" in content.lower():
|
||||
try:
|
||||
kv_list = html_table_to_key_value(content)
|
||||
if kv_list:
|
||||
content = '\n'.join([f"- {item}" for item in kv_list])
|
||||
content = "\n".join([f"- {item}" for item in kv_list])
|
||||
is_converted_table = True
|
||||
except Exception as e:
|
||||
logger.warning(f"HTML表格转KV失败: {e}")
|
||||
|
||||
|
||||
current_content.append(content)
|
||||
if is_converted_table:
|
||||
_flush_content(result, current_content, title_stack, max_length, embed_fn, special_element='Table KV', allow_split=True)
|
||||
_flush_content(
|
||||
result,
|
||||
current_content,
|
||||
title_stack,
|
||||
max_length,
|
||||
embed_fn,
|
||||
special_element="Table KV",
|
||||
allow_split=True,
|
||||
)
|
||||
else:
|
||||
_flush_content(result, current_content, title_stack, max_length, embed_fn, special_element=token.type)
|
||||
i += 1
|
||||
continue
|
||||
elif token.type in ['list_item_close', 'ordered_list_close', 'bullet_list_close', 'list_item_open']:
|
||||
elif token.type in ["list_item_close", "ordered_list_close", "bullet_list_close", "list_item_open"]:
|
||||
i += 1
|
||||
continue
|
||||
elif token.type == 'math_block':
|
||||
elif token.type == "math_block":
|
||||
_flush_content(result, current_content, title_stack, max_length, embed_fn)
|
||||
current_content.append(f"$ {token.content} $")
|
||||
_flush_content(result, current_content, title_stack, max_length, embed_fn, special_element='Math Block')
|
||||
_flush_content(result, current_content, title_stack, max_length, embed_fn, special_element="Math Block")
|
||||
i += 1
|
||||
continue
|
||||
else:
|
||||
@ -260,15 +284,15 @@ def chunk_markdown(
|
||||
chunks = []
|
||||
current_chunk_parts = []
|
||||
for item in result:
|
||||
if item == '-' * 10:
|
||||
if item == "-" * 10:
|
||||
if current_chunk_parts:
|
||||
chunks.append('\n'.join(current_chunk_parts).strip())
|
||||
chunks.append("\n".join(current_chunk_parts).strip())
|
||||
current_chunk_parts = []
|
||||
else:
|
||||
current_chunk_parts.append(item)
|
||||
|
||||
|
||||
if current_chunk_parts:
|
||||
chunks.append('\n'.join(current_chunk_parts).strip())
|
||||
chunks.append("\n".join(current_chunk_parts).strip())
|
||||
|
||||
logger.info(f"语义切分完成: chunks={len(chunks)}")
|
||||
return chunks
|
||||
|
||||
@ -1,35 +1,61 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import List, Callable, Any
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from .semantic_utils import semantic_chunking_with_auto_clusters
|
||||
|
||||
|
||||
def infer_heading_level(title: str) -> int:
|
||||
"""
|
||||
根据标题文本推断其层级级别（1-6级）。
|
||||
|
||||
逻辑说明：
|
||||
1. 数字序号推断：
|
||||
- 匹配如 "1.", "1.1", "1.2.3" 等格式。
|
||||
- 根据点号分隔的数量确定层级，例如 "1.1" 为 2 级，"1.2.3" 为 3 级。
|
||||
- 层级限制在 1-6 之间。
|
||||
2. 中文序号推断：
|
||||
- 匹配如 "一、", "二." 等中文数字序号。
|
||||
- 统一归类为 1 级标题。
|
||||
3. 默认处理：
|
||||
- 若不匹配以上规则，默认返回 1 级。
|
||||
"""
|
||||
m = re.match(r'^\s*(\d+(?:\.\d+)*)[.)、]?\s*', title)
|
||||
m = re.match(r"^\s*(\d+(?:\.\d+)*)[.)、]?\s*", title)
|
||||
if m:
|
||||
return max(1, min(len(m.group(1).split('.')), 6))
|
||||
m_zh = re.match(r'^\s*[一二三四五六七八九十百千]+[、.]\s*', title)
|
||||
return max(1, min(len(m.group(1).split(".")), 6))
|
||||
m_zh = re.match(r"^\s*[一二三四五六七八九十百千]+[、.]\s*", title)
|
||||
if m_zh:
|
||||
return 1
|
||||
return 1
|
||||
|
||||
def get_title_path(stack: List[str]) -> str:
|
||||
|
||||
def get_title_path(stack: list[str]) -> str:
|
||||
"""
|
||||
根据标题栈生成标题路径，用"|"分隔。
|
||||
"""
|
||||
return '|'.join([t for t in stack if t])
|
||||
return "|".join([t for t in stack if t])
|
||||
|
||||
def extract_table_block(tokens: List[Any], i: int, original_lines: List[str]) -> tuple[int, str]:
|
||||
|
||||
def extract_table_block(tokens: list[Any], i: int, original_lines: list[str]) -> tuple[int, str]:
|
||||
"""
|
||||
从token流和原始文本中提取完整的表格块。
|
||||
|
||||
逻辑说明：
|
||||
1. 定位起始：通过当前 token (i) 的 `map` 属性获取表格在原始行中的起始行号 `table_start`。
|
||||
2. 查找结束 token：遍历后续 tokens 直到找到 `table_close`。
|
||||
3. 确定结束行号 (`table_end`)：
|
||||
- 优先使用 `table_close` token 的 `map` 属性。
|
||||
- 若不存在，则尝试查找下一个带有 `map` 信息的 token 的起始行作为当前表格的结束。
|
||||
- 若上述均失败（如文件末尾或解析异常），则回退到基于文本内容的启发式扫描：
|
||||
从 `table_start` 开始向下扫描，直到遇到不符合 Markdown 表格特征（不以 '|' 开头且不含 '|'）的行为止。
|
||||
4. 返回结果：返回 `table_close` 的索引 `j` 以及拼接后的表格原始字符串。
|
||||
"""
|
||||
token = tokens[i]
|
||||
table_start = token.map[0] if token.map else 0
|
||||
j = i + 1
|
||||
while j < len(tokens) and tokens[j].type != 'table_close':
|
||||
while j < len(tokens) and tokens[j].type != "table_close":
|
||||
j += 1
|
||||
if j < len(tokens):
|
||||
end_token = tokens[j]
|
||||
@ -45,78 +71,80 @@ def extract_table_block(tokens: List[Any], i: int, original_lines: List[str]) ->
|
||||
table_end = table_start + 1
|
||||
for line_idx in range(table_start, len(original_lines)):
|
||||
line = original_lines[line_idx].strip()
|
||||
if not line or not (line.startswith('|') or '|' in line):
|
||||
if not line or not (line.startswith("|") or "|" in line):
|
||||
table_end = line_idx
|
||||
break
|
||||
else:
|
||||
table_end = table_start + 1
|
||||
for line_idx in range(table_start, len(original_lines)):
|
||||
line = original_lines[line_idx].strip()
|
||||
if not line or not (line.startswith('|') or '|' in line):
|
||||
if not line or not (line.startswith("|") or "|" in line):
|
||||
table_end = line_idx
|
||||
break
|
||||
return j, '\n'.join(original_lines[table_start:table_end])
|
||||
return j, "\n".join(original_lines[table_start:table_end])
|
||||
|
||||
|
||||
def split_text_by_length_and_newline(
|
||||
text: str,
|
||||
max_length: int,
|
||||
embed_fn: Callable[[List[str]], Any],
|
||||
token_count_fn: Callable[[str], int]
|
||||
) -> List[str]:
|
||||
text: str, max_length: int, embed_fn: Callable[[list[str]], Any], token_count_fn: Callable[[str], int]
|
||||
) -> list[str]:
|
||||
"""
|
||||
层次化文本切分策略。
|
||||
"""
|
||||
chunks = []
|
||||
|
||||
paragraphs = text.split('\n\n')
|
||||
|
||||
|
||||
paragraphs = text.split("\n\n")
|
||||
|
||||
for paragraph in paragraphs:
|
||||
paragraph = paragraph.strip()
|
||||
if not paragraph:
|
||||
continue
|
||||
|
||||
|
||||
paragraph_token_count = token_count_fn(paragraph)
|
||||
|
||||
# 如果当前段落长度未超过最大 Token 数量，直接作为独立分块放入chunks
|
||||
# 否则继续尝试按行切分
|
||||
if paragraph_token_count <= max_length:
|
||||
chunks.append(paragraph)
|
||||
continue
|
||||
|
||||
lines = paragraph.split('\n')
|
||||
|
||||
# 把段落进一步使用换行符进行切分为行
|
||||
lines = paragraph.split("\n")
|
||||
current_chunk_lines = []
|
||||
current_chunk_tokens = 0
|
||||
|
||||
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
if not line: # 跳过空行
|
||||
continue
|
||||
|
||||
line_token_count = token_count_fn(line)
|
||||
|
||||
line_token_count = token_count_fn(line) # 计算当前行的 Token 数量
|
||||
# 为了考虑行之间的空格，需要在计算 Token 数量时加 1（如果当前行不是第一行，需要添加一个换行符的Token数量）
|
||||
added_tokens = line_token_count + (1 if current_chunk_lines else 0)
|
||||
|
||||
# 如果当前行的 Token 数量超过最大 Token 数量，直接作为独立分块放入chunks
|
||||
if line_token_count > max_length:
|
||||
if current_chunk_lines:
|
||||
chunks.append('\n'.join(current_chunk_lines))
|
||||
chunks.append("\n".join(current_chunk_lines))
|
||||
current_chunk_lines = []
|
||||
current_chunk_tokens = 0
|
||||
|
||||
|
||||
sub_chunks = semantic_chunking_with_auto_clusters(
|
||||
line,
|
||||
embed_fn=embed_fn,
|
||||
token_count_fn=token_count_fn,
|
||||
max_chunk_size=max_length
|
||||
line, embed_fn=embed_fn, token_count_fn=token_count_fn, max_chunk_size=max_length
|
||||
)
|
||||
chunks.extend(sub_chunks)
|
||||
|
||||
# 如果当前行的 Token 数量与当前分块的 Token 数量合并后超过最大 Token 数量，直接作为独立分块放入chunks
|
||||
elif current_chunk_tokens + added_tokens > max_length:
|
||||
chunks.append('\n'.join(current_chunk_lines))
|
||||
# 把之前的分块内容放入chunks
|
||||
chunks.append("\n".join(current_chunk_lines))
|
||||
# 重置当前分块为当前行的内容
|
||||
current_chunk_lines = [line]
|
||||
# 更新当前分块的 Token 数量
|
||||
current_chunk_tokens = line_token_count
|
||||
|
||||
# 如果当前行的内容加入当前分块后不会超过最大 Token 数量，直接加入当前分块
|
||||
else:
|
||||
current_chunk_lines.append(line)
|
||||
current_chunk_tokens += added_tokens
|
||||
|
||||
current_chunk_tokens += added_tokens # 更新当前分块的 Token 数量
|
||||
# 最后的收尾，把最后一行内容放入chunks
|
||||
if current_chunk_lines:
|
||||
chunks.append('\n'.join(current_chunk_lines))
|
||||
|
||||
chunks.append("\n".join(current_chunk_lines))
|
||||
|
||||
return chunks
|
||||
|
||||
@ -1,37 +1,62 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
import nltk
|
||||
from nltk.tokenize import sent_tokenize
|
||||
from sklearn.cluster import AgglomerativeClustering
|
||||
from sklearn.metrics import silhouette_score
|
||||
from typing import Callable, List, Any
|
||||
|
||||
# Ensure punkt_tab is available
|
||||
# 模块加载的时候先检查是否已经下载了punkt_tab模型（用于识别句子的边界）
|
||||
try:
|
||||
nltk.data.find('tokenizers/punkt_tab')
|
||||
nltk.data.find("tokenizers/punkt_tab")
|
||||
except LookupError:
|
||||
nltk.download('punkt_tab')
|
||||
nltk.download("punkt_tab")
|
||||
|
||||
def split_sentences_chinese(text: str) -> List[str]:
|
||||
|
||||
def split_sentences_chinese(text: str) -> list[str]:
|
||||
"""
|
||||
Split sentences by Chinese punctuation while keeping the punctuation.
|
||||
使用正则表达式将中文文本分割成句子。
|
||||
|
||||
逻辑：
|
||||
- 匹配中文句号、感叹号、问号（。！？）作为分隔点。
|
||||
- 使用正向/反向预查处理引号：确保如果标点后面紧跟引号（”’"），该引号会被保留在当前句子末尾，而不是被切分到下一句。
|
||||
- 返回去除两端空格且非空的句子列表。
|
||||
"""
|
||||
pattern = r'(?<=[。！？])(?![”’"])|(?<=[。！？][”’"])'
|
||||
pattern = r'(?<=[。！？])(?![”’"])|(?<=[。！？])(?![”’"])|(?<=[。！？][”’"])'
|
||||
sentences = re.split(pattern, text)
|
||||
return [s.strip() for s in sentences if s.strip()]
|
||||
|
||||
def split_mixed_sentences(text: str) -> List[str]:
|
||||
|
||||
def split_mixed_sentences(text: str) -> list[str]:
|
||||
"""
|
||||
Handle both Chinese and English sentence splitting.
|
||||
处理中英文混合文本的分句逻辑，支持按物理段落分发不同的分句策略。
|
||||
|
||||
该函数采用“分而治之”的策略来处理复杂的混合文本：
|
||||
1. **物理分块**：首先按换行符 (`\\n+`) 将原始文本切分为多个物理段落（chunks），确保物理结构不被破坏。
|
||||
2. **语言检测与分发**：
|
||||
- **英文/混合路径**：若段落中包含英文字母 (`[A-Za-z]`)，则视为英文或混合文本，
|
||||
调用 NLTK 的 `sent_tokenize` 进行处理。NLTK 能更好地处理英文缩写、句点等复杂情况。
|
||||
- **中文路径**：若段落不含字母，则视为纯中文文本，调用 `split_sentences_chinese`。
|
||||
该方法通过正则精准匹配中文标点及后续引号。
|
||||
- **兜底方案**：若上述方法未产生结果，则使用简单的正则表达式按中文标点强制分割。
|
||||
3. **清洗与过滤**：汇总所有子句，去除两端空白字符，并过滤掉空字符串。
|
||||
|
||||
Args:
|
||||
text: 待分句的原始字符串。
|
||||
|
||||
Returns:
|
||||
List[str]: 分割后的句子列表。
|
||||
"""
|
||||
chunks = re.split(r'(\n+)', text)
|
||||
chunks = re.split(r"(\n+)", text)
|
||||
sentences = []
|
||||
|
||||
for ch in chunks:
|
||||
if not ch.strip():
|
||||
continue
|
||||
if re.search(r'[A-Za-z]', ch):
|
||||
if re.search(r"[A-Za-z]", ch):
|
||||
parts = sent_tokenize(ch)
|
||||
sentences.extend([p.strip() for p in parts if p.strip()])
|
||||
else:
|
||||
@ -39,13 +64,29 @@ def split_mixed_sentences(text: str) -> List[str]:
|
||||
if sents:
|
||||
sentences.extend([s.strip() for s in sents if s.strip()])
|
||||
else:
|
||||
parts = re.split(r'(?<=[。！？])', ch)
|
||||
parts = re.split(r"(?<=[。！？])", ch)
|
||||
sentences.extend([p.strip() for p in parts if p.strip()])
|
||||
return sentences
|
||||
|
||||
|
||||
def find_best_num_clusters(embeddings: Any, min_clusters: int = 2, max_clusters: int = 10) -> int:
|
||||
"""
|
||||
Select best number of clusters using silhouette score.
|
||||
使用轮廓系数选择最佳聚类数量。让每个分段语义集中，且分段之间界限分明
|
||||
|
||||
逻辑：
|
||||
- 遍历可能的聚类数量（从 min_clusters 到 max_clusters）。
|
||||
- 对每个聚类数量，使用 `AgglomerativeClustering` 进行聚类。
|
||||
- 计算轮廓系数（Silhouette Score）。
|
||||
- 选择轮廓系数最高的聚类数量作为最佳聚类数量。
|
||||
- 如果聚类数量为 1 或更少，直接返回 1。
|
||||
|
||||
Args:
|
||||
embeddings: 待聚类的向量数据（所有句子的嵌入向量列表）。
|
||||
min_clusters: 搜索的最佳聚类数量下限，默认为 2。
|
||||
max_clusters: 搜索的最佳聚类数量上限，默认为 10。
|
||||
|
||||
Returns:
|
||||
int: 轮廓系数表现最好的聚类数量。
|
||||
"""
|
||||
if len(embeddings) <= min_clusters:
|
||||
return len(embeddings)
|
||||
@ -55,42 +96,48 @@ def find_best_num_clusters(embeddings: Any, min_clusters: int = 2, max_clusters:
|
||||
|
||||
limit_k = min(max_clusters, len(embeddings))
|
||||
for k in range(min_clusters, limit_k + 1):
|
||||
labels = AgglomerativeClustering(n_clusters=k, metric='cosine', linkage='average').fit_predict(embeddings)
|
||||
labels = AgglomerativeClustering(n_clusters=k, metric="cosine", linkage="average").fit_predict(embeddings)
|
||||
if len(set(labels)) <= 1:
|
||||
continue
|
||||
score = silhouette_score(embeddings, labels, metric='cosine')
|
||||
score = silhouette_score(embeddings, labels, metric="cosine")
|
||||
if score > best_score:
|
||||
best_score = score
|
||||
best_k = k
|
||||
|
||||
return best_k
|
||||
|
||||
|
||||
def semantic_chunking_with_auto_clusters(
|
||||
text: str,
|
||||
embed_fn: Callable[[List[str]], Any],
|
||||
token_count_fn: Callable[[str], int],
|
||||
max_chunk_size: int = 512
|
||||
) -> List[str]:
|
||||
text: str, embed_fn: Callable[[list[str]], Any], token_count_fn: Callable[[str], int], max_chunk_size: int = 512
|
||||
) -> list[str]:
|
||||
"""
|
||||
Semantic chunking with automatic cluster number selection.
|
||||
对传入的文本进行语义切分，过程中会自动选择最佳的聚集数量。
|
||||
|
||||
逻辑：
|
||||
- 先将文本中的句子按语言进行分发，英文/混合文本使用NLTK的sent_tokenize，中文文本使用split_sentences_chinese。
|
||||
- 对每个句子进行嵌入向量化。
|
||||
- 确定最佳的聚类数量（根据轮廓系数）。
|
||||
- 对句子进行聚类，将每个聚类中的句子连接起来，形成一个分块。
|
||||
|
||||
"""
|
||||
sentences = split_mixed_sentences(text)
|
||||
if len(sentences) < 2:
|
||||
return [text.strip()]
|
||||
|
||||
# Vectorization
|
||||
# 向量化每个句子, 得到他们的嵌入向量
|
||||
embeddings = embed_fn(sentences)
|
||||
|
||||
# Pre-calculate token counts
|
||||
|
||||
# 计算每个句子的token数量
|
||||
sentence_token_counts = [token_count_fn(s) for s in sentences]
|
||||
total_tokens = sum(sentence_token_counts)
|
||||
|
||||
# Determine number of clusters
|
||||
# 决定合适的聚集数量，需要保证每个分块的token数量都不超过max_chunk_size
|
||||
best_k = max(total_tokens // max_chunk_size, 1) + 1
|
||||
best_k = min(best_k, len(sentences))
|
||||
|
||||
# Clustering
|
||||
labels = AgglomerativeClustering(n_clusters=best_k, metric='cosine', linkage='average').fit_predict(embeddings)
|
||||
|
||||
# 根据指定的聚集数量、相似度判断方式、联动方式，对句子进行聚类
|
||||
# 这里返回的labels是一个每个句子的聚类标签列表，例如[0,0,1,2,2]，相同ID的句子被聚类到同一个分块中
|
||||
labels = AgglomerativeClustering(n_clusters=best_k, metric="cosine", linkage="average").fit_predict(embeddings)
|
||||
|
||||
chunks = []
|
||||
current_chunk = ""
|
||||
|
||||
@ -1,57 +1,56 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import List
|
||||
from bs4 import BeautifulSoup
|
||||
from yuxi.utils.logging_config import logger
|
||||
|
||||
|
||||
def html_table_to_markdown(html: str) -> str:
|
||||
"""
|
||||
将HTML表格转换为Markdown格式的具体实现
|
||||
"""
|
||||
soup = BeautifulSoup(html, 'html.parser')
|
||||
table = soup.find('table')
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
table = soup.find("table")
|
||||
if table is None:
|
||||
return ''
|
||||
return ""
|
||||
|
||||
rows = table.find_all('tr')
|
||||
rows = table.find_all("tr")
|
||||
if not rows:
|
||||
return ''
|
||||
return ""
|
||||
|
||||
grid = []
|
||||
|
||||
|
||||
for r_idx, row in enumerate(rows):
|
||||
while len(grid) <= r_idx:
|
||||
grid.append([])
|
||||
|
||||
cells = row.find_all(['td', 'th'])
|
||||
|
||||
cells = row.find_all(["td", "th"])
|
||||
c_idx = 0
|
||||
|
||||
|
||||
for cell in cells:
|
||||
while c_idx < len(grid[r_idx]) and grid[r_idx][c_idx] is not None:
|
||||
c_idx += 1
|
||||
|
||||
|
||||
text = cell.get_text(strip=True)
|
||||
text = text.replace('\n', ' ')
|
||||
|
||||
rowspan = int(cell.get('rowspan', 1))
|
||||
colspan = int(cell.get('colspan', 1))
|
||||
|
||||
text = text.replace("\n", " ")
|
||||
|
||||
rowspan = int(cell.get("rowspan", 1))
|
||||
colspan = int(cell.get("colspan", 1))
|
||||
|
||||
for r in range(rowspan):
|
||||
target_r = r_idx + r
|
||||
while len(grid) <= target_r:
|
||||
grid.append([])
|
||||
|
||||
|
||||
for c in range(colspan):
|
||||
target_c = c_idx + c
|
||||
while len(grid[target_r]) <= target_c:
|
||||
grid[target_r].append(None)
|
||||
|
||||
|
||||
grid[target_r][target_c] = text
|
||||
|
||||
|
||||
c_idx += colspan
|
||||
|
||||
if not grid:
|
||||
return ''
|
||||
return ""
|
||||
|
||||
markdown_lines = []
|
||||
max_cols = max(len(r) for r in grid)
|
||||
@ -61,52 +60,65 @@ def html_table_to_markdown(html: str) -> str:
|
||||
|
||||
header = grid[0]
|
||||
header = [h if h is not None else "" for h in header]
|
||||
markdown_lines.append('| ' + ' | '.join(header) + ' |')
|
||||
markdown_lines.append('|' + '|'.join([' --- ' for _ in range(max_cols)]) + '|')
|
||||
markdown_lines.append("| " + " | ".join(header) + " |")
|
||||
markdown_lines.append("|" + "|".join([" --- " for _ in range(max_cols)]) + "|")
|
||||
|
||||
for row in grid[1:]:
|
||||
row_clean = [cell if cell is not None else "" for cell in row]
|
||||
line = '| ' + ' | '.join(row_clean) + ' |'
|
||||
line = "| " + " | ".join(row_clean) + " |"
|
||||
markdown_lines.append(line)
|
||||
|
||||
return '\n'.join(markdown_lines)
|
||||
return "\n".join(markdown_lines)
|
||||
|
||||
|
||||
def html_table_to_key_value(html: str) -> List[str]:
|
||||
def html_table_to_key_value(html: str) -> list[str]:
|
||||
"""
|
||||
将HTML表格转换为键值对格式的列表
|
||||
将HTML表格转换为键值对格式的列表，为了应对过长的表格的切分问题。
|
||||
|
||||
处理逻辑：
|
||||
1. **网格重建**：由于 HTML 表格可能包含 `rowspan` 和 `colspan`（合并单元格），
|
||||
函数首先构建一个完整的二维网格（grid）。
|
||||
2. **单元格展开**：遍历 HTML 行和列，遇到合并单元格时，将其内容填充到网格中受影响的所有坐标点。
|
||||
这确保了原本被合并的区域在逻辑网格中每个点都有对应的值。
|
||||
3. **键值对转换**：
|
||||
- 将网格的第一行视为表头（Key）。
|
||||
- 从第二行开始，将每一行与表头对应，生成 "键：值" 形式的字符串。
|
||||
|
||||
例如：
|
||||
- 输入：HTML表格，包含姓名、年龄、性别三列。
|
||||
- 输出：['姓名：张三；年龄：25；性别：男', '姓名：李四；年龄：30；性别：女']
|
||||
"""
|
||||
soup = BeautifulSoup(html, 'html.parser')
|
||||
table = soup.find('table')
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
table = soup.find("table")
|
||||
if table is None:
|
||||
return []
|
||||
|
||||
rows = table.find_all('tr')
|
||||
rows = table.find_all("tr")
|
||||
if not rows:
|
||||
return []
|
||||
|
||||
grid = []
|
||||
|
||||
|
||||
for r_idx, row in enumerate(rows):
|
||||
while len(grid) <= r_idx:
|
||||
grid.append([])
|
||||
|
||||
cells = row.find_all(['td', 'th'])
|
||||
|
||||
cells = row.find_all(["td", "th"])
|
||||
c_idx = 0
|
||||
|
||||
|
||||
for cell in cells:
|
||||
while c_idx < len(grid[r_idx]) and grid[r_idx][c_idx] is not None:
|
||||
c_idx += 1
|
||||
|
||||
|
||||
text = cell.get_text(strip=True)
|
||||
rowspan = int(cell.get('rowspan', 1))
|
||||
colspan = int(cell.get('colspan', 1))
|
||||
|
||||
rowspan = int(cell.get("rowspan", 1))
|
||||
colspan = int(cell.get("colspan", 1))
|
||||
|
||||
for r in range(rowspan):
|
||||
target_r = r_idx + r
|
||||
while len(grid) <= target_r:
|
||||
grid.append([])
|
||||
|
||||
|
||||
for c in range(colspan):
|
||||
target_c = c_idx + c
|
||||
while len(grid[target_r]) <= target_c:
|
||||
@ -116,10 +128,10 @@ def html_table_to_key_value(html: str) -> List[str]:
|
||||
|
||||
if not grid:
|
||||
return []
|
||||
|
||||
|
||||
headers = grid[0]
|
||||
headers = [h if h is not None else "" for h in headers]
|
||||
|
||||
|
||||
kv_lines = []
|
||||
for row_values in grid[1:]:
|
||||
min_len = min(len(headers), len(row_values))
|
||||
@ -131,5 +143,5 @@ def html_table_to_key_value(html: str) -> List[str]:
|
||||
row_parts.append(f"{key}：{val}")
|
||||
if row_parts:
|
||||
kv_lines.append("；".join(row_parts) + "；")
|
||||
|
||||
|
||||
return kv_lines
|
||||
|
||||
@ -475,10 +475,7 @@ async def find_user_by_oidc_sub(db, sub: str) -> User | None:
|
||||
# 方法1: 检查是否有用户的 user_id 直接等于 "oidc:{sub}"（标准 OIDC 用户）
|
||||
standard_oidc_user_id = f"oidc:{sub}"
|
||||
# 占位绑定记录会被标记为 is_deleted=1，但我们仍需要查询它们来获取绑定关系
|
||||
result = await db.execute(select(User).filter(
|
||||
User.user_id == standard_oidc_user_id,
|
||||
User.is_deleted == 0
|
||||
))
|
||||
result = await db.execute(select(User).filter(User.user_id == standard_oidc_user_id, User.is_deleted == 0))
|
||||
user = result.scalar_one_or_none()
|
||||
if user:
|
||||
return user
|
||||
@ -486,10 +483,9 @@ async def find_user_by_oidc_sub(db, sub: str) -> User | None:
|
||||
# 方法2: 检查是否有绑定占位用户格式: "oidc:{sub}:{target_user_id}"（use_raw_username 绑定记录）
|
||||
# 绑定占位用户被标记为 is_deleted=1，需要包括deleted来查询
|
||||
legacy_result = await db.execute(
|
||||
select(User).filter(
|
||||
User.user_id.like(f"{standard_oidc_user_id}:%"),
|
||||
User.is_deleted.in_([0, 1])
|
||||
).order_by(User.id.asc())
|
||||
select(User)
|
||||
.filter(User.user_id.like(f"{standard_oidc_user_id}:%"), User.is_deleted.in_([0, 1]))
|
||||
.order_by(User.id.asc())
|
||||
)
|
||||
legacy_users = list(legacy_result.scalars().all())
|
||||
if legacy_users:
|
||||
@ -528,10 +524,7 @@ async def find_deleted_oidc_user_by_sub(db, sub: str) -> User | None:
|
||||
|
||||
# 检查绑定占位格式 oidc:{sub}:{target_user_id}（占位本身是deleted，需要查询目标用户）
|
||||
legacy_result = await db.execute(
|
||||
select(User).filter(
|
||||
User.user_id.like(f"{oidc_user_id}:%"),
|
||||
User.is_deleted == 1
|
||||
).order_by(User.id.asc())
|
||||
select(User).filter(User.user_id.like(f"{oidc_user_id}:%"), User.is_deleted == 1).order_by(User.id.asc())
|
||||
)
|
||||
legacy_users = list(legacy_result.scalars().all())
|
||||
if legacy_users:
|
||||
@ -585,6 +578,7 @@ async def _create_oidc_binding_placeholder(db, sub: str, target_user: User) -> N
|
||||
|
||||
# username 使用 oidc-binding-{sub_hash} 避免冲突，sub_hash 基于完整 sub 生成
|
||||
import hashlib
|
||||
|
||||
sub_hash = hashlib.sha256(sub.encode()).hexdigest()[:8]
|
||||
username = f"oidc-binding-{sub_hash}"
|
||||
|
||||
@ -604,8 +598,7 @@ async def _create_oidc_binding_placeholder(db, sub: str, target_user: User) -> N
|
||||
db.add(placeholder_user)
|
||||
await db.commit()
|
||||
logger.info(
|
||||
f"Created OIDC binding placeholder (deleted) for sub {sub} -> "
|
||||
f"user {target_user.id} ({target_user.user_id})"
|
||||
f"Created OIDC binding placeholder (deleted) for sub {sub} -> user {target_user.id} ({target_user.user_id})"
|
||||
)
|
||||
except IntegrityError:
|
||||
# 并发创建冲突，回滚后忽略
|
||||
@ -661,8 +654,7 @@ async def create_oidc_user(db, user_info: dict, department_id: int | None = None
|
||||
if user_by_sub and user_by_sub.id == existing_user.id:
|
||||
# sub 已经正确绑定到该用户，允许返回
|
||||
logger.info(
|
||||
f"User with raw username {user_id} already exists and "
|
||||
f"bound to sub {sub}, returning existing user"
|
||||
f"User with raw username {user_id} already exists and bound to sub {sub}, returning existing user"
|
||||
)
|
||||
return existing_user
|
||||
elif user_by_sub is None:
|
||||
|
||||
2211
backend/test/resource/test4.md
Normal file
2211
backend/test/resource/test4.md
Normal file
File diff suppressed because it is too large
Load Diff
7
backend/test/unit/backends/resource/semantic_chunks.json
Normal file
7
backend/test/unit/backends/resource/semantic_chunks.json
Normal file
@ -0,0 +1,7 @@
|
||||
[
|
||||
"## 风力发电机组 塔架设计规范|1 概述\n\n本标准规定了风力发电机组塔架的设计、制造、运输和安装要求。",
|
||||
"### 风力发电机组 塔架设计规范|6 钢制塔架|6.1 一般要求\n\n钢制塔架的设计应考虑极端载荷和疲劳载荷。在计算连接强度时，需要用到螺纹截面积 Asp 以及承载力设计值 Rd。\n此外，结构应力 σ _ {y, d} 的计算必须符合相关标准要求。",
|
||||
"### 风力发电机组 塔架设计规范|6 钢制塔架|6.2 疲劳极限状态\n\n疲劳计算应基于 Miner 线性累积损伤理论。",
|
||||
"### 风力发电机组 塔架设计规范|7 混凝土塔架|7.1 材料特性\n\n混凝土强度等级不应低于 C50。",
|
||||
"### 风力发电机组 塔架设计规范|7 混凝土塔架|7.2 施工工艺\n\n混凝土塔架可采用现浇或预制拼装方式。对于预制片段，应严格控制拼装精度。"
|
||||
]
|
||||
@ -1,6 +1,7 @@
|
||||
import pytest
|
||||
import numpy as np
|
||||
import os
|
||||
import json
|
||||
|
||||
# 现在可以安全地导入了，因为顶层不再有重型依赖
|
||||
from yuxi.knowledge.chunking.ragflow_like.parsers import semantic
|
||||
@ -22,57 +23,87 @@ def embed_fn():
|
||||
|
||||
@pytest.fixture
|
||||
def sample_markdown():
|
||||
with open("test/resource/test4.md", "r", encoding="utf-8") as f:
|
||||
return f.read()
|
||||
"""提供一个包含多章节、公式符号和复杂结构的临时 Markdown 样本"""
|
||||
return """
|
||||
# 风力发电机组 塔架设计规范
|
||||
|
||||
## 1 概述
|
||||
本标准规定了风力发电机组塔架的设计、制造、运输和安装要求。
|
||||
|
||||
## 6 钢制塔架
|
||||
### 6.1 一般要求
|
||||
钢制塔架的设计应考虑极端载荷和疲劳载荷。在计算连接强度时，需要用到螺纹截面积 Asp 以及承载力设计值 Rd。
|
||||
此外，结构应力 σ _ {y, d} 的计算必须符合相关标准要求。
|
||||
|
||||
### 6.2 疲劳极限状态
|
||||
疲劳计算应基于 Miner 线性累积损伤理论。
|
||||
|
||||
## 7 混凝土塔架
|
||||
### 7.1 材料特性
|
||||
混凝土强度等级不应低于 C50。
|
||||
|
||||
### 7.2 施工工艺
|
||||
混凝土塔架可采用现浇或预制拼装方式。对于预制片段，应严格控制拼装精度。
|
||||
"""
|
||||
def test_semantic_chunking_basic(embed_fn, sample_markdown):
|
||||
"""测试基本的语义切分逻辑 (使用真实嵌入模型)"""
|
||||
|
||||
|
||||
# 配置切分参数
|
||||
parser_config = {
|
||||
"chunk_token_num": 200, # 适中的 token 数以触发更多切分
|
||||
"overlapped_percent": 0
|
||||
"chunk_token_num": 300, # 针对标准文档调整 token 数
|
||||
"overlapped_percent": 0.1
|
||||
}
|
||||
|
||||
# 直接注入 embed_fn
|
||||
|
||||
# 执行语义切分
|
||||
chunks = semantic.chunk_markdown(
|
||||
sample_markdown,
|
||||
parser_config=parser_config,
|
||||
sample_markdown,
|
||||
parser_config=parser_config,
|
||||
embed_fn=embed_fn
|
||||
)
|
||||
|
||||
# 验证结果
|
||||
|
||||
# 1. 基础验证
|
||||
assert isinstance(chunks, list)
|
||||
assert len(chunks) > 0
|
||||
|
||||
print(f"\n成功切分为 {len(chunks)} 个片段:")
|
||||
for i, chunk in enumerate(chunks):
|
||||
print(f"--- 片段 {i+1} ---")
|
||||
print(chunk)
|
||||
|
||||
# 验证标题路径增强是否生效 (检查是否包含 Part 或特殊元素标记)
|
||||
has_enhanced_title = any("|Part" in chunk or "|Math Block" in chunk for chunk in chunks)
|
||||
assert has_enhanced_title, "标题路径增强或分片标记失效"
|
||||
|
||||
# 验证是否包含特殊的 LaTeX 公式或符号 (匹配输出中的格式)
|
||||
has_formula = any("f _ {0, 1}" in chunk or "\\sigma_ {y, d}" in chunk or "A _ {\\mathrm {s p}}" in chunk for chunk in chunks)
|
||||
assert has_formula, "LaTeX 公式/符号在切分中丢失"
|
||||
assert len(chunks) >= 5, f"预期至少切分为 5 个片段，实际仅有 {len(chunks)} 个"
|
||||
|
||||
# 验证语义聚类是否大概生效 (钢制塔架和混凝土塔架内容应该分开)
|
||||
# 我们检查是否有钢制塔架相关的片段和混凝土塔架相关的片段
|
||||
# 使用 startswith 来更准确地定位章节开始，避免匹配到目次/前言中的文字
|
||||
has_steel = any(chunk.startswith("# 6 钢制塔架") for chunk in chunks)
|
||||
has_concrete = any(chunk.startswith("# 7 混凝土塔架") for chunk in chunks)
|
||||
|
||||
assert has_steel and has_concrete, "语义内容丢失 (钢制或混凝土塔架章节开始部分)"
|
||||
|
||||
# 验证它们是否在不同的片段中
|
||||
steel_chunks = [i for i, c in enumerate(chunks) if c.startswith("# 6 钢制塔架")]
|
||||
concrete_chunks = [i for i, c in enumerate(chunks) if c.startswith("# 7 混凝土塔架")]
|
||||
|
||||
# 理论上语义聚类会将这些大章节分开
|
||||
if steel_chunks and concrete_chunks:
|
||||
assert not set(steel_chunks).intersection(set(concrete_chunks)), "钢制塔架和混凝土塔架大章节内容被错误地聚类在同一个 chunk 中了"
|
||||
# 2. 验证标题路径增强 (检查 RAGFlow 风格的层级标记)
|
||||
# 标准文档通常会被赋予类似 "风力发电机组 塔架 | 6 钢制塔架" 的标题路径
|
||||
has_enhanced_title = any("|" in chunk and ("钢制塔架" in chunk or "混凝土塔架" in chunk) for chunk in chunks)
|
||||
assert has_enhanced_title, "标题路径增强失效：未在 chunk 中发现层级分隔符 '|' 或核心章节标题"
|
||||
|
||||
# 3. 验证公式与符号识别 (匹配 test4.md 4.1 节中的符号)
|
||||
# 重点检查文档中出现的符号：Asp (螺纹截面积), σ _ {y, d} (结构应力), Rd (承载力设计值)
|
||||
symbols_to_check = ["Asp", "Rd", "σ _ {y, d}"]
|
||||
found_symbols = [s for s in symbols_to_check if any(s in chunk for chunk in chunks)]
|
||||
assert len(found_symbols) > 0, f"在切分结果中未找到关键符号: {symbols_to_check}"
|
||||
|
||||
# 4. 验证核心章节内容是否存在
|
||||
has_steel_section = any("6 钢制塔架" in chunk for chunk in chunks)
|
||||
has_concrete_section = any("7 混凝土塔架" in chunk for chunk in chunks)
|
||||
assert has_steel_section, "未找到第 6 章 '钢制塔架' 相关内容"
|
||||
assert has_concrete_section, "未找到第 7 章 '混凝土塔架' 相关内容"
|
||||
|
||||
# 5. 验证语义聚类是否将不同主题分开
|
||||
# 钢制塔架和混凝土塔架是两个独立的大章节，语义聚类应该避免将它们的核心内容混在一个 chunk 中
|
||||
steel_start_chunks = [i for i, c in enumerate(chunks) if "# 6 钢制塔架" in c]
|
||||
concrete_start_chunks = [i for i, c in enumerate(chunks) if "# 7 混凝土塔架" in c]
|
||||
|
||||
if steel_start_chunks and concrete_start_chunks:
|
||||
# 确保起始片段不重合
|
||||
assert not set(steel_start_chunks).intersection(set(concrete_start_chunks)), \
|
||||
"语义聚类错误：钢制塔架与混凝土塔架的章节头部被挤在了同一个 chunk 中"
|
||||
|
||||
print(f"\n[测试成功] 文档成功切分为 {len(chunks)} 个片段")
|
||||
print(f"识别到的关键符号: {found_symbols}")
|
||||
|
||||
# 将切分后的内容写入到 resource 目录
|
||||
output_dir = os.path.join(os.path.dirname(__file__), "resource")
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
output_path = os.path.join(output_dir, "semantic_chunks.json")
|
||||
with open(output_path, "w", encoding="utf-8") as f:
|
||||
json.dump(chunks, f, ensure_ascii=False, indent=2)
|
||||
print(f"切分内容已保存至: {output_path}")
|
||||
|
||||
def test_heading_inference():
|
||||
"""测试标题层级推断工具类"""
|
||||
from yuxi.knowledge.chunking.ragflow_like.utils.md_parser_utils import infer_heading_level
|
||||
|
||||
@ -2521,6 +2521,18 @@ wheels = [
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/9c/46/f6b4ad632c67ef35209a66127e4bddc95759649dd595f71f13fba11bdf9a/mcp-1.27.0-py3-none-any.whl", hash = "sha256:5ce1fa81614958e267b21fb2aa34e0aea8e2c6ede60d52aba45fd47246b4d741", size = 215967, upload-time = "2026-04-02T14:48:07.24Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mdit-py-plugins"
|
||||
version = "0.5.0"
|
||||
source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" }
|
||||
dependencies = [
|
||||
{ name = "markdown-it-py" },
|
||||
]
|
||||
sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/b2/fd/a756d36c0bfba5f6e39a1cdbdbfdd448dc02692467d83816dff4592a1ebc/mdit_py_plugins-0.5.0.tar.gz", hash = "sha256:f4918cb50119f50446560513a8e311d574ff6aaed72606ddae6d35716fe809c6", size = 44655, upload-time = "2025-08-11T07:25:49.083Z" }
|
||||
wheels = [
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/fb/86/dd6e5db36df29e76c7a7699123569a4a18c1623ce68d826ed96c62643cae/mdit_py_plugins-0.5.0-py3-none-any.whl", hash = "sha256:07a08422fc1936a5d26d146759e9155ea466e842f5ab2f7d2266dd084c8dab1f", size = 57205, upload-time = "2025-08-11T07:25:47.597Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mdurl"
|
||||
version = "0.1.2"
|
||||
@ -4337,6 +4349,38 @@ torch = [
|
||||
{ name = "torch", version = "2.8.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform != 'darwin'" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "scikit-learn"
|
||||
version = "1.8.0"
|
||||
source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" }
|
||||
dependencies = [
|
||||
{ name = "joblib" },
|
||||
{ name = "numpy" },
|
||||
{ name = "scipy" },
|
||||
{ name = "threadpoolctl" },
|
||||
]
|
||||
sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/0e/d4/40988bf3b8e34feec1d0e6a051446b1f66225f8529b9309becaeef62b6c4/scikit_learn-1.8.0.tar.gz", hash = "sha256:9bccbb3b40e3de10351f8f5068e105d0f4083b1a65fa07b6634fbc401a6287fd", size = 7335585, upload-time = "2025-12-10T07:08:53.618Z" }
|
||||
wheels = [
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/90/74/e6a7cc4b820e95cc38cf36cd74d5aa2b42e8ffc2d21fe5a9a9c45c1c7630/scikit_learn-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5fb63362b5a7ddab88e52b6dbb47dac3fd7dafeee740dc6c8d8a446ddedade8e", size = 8548242, upload-time = "2025-12-10T07:07:51.568Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/49/d8/9be608c6024d021041c7f0b3928d4749a706f4e2c3832bbede4fb4f58c95/scikit_learn-1.8.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:5025ce924beccb28298246e589c691fe1b8c1c96507e6d27d12c5fadd85bfd76", size = 8079075, upload-time = "2025-12-10T07:07:53.697Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/dd/47/f187b4636ff80cc63f21cd40b7b2d177134acaa10f6bb73746130ee8c2e5/scikit_learn-1.8.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4496bb2cf7a43ce1a2d7524a79e40bc5da45cf598dbf9545b7e8316ccba47bb4", size = 8660492, upload-time = "2025-12-10T07:07:55.574Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/97/74/b7a304feb2b49df9fafa9382d4d09061a96ee9a9449a7cbea7988dda0828/scikit_learn-1.8.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a0bcfe4d0d14aec44921545fd2af2338c7471de9cb701f1da4c9d85906ab847a", size = 8931904, upload-time = "2025-12-10T07:07:57.666Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/9f/c4/0ab22726a04ede56f689476b760f98f8f46607caecff993017ac1b64aa5d/scikit_learn-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:35c007dedb2ffe38fe3ee7d201ebac4a2deccd2408e8621d53067733e3c74809", size = 8019359, upload-time = "2025-12-10T07:07:59.838Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/24/90/344a67811cfd561d7335c1b96ca21455e7e472d281c3c279c4d3f2300236/scikit_learn-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:8c497fff237d7b4e07e9ef1a640887fa4fb765647f86fbe00f969ff6280ce2bb", size = 7641898, upload-time = "2025-12-10T07:08:01.36Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/03/aa/e22e0768512ce9255eba34775be2e85c2048da73da1193e841707f8f039c/scikit_learn-1.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0d6ae97234d5d7079dc0040990a6f7aeb97cb7fa7e8945f1999a429b23569e0a", size = 8513770, upload-time = "2025-12-10T07:08:03.251Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/58/37/31b83b2594105f61a381fc74ca19e8780ee923be2d496fcd8d2e1147bd99/scikit_learn-1.8.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:edec98c5e7c128328124a029bceb09eda2d526997780fef8d65e9a69eead963e", size = 8044458, upload-time = "2025-12-10T07:08:05.336Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/2d/5a/3f1caed8765f33eabb723596666da4ebbf43d11e96550fb18bdec42b467b/scikit_learn-1.8.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:74b66d8689d52ed04c271e1329f0c61635bcaf5b926db9b12d58914cdc01fe57", size = 8610341, upload-time = "2025-12-10T07:08:07.732Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/38/cf/06896db3f71c75902a8e9943b444a56e727418f6b4b4a90c98c934f51ed4/scikit_learn-1.8.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8fdf95767f989b0cfedb85f7ed8ca215d4be728031f56ff5a519ee1e3276dc2e", size = 8900022, upload-time = "2025-12-10T07:08:09.862Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/1c/f9/9b7563caf3ec8873e17a31401858efab6b39a882daf6c1bfa88879c0aa11/scikit_learn-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:2de443b9373b3b615aec1bb57f9baa6bb3a9bd093f1269ba95c17d870422b271", size = 7989409, upload-time = "2025-12-10T07:08:12.028Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/49/bd/1f4001503650e72c4f6009ac0c4413cb17d2d601cef6f71c0453da2732fc/scikit_learn-1.8.0-cp313-cp313-win_arm64.whl", hash = "sha256:eddde82a035681427cbedded4e6eff5e57fa59216c2e3e90b10b19ab1d0a65c3", size = 7619760, upload-time = "2025-12-10T07:08:13.688Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/d2/7d/a630359fc9dcc95496588c8d8e3245cc8fd81980251079bc09c70d41d951/scikit_learn-1.8.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:7cc267b6108f0a1499a734167282c00c4ebf61328566b55ef262d48e9849c735", size = 8826045, upload-time = "2025-12-10T07:08:15.215Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/cc/56/a0c86f6930cfcd1c7054a2bc417e26960bb88d32444fe7f71d5c2cfae891/scikit_learn-1.8.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:fe1c011a640a9f0791146011dfd3c7d9669785f9fed2b2a5f9e207536cf5c2fd", size = 8420324, upload-time = "2025-12-10T07:08:17.561Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/46/1e/05962ea1cebc1cf3876667ecb14c283ef755bf409993c5946ade3b77e303/scikit_learn-1.8.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:72358cce49465d140cc4e7792015bb1f0296a9742d5622c67e31399b75468b9e", size = 8680651, upload-time = "2025-12-10T07:08:19.952Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/fe/56/a85473cd75f200c9759e3a5f0bcab2d116c92a8a02ee08ccd73b870f8bb4/scikit_learn-1.8.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:80832434a6cc114f5219211eec13dcbc16c2bac0e31ef64c6d346cde3cf054cb", size = 8925045, upload-time = "2025-12-10T07:08:22.11Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/cc/b7/64d8cfa896c64435ae57f4917a548d7ac7a44762ff9802f75a79b77cb633/scikit_learn-1.8.0-cp313-cp313t-win_amd64.whl", hash = "sha256:ee787491dbfe082d9c3013f01f5991658b0f38aa8177e4cd4bf434c58f551702", size = 8507994, upload-time = "2025-12-10T07:08:23.943Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/5e/37/e192ea709551799379958b4c4771ec507347027bb7c942662c7fbeba31cb/scikit_learn-1.8.0-cp313-cp313t-win_arm64.whl", hash = "sha256:bf97c10a3f5a7543f9b88cbf488d33d175e9146115a451ae34568597ba33dcde", size = 7869518, upload-time = "2025-12-10T07:08:25.71Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "scipy"
|
||||
version = "1.17.1"
|
||||
@ -4756,6 +4800,15 @@ wheels = [
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/f5/15/a11f7bb3cbc97dfecf32a90552f5a8f8a5c99316a99c6c17bdabf5baf256/thinc-8.3.13-cp313-cp313-win_arm64.whl", hash = "sha256:723949cab11d1925c15447928513a718276316cec6e0de28337cca0a62be0521", size = 1644606, upload-time = "2026-03-23T07:22:21.339Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "threadpoolctl"
|
||||
version = "3.6.0"
|
||||
source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" }
|
||||
sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/b7/4d/08c89e34946fce2aec4fbb45c9016efd5f4d7f24af8e5d93296e935631d8/threadpoolctl-3.6.0.tar.gz", hash = "sha256:8ab8b4aa3491d812b623328249fab5302a68d2d71745c8a4c719a2fcaba9f44e", size = 21274, upload-time = "2025-03-13T13:49:23.031Z" }
|
||||
wheels = [
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl", hash = "sha256:43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb", size = 18638, upload-time = "2025-03-13T13:49:21.846Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tiktoken"
|
||||
version = "0.12.0"
|
||||
@ -4878,9 +4931,9 @@ dependencies = [
|
||||
{ name = "typing-extensions", marker = "sys_platform == 'darwin'" },
|
||||
]
|
||||
wheels = [
|
||||
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:a47b7986bee3f61ad217d8a8ce24605809ab425baf349f97de758815edd2ef54" },
|
||||
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:fbe2e149c5174ef90d29a5f84a554dfaf28e003cb4f61fa2c8c024c17ec7ca58" },
|
||||
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0-cp313-none-macosx_11_0_arm64.whl", hash = "sha256:057efd30a6778d2ee5e2374cd63a63f63311aa6f33321e627c655df60abdd390" },
|
||||
{ url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:a47b7986bee3f61ad217d8a8ce24605809ab425baf349f97de758815edd2ef54" },
|
||||
{ url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:fbe2e149c5174ef90d29a5f84a554dfaf28e003cb4f61fa2c8c024c17ec7ca58" },
|
||||
{ url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0-cp313-none-macosx_11_0_arm64.whl", hash = "sha256:057efd30a6778d2ee5e2374cd63a63f63311aa6f33321e627c655df60abdd390" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -4903,19 +4956,19 @@ dependencies = [
|
||||
{ name = "typing-extensions", marker = "sys_platform != 'darwin'" },
|
||||
]
|
||||
wheels = [
|
||||
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-linux_s390x.whl", hash = "sha256:0e34e276722ab7dd0dffa9e12fe2135a9b34a0e300c456ed7ad6430229404eb5" },
|
||||
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:610f600c102386e581327d5efc18c0d6edecb9820b4140d26163354a99cd800d" },
|
||||
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:cb9a8ba8137ab24e36bf1742cb79a1294bd374db570f09fc15a5e1318160db4e" },
|
||||
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-win_amd64.whl", hash = "sha256:2be20b2c05a0cce10430cc25f32b689259640d273232b2de357c35729132256d" },
|
||||
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-win_arm64.whl", hash = "sha256:99fc421a5d234580e45957a7b02effbf3e1c884a5dd077afc85352c77bf41434" },
|
||||
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313-linux_s390x.whl", hash = "sha256:8b5882276633cf91fe3d2d7246c743b94d44a7e660b27f1308007fdb1bb89f7d" },
|
||||
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:a5064b5e23772c8d164068cc7c12e01a75faf7b948ecd95a0d4007d7487e5f25" },
|
||||
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:8f81dedb4c6076ec325acc3b47525f9c550e5284a18eae1d9061c543f7b6e7de" },
|
||||
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313-win_amd64.whl", hash = "sha256:e1ee1b2346ade3ea90306dfbec7e8ff17bc220d344109d189ae09078333b0856" },
|
||||
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313-win_arm64.whl", hash = "sha256:64c187345509f2b1bb334feed4666e2c781ca381874bde589182f81247e61f88" },
|
||||
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:af81283ac671f434b1b25c95ba295f270e72db1fad48831eb5e4748ff9840041" },
|
||||
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:a9dbb6f64f63258bc811e2c0c99640a81e5af93c531ad96e95c5ec777ea46dab" },
|
||||
{ url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313t-win_amd64.whl", hash = "sha256:6d93a7165419bc4b2b907e859ccab0dea5deeab261448ae9a5ec5431f14c0e64" },
|
||||
{ url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-linux_s390x.whl", hash = "sha256:0e34e276722ab7dd0dffa9e12fe2135a9b34a0e300c456ed7ad6430229404eb5" },
|
||||
{ url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:610f600c102386e581327d5efc18c0d6edecb9820b4140d26163354a99cd800d" },
|
||||
{ url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:cb9a8ba8137ab24e36bf1742cb79a1294bd374db570f09fc15a5e1318160db4e" },
|
||||
{ url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-win_amd64.whl", hash = "sha256:2be20b2c05a0cce10430cc25f32b689259640d273232b2de357c35729132256d" },
|
||||
{ url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-win_arm64.whl", hash = "sha256:99fc421a5d234580e45957a7b02effbf3e1c884a5dd077afc85352c77bf41434" },
|
||||
{ url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313-linux_s390x.whl", hash = "sha256:8b5882276633cf91fe3d2d7246c743b94d44a7e660b27f1308007fdb1bb89f7d" },
|
||||
{ url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:a5064b5e23772c8d164068cc7c12e01a75faf7b948ecd95a0d4007d7487e5f25" },
|
||||
{ url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:8f81dedb4c6076ec325acc3b47525f9c550e5284a18eae1d9061c543f7b6e7de" },
|
||||
{ url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313-win_amd64.whl", hash = "sha256:e1ee1b2346ade3ea90306dfbec7e8ff17bc220d344109d189ae09078333b0856" },
|
||||
{ url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313-win_arm64.whl", hash = "sha256:64c187345509f2b1bb334feed4666e2c781ca381874bde589182f81247e61f88" },
|
||||
{ url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:af81283ac671f434b1b25c95ba295f270e72db1fad48831eb5e4748ff9840041" },
|
||||
{ url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:a9dbb6f64f63258bc811e2c0c99640a81e5af93c531ad96e95c5ec777ea46dab" },
|
||||
{ url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313t-win_amd64.whl", hash = "sha256:6d93a7165419bc4b2b907e859ccab0dea5deeab261448ae9a5ec5431f14c0e64" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -5616,6 +5669,7 @@ dependencies = [
|
||||
{ name = "aiosqlite" },
|
||||
{ name = "argon2-cffi" },
|
||||
{ name = "asyncpg" },
|
||||
{ name = "beautifulsoup4" },
|
||||
{ name = "chardet" },
|
||||
{ name = "colorlog" },
|
||||
{ name = "dashscope" },
|
||||
@ -5642,11 +5696,14 @@ dependencies = [
|
||||
{ name = "llama-index" },
|
||||
{ name = "llama-index-readers-file" },
|
||||
{ name = "loguru" },
|
||||
{ name = "markdown-it-py" },
|
||||
{ name = "markdownify" },
|
||||
{ name = "mcp" },
|
||||
{ name = "mdit-py-plugins" },
|
||||
{ name = "minio" },
|
||||
{ name = "neo4j" },
|
||||
{ name = "networkx" },
|
||||
{ name = "nltk" },
|
||||
{ name = "openai" },
|
||||
{ name = "opencv-python-headless" },
|
||||
{ name = "pillow" },
|
||||
@ -5664,6 +5721,7 @@ dependencies = [
|
||||
{ name = "readability-lxml" },
|
||||
{ name = "redis" },
|
||||
{ name = "rich" },
|
||||
{ name = "scikit-learn" },
|
||||
{ name = "sqlalchemy", extra = ["asyncio"] },
|
||||
{ name = "tabulate" },
|
||||
{ name = "tavily-python" },
|
||||
@ -5689,6 +5747,7 @@ requires-dist = [
|
||||
{ name = "aiosqlite", specifier = ">=0.20.0" },
|
||||
{ name = "argon2-cffi", specifier = ">=25.1.0" },
|
||||
{ name = "asyncpg", specifier = ">=0.30.0" },
|
||||
{ name = "beautifulsoup4", specifier = ">=4.12.0" },
|
||||
{ name = "chardet", specifier = ">=5.0.0" },
|
||||
{ name = "colorlog", specifier = ">=6.9.0" },
|
||||
{ name = "dashscope", specifier = ">=1.23.2" },
|
||||
@ -5715,11 +5774,14 @@ requires-dist = [
|
||||
{ name = "llama-index", specifier = ">=0.14" },
|
||||
{ name = "llama-index-readers-file", specifier = ">=0.4.7" },
|
||||
{ name = "loguru", specifier = ">=0.7.3" },
|
||||
{ name = "markdown-it-py", specifier = ">=3.0.0" },
|
||||
{ name = "markdownify", specifier = ">=1.1.0" },
|
||||
{ name = "mcp", specifier = ">=1.20" },
|
||||
{ name = "mdit-py-plugins", specifier = ">=0.4.0" },
|
||||
{ name = "minio", specifier = ">=7.2.7" },
|
||||
{ name = "neo4j", specifier = ">=5.28.1" },
|
||||
{ name = "networkx", specifier = ">=3.5" },
|
||||
{ name = "nltk", specifier = ">=3.8.1" },
|
||||
{ name = "openai", specifier = ">=1.109" },
|
||||
{ name = "opencv-python-headless", specifier = ">=4.11.0.86" },
|
||||
{ name = "pillow", specifier = ">=10.5.0" },
|
||||
@ -5737,6 +5799,7 @@ requires-dist = [
|
||||
{ name = "readability-lxml", specifier = ">=0.8.1" },
|
||||
{ name = "redis", specifier = ">=5.2.0" },
|
||||
{ name = "rich", specifier = ">=13.7.1" },
|
||||
{ name = "scikit-learn", specifier = ">=1.3.0" },
|
||||
{ name = "sqlalchemy", extras = ["asyncio"], specifier = ">=2.0.0" },
|
||||
{ name = "tabulate", specifier = ">=0.9.0" },
|
||||
{ name = "tavily-python", specifier = ">=0.7.0" },
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
"name": "yuxi-web",
|
||||
"version": "0.6.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
"server": "vite serve --host",
|
||||
|
||||
@ -183,8 +183,13 @@
|
||||
size="16"
|
||||
class="agent-switcher-menu-icon"
|
||||
/>
|
||||
<span class="agent-switcher-menu-text">{{ agent.name || 'Unknown' }}</span>
|
||||
<span v-if="agent.id === currentAgentId" class="agent-switcher-menu-badge">
|
||||
<span class="agent-switcher-menu-text">{{
|
||||
agent.name || 'Unknown'
|
||||
}}</span>
|
||||
<span
|
||||
v-if="agent.id === currentAgentId"
|
||||
class="agent-switcher-menu-badge"
|
||||
>
|
||||
当前
|
||||
</span>
|
||||
</div>
|
||||
@ -605,7 +610,10 @@ const conversationRows = computed(() => {
|
||||
if (currentThreadConfigNotice.value) {
|
||||
const insertAfterCount = Math.max(
|
||||
0,
|
||||
Math.min(Number(currentThreadConfigNotice.value.insertAfterConversationCount) || 0, rows.length)
|
||||
Math.min(
|
||||
Number(currentThreadConfigNotice.value.insertAfterConversationCount) || 0,
|
||||
rows.length
|
||||
)
|
||||
)
|
||||
rows.splice(insertAfterCount, 0, {
|
||||
type: 'notice',
|
||||
@ -646,7 +654,10 @@ const showStartAgentSelector = computed(() => {
|
||||
})
|
||||
|
||||
const showStartAgentDropdown = computed(() => {
|
||||
return showStartAgentSelector.value && (startAgents.value.length >= 4 || localUIState.chatMainWidth < 380)
|
||||
return (
|
||||
showStartAgentSelector.value &&
|
||||
(startAgents.value.length >= 4 || localUIState.chatMainWidth < 380)
|
||||
)
|
||||
})
|
||||
|
||||
const showStartAgentSegment = computed(() => {
|
||||
@ -859,7 +870,11 @@ const queuePendingThreadConfigNotice = (threadId) => {
|
||||
}
|
||||
|
||||
const flushPendingThreadConfigNotice = (threadId) => {
|
||||
if (!threadId || !currentThreadHasHistory.value || !threadPendingConfigNoticeMap.value[threadId]) {
|
||||
if (
|
||||
!threadId ||
|
||||
!currentThreadHasHistory.value ||
|
||||
!threadPendingConfigNoticeMap.value[threadId]
|
||||
) {
|
||||
return
|
||||
}
|
||||
|
||||
@ -1419,7 +1434,11 @@ const selectChat = async (chatId) => {
|
||||
// 先更新当前线程，确保底部智能体名称与选中项即时同步。
|
||||
chatState.currentThreadId = chatId
|
||||
|
||||
if (!props.singleMode && targetChat?.agent_id && targetChat.agent_id !== currentAgentId.value) {
|
||||
if (
|
||||
!props.singleMode &&
|
||||
targetChat?.agent_id &&
|
||||
targetChat.agent_id !== currentAgentId.value
|
||||
) {
|
||||
await agentStore.selectAgent(targetChat.agent_id)
|
||||
}
|
||||
|
||||
@ -1912,10 +1931,10 @@ const hasVisibleAssistantBody = (message) => {
|
||||
const { content, reasoningContent } = extractAssistantMessageBody(message)
|
||||
return Boolean(
|
||||
content ||
|
||||
reasoningContent ||
|
||||
message.error_type ||
|
||||
message.extra_metadata?.error_type ||
|
||||
message.isStoppedByUser
|
||||
reasoningContent ||
|
||||
message.error_type ||
|
||||
message.extra_metadata?.error_type ||
|
||||
message.isStoppedByUser
|
||||
)
|
||||
}
|
||||
|
||||
@ -1997,9 +2016,7 @@ const isDisplayMessageProcessing = (conv, displayItem) => {
|
||||
|
||||
const isToolGroupActive = (conv, itemIndex, displayItems) => {
|
||||
return (
|
||||
isReplyLoading.value &&
|
||||
conv?.status === 'streaming' &&
|
||||
itemIndex === displayItems.length - 1
|
||||
isReplyLoading.value && conv?.status === 'streaming' && itemIndex === displayItems.length - 1
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@ -57,11 +57,7 @@
|
||||
/>
|
||||
<!-- 统一显示所有配置项 -->
|
||||
<template v-for="(value, key) in filteredConfigurableItems" :key="key">
|
||||
<a-form-item
|
||||
:label="getConfigLabel(key, value)"
|
||||
:name="key"
|
||||
class="config-item"
|
||||
>
|
||||
<a-form-item :label="getConfigLabel(key, value)" :name="key" class="config-item">
|
||||
<p v-if="value.description" class="config-description">{{ value.description }}</p>
|
||||
|
||||
<!-- <div>{{ value }}</div> -->
|
||||
|
||||
@ -77,7 +77,7 @@
|
||||
</div>
|
||||
|
||||
<!-- Result Slot -->
|
||||
<div class="tool-result" style="opacity: 0.8;" v-if="hasResult">
|
||||
<div class="tool-result" style="opacity: 0.8" v-if="hasResult">
|
||||
<slot name="result" :tool-call="toolCall" :result-content="resultContent">
|
||||
<div class="tool-result-content" :data-tool-call-id="toolCall.id">
|
||||
<!-- Default rendering -->
|
||||
@ -234,7 +234,7 @@ const formatResultData = (data) => {
|
||||
background-color: var(--gray-25);
|
||||
}
|
||||
|
||||
&>span {
|
||||
& > span {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
@ -13,8 +13,12 @@
|
||||
</span>
|
||||
<span class="summary-content">
|
||||
<span class="summary-title">{{ toolCallsSummaryTitle }}</span>
|
||||
<span class="summary-separator" v-if="normalizedToolCalls.length > 1 && toolCallsNamesMeta">·</span>
|
||||
<span class="summary-meta" v-if="normalizedToolCalls.length > 1 && toolCallsNamesMeta">{{ toolCallsNamesMeta }}</span>
|
||||
<span class="summary-separator" v-if="normalizedToolCalls.length > 1 && toolCallsNamesMeta"
|
||||
>·</span
|
||||
>
|
||||
<span class="summary-meta" v-if="normalizedToolCalls.length > 1 && toolCallsNamesMeta">{{
|
||||
toolCallsNamesMeta
|
||||
}}</span>
|
||||
<span class="summary-status-tag" v-if="statusSummary">{{ statusSummary }}</span>
|
||||
</span>
|
||||
<span class="summary-trailing">
|
||||
@ -123,8 +127,9 @@ const statusSummary = computed(() => {
|
||||
(toolCall) =>
|
||||
toolCall.status !== 'success' && toolCall.status !== 'error' && !toolCall.tool_call_result
|
||||
).length
|
||||
const errorCount = normalizedToolCalls.value.filter((toolCall) => toolCall.status === 'error')
|
||||
.length
|
||||
const errorCount = normalizedToolCalls.value.filter(
|
||||
(toolCall) => toolCall.status === 'error'
|
||||
).length
|
||||
|
||||
const parts = []
|
||||
if (successCount > 0 && successCount === normalizedToolCalls.value.length) {
|
||||
|
||||
@ -208,10 +208,7 @@
|
||||
|
||||
<!-- 部门选择器（仅超级管理员可见） -->
|
||||
<a-form-item v-if="userStore.isSuperAdmin" label="部门" class="form-item">
|
||||
<a-select
|
||||
v-model:value="userManagement.form.departmentId"
|
||||
placeholder="请选择部门"
|
||||
>
|
||||
<a-select v-model:value="userManagement.form.departmentId" placeholder="请选择部门">
|
||||
<a-select-option
|
||||
v-for="dept in departmentManagement.departments"
|
||||
:key="dept.id"
|
||||
|
||||
@ -167,13 +167,8 @@ const route = useRoute()
|
||||
const router = useRouter()
|
||||
|
||||
// 从 agentStore 中获取响应式状态
|
||||
const {
|
||||
selectedAgentId,
|
||||
defaultAgentId,
|
||||
selectedAgentConfigId,
|
||||
agentConfigs,
|
||||
isLoadingConfig
|
||||
} = storeToRefs(agentStore)
|
||||
const { selectedAgentId, defaultAgentId, selectedAgentConfigId, agentConfigs, isLoadingConfig } =
|
||||
storeToRefs(agentStore)
|
||||
|
||||
const syncingRouteThread = ref(false)
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user