diff --git a/backend/package/pyproject.toml b/backend/package/pyproject.toml index d422184f..28275d10 100644 --- a/backend/package/pyproject.toml +++ b/backend/package/pyproject.toml @@ -76,6 +76,12 @@ dependencies = [ "typer>=0.16.0", "unstructured>=0.17.2", "wcmatch>=8.0.0", + # Markdown 解析语义切分相关依赖 + "markdown-it-py>=3.0.0", + "mdit-py-plugins>=0.4.0", + "scikit-learn>=1.3.0", + "nltk>=3.8.1", + "beautifulsoup4>=4.12.0", ] [tool.setuptools] diff --git a/backend/package/yuxi/knowledge/chunking/ragflow_like/dispatcher.py b/backend/package/yuxi/knowledge/chunking/ragflow_like/dispatcher.py index 598b96f6..a9d5971a 100644 --- a/backend/package/yuxi/knowledge/chunking/ragflow_like/dispatcher.py +++ b/backend/package/yuxi/knowledge/chunking/ragflow_like/dispatcher.py @@ -2,7 +2,7 @@ from __future__ import annotations from typing import Any -from yuxi.knowledge.chunking.ragflow_like.parsers import book, general, laws, qa +from yuxi.knowledge.chunking.ragflow_like.parsers import book, general, laws, qa, semantic from yuxi.knowledge.chunking.ragflow_like.presets import map_to_internal_parser_id, normalize_chunk_preset_id @@ -42,6 +42,8 @@ def _dispatch_markdown_parser( return book.chunk_markdown(markdown_content, parser_config) if parser_id == "laws": return laws.chunk_markdown(filename, markdown_content, parser_config) + if parser_id == "semantic": + return semantic.chunk_markdown(markdown_content, parser_config) return general.chunk_markdown(markdown_content, parser_config) diff --git a/backend/package/yuxi/knowledge/chunking/ragflow_like/parsers/semantic.py b/backend/package/yuxi/knowledge/chunking/ragflow_like/parsers/semantic.py new file mode 100644 index 00000000..a061e41e --- /dev/null +++ b/backend/package/yuxi/knowledge/chunking/ragflow_like/parsers/semantic.py @@ -0,0 +1,298 @@ +from __future__ import annotations + +import re +from typing import Any + +from markdown_it import MarkdownIt +from mdit_py_plugins.dollarmath import dollarmath_plugin + +from yuxi.knowledge.chunking.ragflow_like.nlp import count_tokens +from yuxi.utils.logging_config import logger + +from ..utils.md_parser_utils import ( + extract_table_block, + get_title_path, + split_text_by_length_and_newline, +) +from ..utils.table_utils import html_table_to_key_value + + +def _flush_content( + result: list, + current_content: list, + title_stack: list, + max_length: int, + embed_fn: Any, + special_element: str = None, + allow_split: bool = False, +) -> None: + if not current_content: + return + + content = "\n".join(current_content).strip() + if not content: + current_content.clear() + return + + level = next((i + 1 for i in range(5, -1, -1) if title_stack[i]), 1) + title_path = get_title_path(title_stack) + + if special_element and not allow_split: + header = f"{'#' * level} {title_path}|{special_element}" if title_path else f"{'#' * level} {special_element}" + result.extend([header, content, "-" * 10]) + else: + if count_tokens(content) > max_length: + chunks = split_text_by_length_and_newline( + content, max_length, embed_fn=embed_fn, token_count_fn=count_tokens + ) + for idx, chunk in enumerate(chunks, 1): + base_header = f"{'#' * level} {title_path}" if title_path else f"{'#' * level}" + if special_element: + header = f"{base_header}|{special_element}|Part {idx}" + else: + header = f"{base_header}|Part {idx}" + result.extend([header, chunk, "-" * 10]) + else: + base_header = f"{'#' * level} {title_path}" if title_path else f"{'#' * level}" + if special_element: + header = f"{base_header}|{special_element}" + else: + header = base_header + + if header: + result.append(header) + result.append("") + result.extend([content, "-" * 10]) + + current_content.clear() + + +def _handle_image_caption(tokens, i, result, current_content, title_stack, max_length, embed_fn): + token = tokens[i] + if token.type != "paragraph_open": + return False, i + + inline_token = tokens[i + 1] + if inline_token.type != "inline": + return False, i + + content = inline_token.content.strip() + image_pattern = r"^!\[.*?\]\(.*?\)\s*$" + caption_pattern = r"^(?:Figure|图|Fig\.|表|Table)\s*[\d\w\.]+" + + img_match = re.search(r"^(!\[.*?\]\(.*?\))", content) + if img_match: + rest = content[img_match.end() :].strip() + if rest and re.match(caption_pattern, rest, re.IGNORECASE): + _flush_content(result, current_content, title_stack, max_length, embed_fn) + current_content.append(content) + caption_title = rest.split("\n")[0].strip() + _flush_content(result, current_content, title_stack, max_length, embed_fn, special_element=caption_title) + return True, i + 3 + + if re.match(image_pattern, content): + next_p_idx = i + 3 + if next_p_idx + 1 < len(tokens) and tokens[next_p_idx].type == "paragraph_open": + next_inline = tokens[next_p_idx + 1] + if next_inline.type == "inline": + next_content = next_inline.content.strip() + if re.match(caption_pattern, next_content, re.IGNORECASE): + _flush_content(result, current_content, title_stack, max_length, embed_fn) + current_content.append(content) + current_content.append(next_content) + _flush_content( + result, current_content, title_stack, max_length, embed_fn, special_element=next_content + ) + return True, i + 6 + + if current_content and re.match(caption_pattern, content, re.IGNORECASE): + last_item = current_content[-1].strip() + if re.match(image_pattern, last_item): + image_tag = current_content.pop() + _flush_content(result, current_content, title_stack, max_length, embed_fn) + current_content.append(image_tag) + current_content.append(content) + _flush_content(result, current_content, title_stack, max_length, embed_fn, special_element=content) + return True, i + 3 + + return False, i + + +def chunk_markdown( + markdown_content: str, parser_config: dict[str, Any] | None = None, embed_fn: Any | None = None +) -> list[str]: + """ + 语义化切分 Markdown 内容。 + + Args: + markdown_content: 待切分的 Markdown 文本 + parser_config: 切分参数，如 chunk_token_num + embed_fn: 可选。传入用于生成向量的函数。如果不传，将从系统配置中加载模型。 + 通过注入此参数可以避免在单元测试中加载重型资源。 + """ + parser_config = parser_config or {} + max_length = int(parser_config.get("chunk_token_num", 512)) + logger.info(f"语义切分开始: max_length={max_length}, content_length={len(markdown_content)}") + + # 延迟加载重型资源，仅在没有注入 embed_fn 时触发 + if embed_fn is None: + try: + from yuxi.config.app import config + from yuxi.models.embed import select_embedding_model + + embed_model_id = parser_config.get("embed_model_id") or config.embed_model + logger.info(f"语义切分加载Embedding模型: {embed_model_id}") + embed_model = select_embedding_model(embed_model_id) + embed_fn = embed_model.encode + except Exception as e: + logger.error(f"加载 Embedding 模型失败: {e}。将退化为简单切分。") + embed_fn = None + + md = MarkdownIt("commonmark").enable("table") + md.use(dollarmath_plugin, allow_space=True, allow_digits=True) + + tokens: list = md.parse(markdown_content) + original_lines: list = markdown_content.split("\n") + + result: list = [] + current_content: list = [] + title_stack: list = [""] * 6 + + i = 0 + while i < len(tokens): + token = tokens[i] + if token.type == "heading_open": + _flush_content(result, current_content, title_stack, max_length, embed_fn) + level = int(token.tag[1:]) if token.tag and len(token.tag) > 1 else 1 + inline_token = tokens[i + 1] + if inline_token.type == "inline": + full_title = inline_token.content.strip() + title_stack[level - 1] = full_title + for j in range(level, 6): + title_stack[j] = "" + i += 3 + continue + elif token.type == "table_open": + _flush_content(result, current_content, title_stack, max_length, embed_fn) + j, table_content = extract_table_block(tokens, i, original_lines) + current_content.append(table_content) + _flush_content(result, current_content, title_stack, max_length, embed_fn, special_element="Table") + i = j + 1 if j < len(tokens) else len(tokens) + continue + elif token.type == "paragraph_open": + handled, new_i = _handle_image_caption( + tokens, i, result, current_content, title_stack, max_length, embed_fn + ) + if handled: + i = new_i + continue + inline_token = tokens[i + 1] + if inline_token.type == "inline": + current_content.append(inline_token.content.strip()) + i += 3 + continue + elif token.type == "fence": + current_content.append(f"```\n{token.content}\n```") + i += 1 + continue + elif token.type == "ordered_list_open": + # _flush_content(result, current_content, title_stack, max_length, embed_fn) + list_content = [] + j = i + 1 + list_item_counter = 1 + while j < len(tokens) and tokens[j].type != "ordered_list_close": + if tokens[j].type == "list_item_open": + k = j + 1 + while k < len(tokens) and tokens[k].type != "list_item_close": + if ( + tokens[k].type == "paragraph_open" + and k + 1 < len(tokens) + and tokens[k + 1].type == "inline" + ): + list_content.append(f"{list_item_counter}. {tokens[k + 1].content.strip()}") + list_item_counter += 1 + k += 1 + j += 1 + if list_content: + current_content.extend(list_content) + _flush_content(result, current_content, title_stack, max_length, embed_fn, special_element=token.type) + i = j + 1 + continue + elif token.type == "bullet_list_open": + # _flush_content(result, current_content, title_stack, max_length, embed_fn) + list_content = [] + j = i + 1 + while j < len(tokens) and tokens[j].type != "bullet_list_close": + if tokens[j].type == "list_item_open": + k = j + 1 + while k < len(tokens) and tokens[k].type != "list_item_close": + if ( + tokens[k].type == "paragraph_open" + and k + 1 < len(tokens) + and tokens[k + 1].type == "inline" + ): + list_content.append(f"- {tokens[k + 1].content.strip()}") + k += 1 + j += 1 + if list_content: + current_content.extend(list_content) + _flush_content(result, current_content, title_stack, max_length, embed_fn, special_element=token.type) + i = j + 1 + continue + elif token.type == "html_block": + _flush_content(result, current_content, title_stack, max_length, embed_fn) + content = token.content.strip() + is_converted_table = False + if " list[dict[str, str]]: "label": "Laws", "description": CHUNK_PRESET_DESCRIPTIONS[CHUNK_PRESET_LAWS], }, + { + "value": CHUNK_PRESET_SEMANTIC, + "label": "Semantic", + "description": CHUNK_PRESET_DESCRIPTIONS[CHUNK_PRESET_SEMANTIC], + }, ] diff --git a/backend/package/yuxi/knowledge/chunking/ragflow_like/utils/md_parser_utils.py b/backend/package/yuxi/knowledge/chunking/ragflow_like/utils/md_parser_utils.py new file mode 100644 index 00000000..2b4bd195 --- /dev/null +++ b/backend/package/yuxi/knowledge/chunking/ragflow_like/utils/md_parser_utils.py @@ -0,0 +1,150 @@ +from __future__ import annotations + +import re +from collections.abc import Callable +from typing import Any + +from .semantic_utils import semantic_chunking_with_auto_clusters + + +def infer_heading_level(title: str) -> int: + """ + 根据标题文本推断其层级级别（1-6级）。 + + 逻辑说明： + 1. 数字序号推断： + - 匹配如 "1.", "1.1", "1.2.3" 等格式。 + - 根据点号分隔的数量确定层级，例如 "1.1" 为 2 级，"1.2.3" 为 3 级。 + - 层级限制在 1-6 之间。 + 2. 中文序号推断： + - 匹配如 "一、", "二." 等中文数字序号。 + - 统一归类为 1 级标题。 + 3. 默认处理： + - 若不匹配以上规则，默认返回 1 级。 + """ + m = re.match(r"^\s*(\d+(?:\.\d+)*)[.)、]?\s*", title) + if m: + return max(1, min(len(m.group(1).split(".")), 6)) + m_zh = re.match(r"^\s*[一二三四五六七八九十百千]+[、.]\s*", title) + if m_zh: + return 1 + return 1 + + +def get_title_path(stack: list[str]) -> str: + """ + 根据标题栈生成标题路径，用"|"分隔。 + """ + return "|".join([t for t in stack if t]) + + +def extract_table_block(tokens: list[Any], i: int, original_lines: list[str]) -> tuple[int, str]: + """ + 从token流和原始文本中提取完整的表格块。 + + 逻辑说明： + 1. 定位起始：通过当前 token (i) 的 `map` 属性获取表格在原始行中的起始行号 `table_start`。 + 2. 查找结束 token：遍历后续 tokens 直到找到 `table_close`。 + 3. 确定结束行号 (`table_end`)： + - 优先使用 `table_close` token 的 `map` 属性。 + - 若不存在，则尝试查找下一个带有 `map` 信息的 token 的起始行作为当前表格的结束。 + - 若上述均失败（如文件末尾或解析异常），则回退到基于文本内容的启发式扫描： + 从 `table_start` 开始向下扫描，直到遇到不符合 Markdown 表格特征（不以 '|' 开头且不含 '|'）的行为止。 + 4. 返回结果：返回 `table_close` 的索引 `j` 以及拼接后的表格原始字符串。 + """ + token = tokens[i] + table_start = token.map[0] if token.map else 0 + j = i + 1 + while j < len(tokens) and tokens[j].type != "table_close": + j += 1 + if j < len(tokens): + end_token = tokens[j] + if end_token.map and end_token.map[1] is not None: + table_end = end_token.map[1] + else: + table_end = None + for k in range(j + 1, len(tokens)): + if tokens[k].map and tokens[k].map[0] is not None: + table_end = tokens[k].map[0] + break + if table_end is None: + table_end = table_start + 1 + for line_idx in range(table_start, len(original_lines)): + line = original_lines[line_idx].strip() + if not line or not (line.startswith("|") or "|" in line): + table_end = line_idx + break + else: + table_end = table_start + 1 + for line_idx in range(table_start, len(original_lines)): + line = original_lines[line_idx].strip() + if not line or not (line.startswith("|") or "|" in line): + table_end = line_idx + break + return j, "\n".join(original_lines[table_start:table_end]) + + +def split_text_by_length_and_newline( + text: str, max_length: int, embed_fn: Callable[[list[str]], Any] | None, token_count_fn: Callable[[str], int] +) -> list[str]: + """ + 层次化文本切分策略。 + """ + chunks = [] + + paragraphs = text.split("\n\n") + + for paragraph in paragraphs: + paragraph = paragraph.strip() + if not paragraph: + continue + + paragraph_token_count = token_count_fn(paragraph) + + # 如果当前段落长度未超过最大 Token 数量，直接作为独立分块放入chunks + # 否则继续尝试按行切分 + if paragraph_token_count <= max_length: + chunks.append(paragraph) + continue + + # 把段落进一步使用换行符进行切分为行 + lines = paragraph.split("\n") + current_chunk_lines = [] + current_chunk_tokens = 0 + + for line in lines: + line = line.strip() + if not line: # 跳过空行 + continue + + line_token_count = token_count_fn(line) # 计算当前行的 Token 数量 + # 为了考虑行之间的空格，需要在计算 Token 数量时加 1（如果当前行不是第一行，需要添加一个换行符的Token数量） + added_tokens = line_token_count + (1 if current_chunk_lines else 0) + # 如果当前行的 Token 数量超过最大 Token 数量，直接作为独立分块放入chunks + if line_token_count > max_length: + if current_chunk_lines: + chunks.append("\n".join(current_chunk_lines)) + current_chunk_lines = [] + current_chunk_tokens = 0 + + sub_chunks = semantic_chunking_with_auto_clusters( + line, embed_fn=embed_fn, token_count_fn=token_count_fn, max_chunk_size=max_length + ) + chunks.extend(sub_chunks) + # 如果当前行的 Token 数量与当前分块的 Token 数量合并后超过最大 Token 数量，直接作为独立分块放入chunks + elif current_chunk_tokens + added_tokens > max_length: + # 把之前的分块内容放入chunks + chunks.append("\n".join(current_chunk_lines)) + # 重置当前分块为当前行的内容 + current_chunk_lines = [line] + # 更新当前分块的 Token 数量 + current_chunk_tokens = line_token_count + # 如果当前行的内容加入当前分块后不会超过最大 Token 数量，直接加入当前分块 + else: + current_chunk_lines.append(line) + current_chunk_tokens += added_tokens # 更新当前分块的 Token 数量 + # 最后的收尾，把最后一行内容放入chunks + if current_chunk_lines: + chunks.append("\n".join(current_chunk_lines)) + + return chunks diff --git a/backend/package/yuxi/knowledge/chunking/ragflow_like/utils/semantic_utils.py b/backend/package/yuxi/knowledge/chunking/ragflow_like/utils/semantic_utils.py new file mode 100644 index 00000000..69edd9a4 --- /dev/null +++ b/backend/package/yuxi/knowledge/chunking/ragflow_like/utils/semantic_utils.py @@ -0,0 +1,192 @@ +from __future__ import annotations + +import re +from collections.abc import Callable +from typing import Any + +import nltk +from nltk.tokenize import sent_tokenize +from sklearn.cluster import AgglomerativeClustering +from sklearn.metrics import silhouette_score + +_punkt_checked = False + + +def _ensure_punkt_tab() -> None: + """首次使用分句能力时检查 NLTK punkt_tab 资源，缺失时给出可操作错误。""" + global _punkt_checked + if _punkt_checked: + return + + try: + nltk.data.find("tokenizers/punkt_tab") + except LookupError as e: + raise RuntimeError( + "缺少 NLTK 资源 punkt_tab。请先执行: python -m nltk.downloader punkt_tab" + ) from e + + _punkt_checked = True + + +def split_sentences_chinese(text: str) -> list[str]: + """ + 使用正则表达式将中文文本分割成句子。 + + 逻辑： + - 匹配中文句号、感叹号、问号（。！？）作为分隔点。 + - 使用正向/反向预查处理引号：确保如果标点后面紧跟引号（”’"），该引号会被保留在当前句子末尾，而不是被切分到下一句。 + - 返回去除两端空格且非空的句子列表。 + """ + pattern = r'(?<=[。！？][”’"])|(?<=[。！？])(?![”’"])' + sentences = re.split(pattern, text) + return [s.strip() for s in sentences if s.strip()] + + +def split_mixed_sentences(text: str) -> list[str]: + """ + 处理中英文混合文本的分句逻辑，支持按物理段落分发不同的分句策略。 + + 该函数采用“分而治之”的策略来处理复杂的混合文本： + 1. **物理分块**：首先按换行符 (`\\n+`) 将原始文本切分为多个物理段落（chunks），确保物理结构不被破坏。 + 2. **语言检测与分发**： + - **英文/混合路径**：若段落中包含英文字母 (`[A-Za-z]`)，则视为英文或混合文本， + 调用 NLTK 的 `sent_tokenize` 进行处理。NLTK 能更好地处理英文缩写、句点等复杂情况。 + - **中文路径**：若段落不含字母，则视为纯中文文本，调用 `split_sentences_chinese`。 + 该方法通过正则精准匹配中文标点及后续引号。 + - **兜底方案**：若上述方法未产生结果，则使用简单的正则表达式按中文标点强制分割。 + 3. **清洗与过滤**：汇总所有子句，去除两端空白字符，并过滤掉空字符串。 + + Args: + text: 待分句的原始字符串。 + + Returns: + List[str]: 分割后的句子列表。 + """ + _ensure_punkt_tab() + + chunks = re.split(r"(\n+)", text) + sentences = [] + + for ch in chunks: + if not ch.strip(): + continue + if re.search(r"[A-Za-z]", ch): + parts = sent_tokenize(ch) + sentences.extend([p.strip() for p in parts if p.strip()]) + else: + sents = split_sentences_chinese(ch) + if sents: + sentences.extend([s.strip() for s in sents if s.strip()]) + else: + parts = re.split(r"(?<=[。！？])", ch) + sentences.extend([p.strip() for p in parts if p.strip()]) + return sentences + + +def find_best_num_clusters(embeddings: Any, min_clusters: int = 2, max_clusters: int = 10) -> int: + """ + 使用轮廓系数选择最佳聚类数量。让每个分段语义集中，且分段之间界限分明 + + 逻辑： + - 遍历可能的聚类数量（从 min_clusters 到 max_clusters）。 + - 对每个聚类数量，使用 `AgglomerativeClustering` 进行聚类。 + - 计算轮廓系数（Silhouette Score）。 + - 选择轮廓系数最高的聚类数量作为最佳聚类数量。 + - 如果聚类数量为 1 或更少，直接返回 1。 + + Args: + embeddings: 待聚类的向量数据（所有句子的嵌入向量列表）。 + min_clusters: 搜索的最佳聚类数量下限，默认为 2。 + max_clusters: 搜索的最佳聚类数量上限，默认为 10。 + + Returns: + int: 轮廓系数表现最好的聚类数量。 + """ + if len(embeddings) <= min_clusters: + return len(embeddings) + + best_score = -1 + best_k = min_clusters + + limit_k = min(max_clusters, len(embeddings)) + for k in range(min_clusters, limit_k + 1): + labels = AgglomerativeClustering(n_clusters=k, metric="cosine", linkage="average").fit_predict(embeddings) + if len(set(labels)) <= 1: + continue + score = silhouette_score(embeddings, labels, metric="cosine") + if score > best_score: + best_score = score + best_k = k + + return best_k + + +def semantic_chunking_with_auto_clusters( + text: str, embed_fn: Callable[[list[str]], Any] | None, token_count_fn: Callable[[str], int], max_chunk_size: int = 512 +) -> list[str]: + """ + 对传入的文本进行语义切分，过程中会自动选择最佳的聚集数量。 + + 逻辑： + - 先将文本中的句子按语言进行分发，英文/混合文本使用NLTK的sent_tokenize，中文文本使用split_sentences_chinese。 + - 对每个句子进行嵌入向量化。 + - 确定最佳的聚类数量（根据轮廓系数）。 + - 对句子进行聚类后，按原文顺序遍历：当聚类标签变化或达到长度上限时切分，形成连续分块。 + - 如果嵌入模型缺失，则退化为原始切分方式 + """ + sentences = split_mixed_sentences(text) + if len(sentences) < 2: + return [text.strip()] + + # 计算每个句子的token数量 + sentence_token_counts = [token_count_fn(s) for s in sentences] + total_tokens = sum(sentence_token_counts) + + # 如果没有提供向量化函数，或者整体未超长，则直接进行简单合并/返回 + if embed_fn is None or total_tokens <= max_chunk_size: + chunks = [] + current_chunk = "" + current_chunk_tokens = 0 + for s, cnt in zip(sentences, sentence_token_counts): + if current_chunk_tokens + cnt > max_chunk_size and current_chunk: + chunks.append(current_chunk.strip()) + current_chunk = s + current_chunk_tokens = cnt + else: + current_chunk += s + current_chunk_tokens += cnt + if current_chunk: + chunks.append(current_chunk.strip()) + return chunks + + # 向量化每个句子, 得到他们的嵌入向量 + embeddings = embed_fn(sentences) + + # 决定合适的聚集数量：超长时按上限向上取整，避免整除时多切一块 + best_k = (total_tokens + max_chunk_size - 1) // max_chunk_size + best_k = min(best_k, len(sentences)) + + # 根据指定的聚集数量、相似度判断方式、联动方式，对句子进行聚类 + # labels 是每个句子的聚类标签列表（如 [0,0,1,2,2]），后续会按原文顺序在标签变化处切分连续分块 + labels = AgglomerativeClustering(n_clusters=best_k, metric="cosine", linkage="average").fit_predict(embeddings) + + chunks = [] + current_chunk = "" + current_chunk_tokens = 0 + current_label = labels[0] + + for sentence, label, token_count in zip(sentences, labels, sentence_token_counts): + if label != current_label or current_chunk_tokens + token_count > max_chunk_size: + if current_chunk.strip(): + chunks.append(current_chunk.strip()) + current_chunk = sentence + current_chunk_tokens = token_count + current_label = label + else: + current_chunk += sentence + current_chunk_tokens += token_count + + if current_chunk.strip(): + chunks.append(current_chunk.strip()) + + return chunks diff --git a/backend/package/yuxi/knowledge/chunking/ragflow_like/utils/table_utils.py b/backend/package/yuxi/knowledge/chunking/ragflow_like/utils/table_utils.py new file mode 100644 index 00000000..72b6fd9c --- /dev/null +++ b/backend/package/yuxi/knowledge/chunking/ragflow_like/utils/table_utils.py @@ -0,0 +1,147 @@ +from __future__ import annotations + +from bs4 import BeautifulSoup + + +def html_table_to_markdown(html: str) -> str: + """ + 将HTML表格转换为Markdown格式的具体实现 + """ + soup = BeautifulSoup(html, "html.parser") + table = soup.find("table") + if table is None: + return "" + + rows = table.find_all("tr") + if not rows: + return "" + + grid = [] + + for r_idx, row in enumerate(rows): + while len(grid) <= r_idx: + grid.append([]) + + cells = row.find_all(["td", "th"]) + c_idx = 0 + + for cell in cells: + while c_idx < len(grid[r_idx]) and grid[r_idx][c_idx] is not None: + c_idx += 1 + + text = cell.get_text(strip=True) + text = text.replace("\n", " ") + + rowspan = int(cell.get("rowspan", 1)) + colspan = int(cell.get("colspan", 1)) + + for r in range(rowspan): + target_r = r_idx + r + while len(grid) <= target_r: + grid.append([]) + + for c in range(colspan): + target_c = c_idx + c + while len(grid[target_r]) <= target_c: + grid[target_r].append(None) + + grid[target_r][target_c] = text + + c_idx += colspan + + if not grid: + return "" + + markdown_lines = [] + max_cols = max(len(r) for r in grid) + for r in grid: + while len(r) < max_cols: + r.append("") + + header = grid[0] + header = [h if h is not None else "" for h in header] + markdown_lines.append("| " + " | ".join(header) + " |") + markdown_lines.append("|" + "|".join([" --- " for _ in range(max_cols)]) + "|") + + for row in grid[1:]: + row_clean = [cell if cell is not None else "" for cell in row] + line = "| " + " | ".join(row_clean) + " |" + markdown_lines.append(line) + + return "\n".join(markdown_lines) + + +def html_table_to_key_value(html: str) -> list[str]: + """ + 将HTML表格转换为键值对格式的列表，为了应对过长的表格的切分问题。 + + 处理逻辑： + 1. **网格重建**：由于 HTML 表格可能包含 `rowspan` 和 `colspan`（合并单元格）， + 函数首先构建一个完整的二维网格（grid）。 + 2. **单元格展开**：遍历 HTML 行和列，遇到合并单元格时，将其内容填充到网格中受影响的所有坐标点。 + 这确保了原本被合并的区域在逻辑网格中每个点都有对应的值。 + 3. **键值对转换**： + - 将网格的第一行视为表头（Key）。 + - 从第二行开始，将每一行与表头对应，生成 "键：值" 形式的字符串。 + + 例如： + - 输入：HTML表格，包含姓名、年龄、性别三列。 + - 输出：['姓名：张三；年龄：25；性别：男', '姓名：李四；年龄：30；性别：女'] + """ + soup = BeautifulSoup(html, "html.parser") + table = soup.find("table") + if table is None: + return [] + + rows = table.find_all("tr") + if not rows: + return [] + + grid = [] + + for r_idx, row in enumerate(rows): + while len(grid) <= r_idx: + grid.append([]) + + cells = row.find_all(["td", "th"]) + c_idx = 0 + + for cell in cells: + while c_idx < len(grid[r_idx]) and grid[r_idx][c_idx] is not None: + c_idx += 1 + + text = cell.get_text(strip=True) + rowspan = int(cell.get("rowspan", 1)) + colspan = int(cell.get("colspan", 1)) + + for r in range(rowspan): + target_r = r_idx + r + while len(grid) <= target_r: + grid.append([]) + + for c in range(colspan): + target_c = c_idx + c + while len(grid[target_r]) <= target_c: + grid[target_r].append(None) + grid[target_r][target_c] = text + c_idx += colspan + + if not grid: + return [] + + headers = grid[0] + headers = [h if h is not None else "" for h in headers] + + kv_lines = [] + for row_values in grid[1:]: + min_len = min(len(headers), len(row_values)) + row_parts = [] + for i in range(min_len): + key = headers[i] + val = row_values[i] if row_values[i] is not None else "" + if key: + row_parts.append(f"{key}：{val}") + if row_parts: + kv_lines.append("；".join(row_parts) + "；") + + return kv_lines diff --git a/backend/test/unit/backends/test_semantic_chunking.py b/backend/test/unit/backends/test_semantic_chunking.py new file mode 100644 index 00000000..ea0be0a2 --- /dev/null +++ b/backend/test/unit/backends/test_semantic_chunking.py @@ -0,0 +1,319 @@ +import pytest +import numpy as np + +# 现在可以安全地导入了，因为顶层不再有重型依赖 +from yuxi.knowledge.chunking.ragflow_like.parsers import semantic +from yuxi.models import select_embedding_model + +@pytest.fixture +def embed_fn(): + """使用真实的嵌入函数 (从 SiliconFlow 获取)""" + model_id = "siliconflow/Qwen/Qwen3-Embedding-0.6B" + try: + model = select_embedding_model(model_id) + def encode(sentences): + if isinstance(sentences, str): + sentences = [sentences] + return model.encode(sentences) + return encode + except Exception as e: + pytest.skip(f"无法初始化真实嵌入模型 {model_id}: {e}") + +@pytest.fixture +def sample_markdown(): + """提供一个包含多章节、公式符号和复杂结构的临时 Markdown 样本""" + return """ +## 痛点与挑战 + +在传统的RAG知识库前置处理过程中，我们面临着诸多挑战： + +- **格式繁杂兼容难**：企业文档格式多样，传统工具（例如python-docx和pymupdf）难以同时高质量处理 Office 文档和复杂的 PDF 扫描件。 +- **语义割裂检索差**：简单的按字符切分导致上下文丢失，检索匹配度低，无法满足业务需求。 +- **解析效率瓶颈**：面对海量存量文档，缺乏高并发处理能力，响应速度慢，难以支撑大规模应用。 + +## 技术方案概览 + +为了解决上述问题，我们构建了一套高性能、可扩展的文档处理架构： + +- **全格式支持**：全面覆盖 **Word、Excel、PDF、PPT和图片**等主流办公文档格式。 +- **高精度解析**：通过引入MinerU VLM模型实现准确的布局解析 and 元素提取。 +- **跨平台支持**：同时支持**英伟达CUDA和华为CANN框架。** +- **知识库应用优化**：为了更好支持上层应用，解析后增加了智能切分和实体识别功能，增强后续的检索效果。 +- **高并发架构设计**： + - 采用 **FastAPI** 作为服务入口，保障请求的高效接收与响应。 + - 引入 **Celery** 分布式任务队列，实现请求响应与处理过程的完全解耦，有效应对流量洪峰，保障系统稳定性和可用性。 + - **资源利用最大化**：通过创建多组 **Celery Worker**（执行CPU密集任务） + **vLLM Server**（执行GPU密集推理任务）的Pod单元，实现对多核CPU+多卡GPU的并行调用，大幅提升单机处理吞吐量。 + - 使用Docker Compose进行服务编排，不同功能划分为不同的进程，保证服务的并发量和性能。 + +![image.png](attachment:393b3113-298b-4063-b572-db9694dddb4b:image.png) + +## 架构设计 + +| 服务名称 | 容器名称 | 职责描述 | +| --- | --- | --- | +| **mineru-api** | `mineru-api` | **API 网关：**基于 FastAPI，提供对外 HTTP 接口 。负责接收请求、鉴权及任务分发。 | +| **mineru-worker** | `mineru-worker` | **异步任务处理器：**基于 Celery，负责 PDF 解析、OCR、文档转换等耗时任务。 | +| **vllm** | `mineru-vllm` | **推理后端：**运行 MinerU 视觉大模型 (VLM)，提供高并发的文档理解与提取能力。 | +| **redis** | `mineru-redis` | **消息中间件：**作为 Celery 的 Broker 和 Backend，同时缓存部分运行时数据。 | + +## 核心技术亮点 + +### 1. 多模态融合的高精度解析 + +针对不同类型的文档，我们采用了差异化的解析策略，以确保最佳效果： +* **Office 文档**：利用 **Docling** ，快速、精准地从文件中提取文本与格式信息。 +* **复杂 PDF 文档**：引入 **MinerU 视觉大模型（VLM）**，该模型具备类似人类的视觉能力，能够深度理解文档排版，完美还原文档结构和准确提取页面上的公式和表格元素，并统一输出为标准的 Markdown 格式。 + +![原始文件](attachment:45b43827-6c38-49ed-9996-e08c037d6858:公式1_1原始文件.png) + +原始文件 + +![解析后效果](attachment:90e6a6da-0ac6-4b43-bfcb-f0921960c5f1:公式1_3解析后预览.png) + +解析后效果 + +### 2. 智能表格处理 + +针对文档中的表格数据，我们提供了两种灵活的转换方式，满足不同应用场景的需求： +* **键值对转换**：将表格行列关系转换为自然语义表达，便于大模型理解和问答。并且**支持合并单元格的处理**，避免生成关系错乱的表格数据。例如，将”产品名称 | 价格 | 库存”的表格转换为”产品名称：A；价格：100元，库存：50件；产品名称：B；价格：150元，库存：30件”等键值对形式。 +* **Markdown表格转换**：保持表格的原始结构，转换为标准的Markdown表格格式，便于上层应用中可视化展示。 + +![原始表格](attachment:b02cd061-728f-4fd8-ac12-5298dd427cc3:表格1_1原始文件.png) + +原始表格 + +![markdown格式表格](attachment:efaa0df1-96d6-4c9f-bbb4-80a561d10009:表格1_3解析后预览.png) + +markdown格式表格 + +![key-value格式表格](attachment:a476994c-6d6b-4a74-9a55-b6907ce40bde:表格1_4切分后.png) + +key-value格式表格 + +此外，**针对过长的表格，支持按照长度切分，在不丢失信息的前提下**适配向量数据库的存储，保证后续的检索效果。 + +![原始表格](attachment:71523288-f21f-413e-a69b-e9fda14532e9:表格2_3解析后预览.png) + +原始表格 + +![表格切分后](attachment:ffa664bb-5eb8-4820-a4fb-9968266e8fd2:表格2_4切分后.png) + +表格切分后 + +### 3. 基于语义的智能切分 + +摒弃了传统机械的字数切分方式，我们引入了 **BGE Embedding 模型**。通过计算文本向量并利用相似度聚类算法，将含义紧密相关的段落自动聚合。这种方式确保了每一个数据切片都是一个完整的”语义单元”，大幅提升了后续检索的准确性。与此同时，模型规模为24M，资源占用率极小，对整体解析流程的影响微乎其微。 + +![语义切分前1](attachment:7e4e4c3c-126b-4c5b-8dc7-e4da93252a54:语义分析1_2解析后.png) + +语义切分前1 + +![语义切分后2](attachment:3febe837-7722-4c59-8e40-25e9bc8dbbaa:语义分析1_4切分后.png) + +语义切分后2 + +![语义切分前2](attachment:e17278be-b749-4618-8b51-793f83892caf:语义分析2_2解析后.png) + +语义切分前2 + +![语义切分后2](attachment:18f40c6d-6c08-40e8-b369-927b33f09816:语义分析2_3切分后.png) + +语义切分后2 + +### 4. 上下文感知的标题聚合 + +针对企业长文档层级复杂的特点，我们开发了**标题聚合功能**。系统会自动将多级标题信息聚合到对应的段落中。无论文档如何切分，每个切片都能保留”父级标题-子标题”的完整路径，有效解决了碎片化导致的上下文丢失问题，从而提升了后续检索的准确性和相关性。 + +![多级标题聚合](attachment:7dd756ee-b097-4b2c-b7ca-638915fd7b23:语义分析2_3切分后.png) + +多级标题聚合 + +### 5. 关键信息自动提取 + +为了进一步提升检索效率，我们集成了中英文的 **NER（命名实体识别）模型**。在解析过程中，系统会自动抽取文档中的组织机构、人名、专有名词等关键实体，为文档打上”智能标签”，实现多维度的精准检索。 + +![image.png](attachment:62685770-d067-4df5-9415-a559b0e20658:image.png) + +![image.png](attachment:4d3847f8-317d-4393-8e7c-bddb5f3c50f7:image.png) + +### 6. 内容检索 + +![原始PDF页面](attachment:4ad6a991-7d23-494d-a653-95ec8ce24f75:image.png) + +原始PDF页面 + +![搜索关键字“叶片描述”](attachment:cc341750-9fb0-4863-b758-d77194a783c1:image.png) + +搜索关键字“叶片描述” + +返回结果如下： + +```json +{ + "status": "success", + "message": "搜索成功", + "data": { + "result": [ + { + "page_idx": 14, + "span_range": [ + 0, + 0 + ], + "bbox": [ + 70, + 586, + 525, + 614 + ] + }, + { + "page_idx": 14, + "span_range": [ + 0, + 0 + ], + "bbox": [ + 69, + 625, + 147, + 638 + ] + } + ] + } +} +``` + +应用展示 + +![c17614a4ad447a787d3a1131825a2d22.png](attachment:0c3997f7-3107-43c9-8c90-b99b8283dbf7:c17614a4ad447a787d3a1131825a2d22.png) + +### 7. 性能优化 + +通过将CPU密集型任务和GPU密集型任务切分为相互独立的进程，更好利用了高性能服务器的硬件资源，不仅可以进行多路解析，在单任务的处理上也有显著提升。在附加了实体识别和智能分段等附加功能，处理时间和使用MinerU CLI所需的时间几乎没有差别。在多路服务器上可以通过简单配置，使不同的任务使用服务器上独立硬件资源，成倍提速。 + +| 解析样例 | 原生MinerU | 本方案 | +| --- | --- | --- | +| PDF测试1（158页英文内容） | 79.26s | 79.661 | +| PDF测试2（731页英文内容） | 402s | 393s | +| PDF测试2（670页中文内容） | 163 | 116.53 | + +测试环境： + +- GPU: NVIDIA A100 80GB +- CPU: Intel(R) Xeon(R) Gold 6348 CPU @ 2.60GHz + +## 核心组件 + +| 依赖名称 | 版本范围 | 功能描述 | +| --- | --- | --- | +| **fastapi** | `>=0.115.12` | **Web 框架**：提供高性能的 API 服务接口，支持异步编程。 | +| **mineru** | `>=2.5.0` | **核心引擎**：MinerU 核心库，提供 PDF 解析、布局分析和公式提取能力。 | +| **celery** | `>=5.5.3` | **异步任务队列**：用于处理耗时的 PDF 解析和 NLP任务。 | +| **vllm** | `>=0.11.0` | **大模型推理**：高性能 LLM/VLM 推理引擎，用于加速VLM视觉大模型的推理。 | +| **transformers** | `>=4.53.0` | **NLP 模型库**：用于加载和运行各种预训练模型（如 NER、Embedding）。 | +| **minio** | `>=7.2.15` | **对象存储客户端**：用于上传和下载 PDF 源文件及解析后的结果文件。 | +| **sentence-transformers** | `>=5.1.0` | **文本向量化**：用于生成文本 Embedding，实现基于文本语义的切分。 | +| **sqlalchemy** | `>=2.0.41` | **ORM 框架**：用于数据库操作 and 连接管理。 | +| **redis** | (Implied) | **缓存与消息中间件**：作为 Celery 的 Broker 和应用缓存（通过 `redislite` 或外部服务）。 | + +## 应用价值 + +- **提质**：将非结构化文档转化为高质量的 Markdown 数据，为大模型知识库建设提供了坚实的基础。 +- **增效**：通过自动化、并行化的处理流程，结合软件架构的优化，将文档处理效率提升数倍，实现了从“小时级”到“分钟级”的跨越。 +- **赋能**：该平台可广泛应用于政策文件检索、合同智能审核、技术文档问答等多种业务场景，切实助力企业提效减负。 + +| 对比维度 | 原生 MinerU | 本方案 | 优势 | +| --- | --- | --- | --- | +| **系统架构** | 单体应用，命令行工具 | 分布式微服务架构（FastAPI + Celery + vLLM + Redis） | 支持高并发、可水平扩展，适合企业级部署 | +| **服务化能力** | 无服务接口，需本地调用 | 提供 RESTful API，支持 HTTP 调用 | 易于集成到现有系统，支持多语言调用 | +| **并发处理** | 单任务串行处理 | 分布式任务队列，支持多 Worker 并行 | 处理效率提升数倍，支持海量文档批量处理 | +| **资源利用** | CPU/GPU 资源利用率低 | CPU 密集任务与 GPU 推理任务分离，多卡并行 | 单机吞吐量大幅提升，资源利用最大化 | +| **文档格式支持** | 主要支持 PDF | 全格式支持（Word、Excel、PDF、PPT、图片） | 一站式解决企业多格式文档处理需求 | +| **表格处理** | 基础表格识别 | 智能表格处理（键值对转换 + Markdown 表格） | 满足问答 and 可视化展示双重场景需求 | +| **文本切分** | 无切分功能 | 基于语义的智能切分（BGE Embedding + 聚类） | 保证语义完整性，提升检索准确性 | +| **上下文管理** | 无标题聚合功能 | 上下文感知的标题聚合（多级标题路径保留） | 解决碎片化导致的上下文丢失问题 | +| **信息提取** | 无实体识别功能 | 集成 NER 模型，自动提取关键实体 | 支持多维度精准检索，为文档打智能标签 | + +""" +def test_semantic_chunking_basic(embed_fn, sample_markdown): + """测试基本的语义切分逻辑 (使用真实嵌入模型)""" + + # 配置切分参数 + parser_config = { + "chunk_token_num": 1000, # 针对标准文档调整 token 数 + "overlapped_percent": 0.1 + } + + # 执行语义切分 + chunks = semantic.chunk_markdown( + sample_markdown, + parser_config=parser_config, + embed_fn=embed_fn + ) + + # 1. 基础验证 + assert isinstance(chunks, list) + assert len(chunks) >= 5, f"预期至少切分为 5 个片段，实际仅有 {len(chunks)} 个" + + # 2. 验证上下文感知与标题聚合 (检查 RAGFlow 风格的层级路径) + # 验证子章节片段是否保留了父级标题路径 + # 检查 "Docling" 相关的片段 (属于 "核心技术亮点" -> "多模态融合的高精度解析") + docling_chunks = [c for c in chunks if "Docling" in c] + assert docling_chunks, "未找到包含 'Docling' 的片段" + for chunk in docling_chunks: + assert "核心技术亮点" in chunk, "子章节片段丢失了父级标题 '核心技术亮点'" + assert "多模态融合的高精度解析" in chunk, "子章节片段丢失了自身标题 '多模态融合的高精度解析'" + # 验证层级分隔符 + path_part = chunk.split("\n")[0] if "\n" in chunk else chunk + assert "|" in path_part, f"标题路径中缺失层级分隔符 '|': {path_part}" + assert path_part.count("|") >= 1, f"标题路径层级深度不足: {path_part}" + + # 检查顶级章节 (Level 2) 是否保持独立 (不应包含其他不相关的顶级标题) + pain_point_chunks = [c for c in chunks if "格式繁杂兼容难" in c] + for chunk in pain_point_chunks: + path_part = chunk.split("\n")[0] + assert "痛点与挑战" in path_part + assert "技术方案概览" not in path_part, "顶级标题路径污染：包含了不相关的顶级标题" + + # 3. 验证关键技术词汇识别 + # 重点检查文档中出现的核心组件词汇 + keywords_to_check = ["FastAPI", "Celery", "vLLM", "MinerU"] + found_keywords = [s for s in keywords_to_check if any(s in chunk for chunk in chunks)] + assert len(found_keywords) > 0, f"在切分结果中未找到关键技术词汇: {keywords_to_check}" + + # 4. 验证核心章节内容是否存在 + has_arch_section = any("架构设计" in chunk for chunk in chunks) + has_highlight_section = any("核心技术亮点" in chunk for chunk in chunks) + assert has_arch_section, "未找到 '架构设计' 相关内容" + assert has_highlight_section, "未找到 '核心技术亮点' 相关内容" + + # 5. 验证语义聚类是否将不同主题分开 + # 架构设计和核心技术亮点是两个独立的大章节，语义聚类应该尽量避免将它们的核心内容混在一个 chunk 中 + arch_start_chunks = [i for i, c in enumerate(chunks) if "## 架构设计" in c] + highlight_start_chunks = [i for i, c in enumerate(chunks) if "## 核心技术亮点" in c] + + if arch_start_chunks and highlight_start_chunks: + # 确保起始片段不重合 + assert not set(arch_start_chunks).intersection(set(highlight_start_chunks)), \ + "语义聚类错误：架构设计与核心技术亮点的章节头部被挤在了同一个 chunk 中" + + print(f"\n[测试成功] 文档成功切分为 {len(chunks)} 个片段") + print(f"识别到的关键技术词汇: {found_keywords}") + + # 直接输出到控制台预览切分结果，避免文件副作用 + print("\n--- 语义切分结果开始 ---") + for idx, chunk in enumerate(chunks, 1): + print(f"\n[Chunk {idx}]\n{chunk}") + print("\n--- 语义切分结果结束 ---") + +def test_heading_inference(): + """测试标题层级推断工具类""" + from yuxi.knowledge.chunking.ragflow_like.utils.md_parser_utils import infer_heading_level + + assert infer_heading_level("1. 简介") == 1 + assert infer_heading_level("1.1 详细设计") == 2 + assert infer_heading_level("1.2.3 核心逻辑") == 3 + assert infer_heading_level("一、 背景") == 1 + assert infer_heading_level("普通文本") == 1 \ No newline at end of file diff --git a/backend/test/unit/plugins/test_ragflow_like_chunking.py b/backend/test/unit/plugins/test_ragflow_like_chunking.py index 3e900c63..c36a0aea 100644 --- a/backend/test/unit/plugins/test_ragflow_like_chunking.py +++ b/backend/test/unit/plugins/test_ragflow_like_chunking.py @@ -7,8 +7,10 @@ sys.path.append(os.getcwd()) from yuxi.knowledge.chunking.ragflow_like.dispatcher import chunk_markdown from yuxi.knowledge.chunking.ragflow_like.nlp import bullets_category, count_tokens +from yuxi.knowledge.chunking.ragflow_like.utils.semantic_utils import split_sentences_chinese from yuxi.knowledge.chunking.ragflow_like.presets import ( CHUNK_ENGINE_VERSION, + CHUNK_PRESET_IDS, get_chunk_preset_options, map_to_internal_parser_id, resolve_chunk_processing_params, @@ -85,6 +87,13 @@ def test_book_chunking_hierarchical_merge() -> None: assert any("第一章" in ck["content"] for ck in chunks) +def test_split_sentences_chinese_should_keep_quote_boundary() -> None: + text = '他说：“你好。”然后问：“你在吗？”最后结束！' + sentences = split_sentences_chinese(text) + + assert sentences == ["他说：“你好。”", "然后问：“你在吗？”", "最后结束！"] + + def test_markdown_heading_has_higher_weight_in_bullet_category() -> None: sections = [ "# 3.2 个人所得项目及计税、申报方式概括", @@ -108,7 +117,7 @@ def test_mid_sentence_bullet_marker_should_not_be_treated_as_heading() -> None: def test_chunk_preset_options_include_description() -> None: options = get_chunk_preset_options() - assert len(options) == 4 + assert {option["value"] for option in options} == CHUNK_PRESET_IDS assert all(isinstance(option.get("description"), str) and option["description"] for option in options) diff --git a/backend/uv.lock b/backend/uv.lock index 545804cf..9857a198 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -2521,6 +2521,18 @@ wheels = [ { url = "https://pypi.tuna.tsinghua.edu.cn/packages/9c/46/f6b4ad632c67ef35209a66127e4bddc95759649dd595f71f13fba11bdf9a/mcp-1.27.0-py3-none-any.whl", hash = "sha256:5ce1fa81614958e267b21fb2aa34e0aea8e2c6ede60d52aba45fd47246b4d741", size = 215967, upload-time = "2026-04-02T14:48:07.24Z" }, ] +[[package]] +name = "mdit-py-plugins" +version = "0.5.0" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +dependencies = [ + { name = "markdown-it-py" }, +] +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/b2/fd/a756d36c0bfba5f6e39a1cdbdbfdd448dc02692467d83816dff4592a1ebc/mdit_py_plugins-0.5.0.tar.gz", hash = "sha256:f4918cb50119f50446560513a8e311d574ff6aaed72606ddae6d35716fe809c6", size = 44655, upload-time = "2025-08-11T07:25:49.083Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/fb/86/dd6e5db36df29e76c7a7699123569a4a18c1623ce68d826ed96c62643cae/mdit_py_plugins-0.5.0-py3-none-any.whl", hash = "sha256:07a08422fc1936a5d26d146759e9155ea466e842f5ab2f7d2266dd084c8dab1f", size = 57205, upload-time = "2025-08-11T07:25:47.597Z" }, +] + [[package]] name = "mdurl" version = "0.1.2" @@ -4337,6 +4349,38 @@ torch = [ { name = "torch", version = "2.8.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform != 'darwin'" }, ] +[[package]] +name = "scikit-learn" +version = "1.8.0" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +dependencies = [ + { name = "joblib" }, + { name = "numpy" }, + { name = "scipy" }, + { name = "threadpoolctl" }, +] +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/0e/d4/40988bf3b8e34feec1d0e6a051446b1f66225f8529b9309becaeef62b6c4/scikit_learn-1.8.0.tar.gz", hash = "sha256:9bccbb3b40e3de10351f8f5068e105d0f4083b1a65fa07b6634fbc401a6287fd", size = 7335585, upload-time = "2025-12-10T07:08:53.618Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/90/74/e6a7cc4b820e95cc38cf36cd74d5aa2b42e8ffc2d21fe5a9a9c45c1c7630/scikit_learn-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5fb63362b5a7ddab88e52b6dbb47dac3fd7dafeee740dc6c8d8a446ddedade8e", size = 8548242, upload-time = "2025-12-10T07:07:51.568Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/49/d8/9be608c6024d021041c7f0b3928d4749a706f4e2c3832bbede4fb4f58c95/scikit_learn-1.8.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:5025ce924beccb28298246e589c691fe1b8c1c96507e6d27d12c5fadd85bfd76", size = 8079075, upload-time = "2025-12-10T07:07:53.697Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/dd/47/f187b4636ff80cc63f21cd40b7b2d177134acaa10f6bb73746130ee8c2e5/scikit_learn-1.8.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4496bb2cf7a43ce1a2d7524a79e40bc5da45cf598dbf9545b7e8316ccba47bb4", size = 8660492, upload-time = "2025-12-10T07:07:55.574Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/97/74/b7a304feb2b49df9fafa9382d4d09061a96ee9a9449a7cbea7988dda0828/scikit_learn-1.8.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a0bcfe4d0d14aec44921545fd2af2338c7471de9cb701f1da4c9d85906ab847a", size = 8931904, upload-time = "2025-12-10T07:07:57.666Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/9f/c4/0ab22726a04ede56f689476b760f98f8f46607caecff993017ac1b64aa5d/scikit_learn-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:35c007dedb2ffe38fe3ee7d201ebac4a2deccd2408e8621d53067733e3c74809", size = 8019359, upload-time = "2025-12-10T07:07:59.838Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/24/90/344a67811cfd561d7335c1b96ca21455e7e472d281c3c279c4d3f2300236/scikit_learn-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:8c497fff237d7b4e07e9ef1a640887fa4fb765647f86fbe00f969ff6280ce2bb", size = 7641898, upload-time = "2025-12-10T07:08:01.36Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/03/aa/e22e0768512ce9255eba34775be2e85c2048da73da1193e841707f8f039c/scikit_learn-1.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0d6ae97234d5d7079dc0040990a6f7aeb97cb7fa7e8945f1999a429b23569e0a", size = 8513770, upload-time = "2025-12-10T07:08:03.251Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/58/37/31b83b2594105f61a381fc74ca19e8780ee923be2d496fcd8d2e1147bd99/scikit_learn-1.8.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:edec98c5e7c128328124a029bceb09eda2d526997780fef8d65e9a69eead963e", size = 8044458, upload-time = "2025-12-10T07:08:05.336Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/2d/5a/3f1caed8765f33eabb723596666da4ebbf43d11e96550fb18bdec42b467b/scikit_learn-1.8.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:74b66d8689d52ed04c271e1329f0c61635bcaf5b926db9b12d58914cdc01fe57", size = 8610341, upload-time = "2025-12-10T07:08:07.732Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/38/cf/06896db3f71c75902a8e9943b444a56e727418f6b4b4a90c98c934f51ed4/scikit_learn-1.8.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8fdf95767f989b0cfedb85f7ed8ca215d4be728031f56ff5a519ee1e3276dc2e", size = 8900022, upload-time = "2025-12-10T07:08:09.862Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/1c/f9/9b7563caf3ec8873e17a31401858efab6b39a882daf6c1bfa88879c0aa11/scikit_learn-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:2de443b9373b3b615aec1bb57f9baa6bb3a9bd093f1269ba95c17d870422b271", size = 7989409, upload-time = "2025-12-10T07:08:12.028Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/49/bd/1f4001503650e72c4f6009ac0c4413cb17d2d601cef6f71c0453da2732fc/scikit_learn-1.8.0-cp313-cp313-win_arm64.whl", hash = "sha256:eddde82a035681427cbedded4e6eff5e57fa59216c2e3e90b10b19ab1d0a65c3", size = 7619760, upload-time = "2025-12-10T07:08:13.688Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/d2/7d/a630359fc9dcc95496588c8d8e3245cc8fd81980251079bc09c70d41d951/scikit_learn-1.8.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:7cc267b6108f0a1499a734167282c00c4ebf61328566b55ef262d48e9849c735", size = 8826045, upload-time = "2025-12-10T07:08:15.215Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/cc/56/a0c86f6930cfcd1c7054a2bc417e26960bb88d32444fe7f71d5c2cfae891/scikit_learn-1.8.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:fe1c011a640a9f0791146011dfd3c7d9669785f9fed2b2a5f9e207536cf5c2fd", size = 8420324, upload-time = "2025-12-10T07:08:17.561Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/46/1e/05962ea1cebc1cf3876667ecb14c283ef755bf409993c5946ade3b77e303/scikit_learn-1.8.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:72358cce49465d140cc4e7792015bb1f0296a9742d5622c67e31399b75468b9e", size = 8680651, upload-time = "2025-12-10T07:08:19.952Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/fe/56/a85473cd75f200c9759e3a5f0bcab2d116c92a8a02ee08ccd73b870f8bb4/scikit_learn-1.8.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:80832434a6cc114f5219211eec13dcbc16c2bac0e31ef64c6d346cde3cf054cb", size = 8925045, upload-time = "2025-12-10T07:08:22.11Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/cc/b7/64d8cfa896c64435ae57f4917a548d7ac7a44762ff9802f75a79b77cb633/scikit_learn-1.8.0-cp313-cp313t-win_amd64.whl", hash = "sha256:ee787491dbfe082d9c3013f01f5991658b0f38aa8177e4cd4bf434c58f551702", size = 8507994, upload-time = "2025-12-10T07:08:23.943Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/5e/37/e192ea709551799379958b4c4771ec507347027bb7c942662c7fbeba31cb/scikit_learn-1.8.0-cp313-cp313t-win_arm64.whl", hash = "sha256:bf97c10a3f5a7543f9b88cbf488d33d175e9146115a451ae34568597ba33dcde", size = 7869518, upload-time = "2025-12-10T07:08:25.71Z" }, +] + [[package]] name = "scipy" version = "1.17.1" @@ -4756,6 +4800,15 @@ wheels = [ { url = "https://pypi.tuna.tsinghua.edu.cn/packages/f5/15/a11f7bb3cbc97dfecf32a90552f5a8f8a5c99316a99c6c17bdabf5baf256/thinc-8.3.13-cp313-cp313-win_arm64.whl", hash = "sha256:723949cab11d1925c15447928513a718276316cec6e0de28337cca0a62be0521", size = 1644606, upload-time = "2026-03-23T07:22:21.339Z" }, ] +[[package]] +name = "threadpoolctl" +version = "3.6.0" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/b7/4d/08c89e34946fce2aec4fbb45c9016efd5f4d7f24af8e5d93296e935631d8/threadpoolctl-3.6.0.tar.gz", hash = "sha256:8ab8b4aa3491d812b623328249fab5302a68d2d71745c8a4c719a2fcaba9f44e", size = 21274, upload-time = "2025-03-13T13:49:23.031Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl", hash = "sha256:43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb", size = 18638, upload-time = "2025-03-13T13:49:21.846Z" }, +] + [[package]] name = "tiktoken" version = "0.12.0" @@ -4878,9 +4931,9 @@ dependencies = [ { name = "typing-extensions", marker = "sys_platform == 'darwin'" }, ] wheels = [ - { url = "https://download.pytorch.org/whl/cpu/torch-2.8.0-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:a47b7986bee3f61ad217d8a8ce24605809ab425baf349f97de758815edd2ef54" }, - { url = "https://download.pytorch.org/whl/cpu/torch-2.8.0-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:fbe2e149c5174ef90d29a5f84a554dfaf28e003cb4f61fa2c8c024c17ec7ca58" }, - { url = "https://download.pytorch.org/whl/cpu/torch-2.8.0-cp313-none-macosx_11_0_arm64.whl", hash = "sha256:057efd30a6778d2ee5e2374cd63a63f63311aa6f33321e627c655df60abdd390" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:a47b7986bee3f61ad217d8a8ce24605809ab425baf349f97de758815edd2ef54" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:fbe2e149c5174ef90d29a5f84a554dfaf28e003cb4f61fa2c8c024c17ec7ca58" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0-cp313-none-macosx_11_0_arm64.whl", hash = "sha256:057efd30a6778d2ee5e2374cd63a63f63311aa6f33321e627c655df60abdd390" }, ] [[package]] @@ -4903,19 +4956,19 @@ dependencies = [ { name = "typing-extensions", marker = "sys_platform != 'darwin'" }, ] wheels = [ - { url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-linux_s390x.whl", hash = "sha256:0e34e276722ab7dd0dffa9e12fe2135a9b34a0e300c456ed7ad6430229404eb5" }, - { url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:610f600c102386e581327d5efc18c0d6edecb9820b4140d26163354a99cd800d" }, - { url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:cb9a8ba8137ab24e36bf1742cb79a1294bd374db570f09fc15a5e1318160db4e" }, - { url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-win_amd64.whl", hash = "sha256:2be20b2c05a0cce10430cc25f32b689259640d273232b2de357c35729132256d" }, - { url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-win_arm64.whl", hash = "sha256:99fc421a5d234580e45957a7b02effbf3e1c884a5dd077afc85352c77bf41434" }, - { url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313-linux_s390x.whl", hash = "sha256:8b5882276633cf91fe3d2d7246c743b94d44a7e660b27f1308007fdb1bb89f7d" }, - { url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:a5064b5e23772c8d164068cc7c12e01a75faf7b948ecd95a0d4007d7487e5f25" }, - { url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:8f81dedb4c6076ec325acc3b47525f9c550e5284a18eae1d9061c543f7b6e7de" }, - { url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313-win_amd64.whl", hash = "sha256:e1ee1b2346ade3ea90306dfbec7e8ff17bc220d344109d189ae09078333b0856" }, - { url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313-win_arm64.whl", hash = "sha256:64c187345509f2b1bb334feed4666e2c781ca381874bde589182f81247e61f88" }, - { url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:af81283ac671f434b1b25c95ba295f270e72db1fad48831eb5e4748ff9840041" }, - { url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:a9dbb6f64f63258bc811e2c0c99640a81e5af93c531ad96e95c5ec777ea46dab" }, - { url = "https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313t-win_amd64.whl", hash = "sha256:6d93a7165419bc4b2b907e859ccab0dea5deeab261448ae9a5ec5431f14c0e64" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-linux_s390x.whl", hash = "sha256:0e34e276722ab7dd0dffa9e12fe2135a9b34a0e300c456ed7ad6430229404eb5" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:610f600c102386e581327d5efc18c0d6edecb9820b4140d26163354a99cd800d" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:cb9a8ba8137ab24e36bf1742cb79a1294bd374db570f09fc15a5e1318160db4e" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-win_amd64.whl", hash = "sha256:2be20b2c05a0cce10430cc25f32b689259640d273232b2de357c35729132256d" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp312-cp312-win_arm64.whl", hash = "sha256:99fc421a5d234580e45957a7b02effbf3e1c884a5dd077afc85352c77bf41434" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313-linux_s390x.whl", hash = "sha256:8b5882276633cf91fe3d2d7246c743b94d44a7e660b27f1308007fdb1bb89f7d" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:a5064b5e23772c8d164068cc7c12e01a75faf7b948ecd95a0d4007d7487e5f25" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:8f81dedb4c6076ec325acc3b47525f9c550e5284a18eae1d9061c543f7b6e7de" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313-win_amd64.whl", hash = "sha256:e1ee1b2346ade3ea90306dfbec7e8ff17bc220d344109d189ae09078333b0856" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313-win_arm64.whl", hash = "sha256:64c187345509f2b1bb334feed4666e2c781ca381874bde589182f81247e61f88" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:af81283ac671f434b1b25c95ba295f270e72db1fad48831eb5e4748ff9840041" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:a9dbb6f64f63258bc811e2c0c99640a81e5af93c531ad96e95c5ec777ea46dab" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp313-cp313t-win_amd64.whl", hash = "sha256:6d93a7165419bc4b2b907e859ccab0dea5deeab261448ae9a5ec5431f14c0e64" }, ] [[package]] @@ -5616,6 +5669,7 @@ dependencies = [ { name = "aiosqlite" }, { name = "argon2-cffi" }, { name = "asyncpg" }, + { name = "beautifulsoup4" }, { name = "chardet" }, { name = "colorlog" }, { name = "dashscope" }, @@ -5642,11 +5696,14 @@ dependencies = [ { name = "llama-index" }, { name = "llama-index-readers-file" }, { name = "loguru" }, + { name = "markdown-it-py" }, { name = "markdownify" }, { name = "mcp" }, + { name = "mdit-py-plugins" }, { name = "minio" }, { name = "neo4j" }, { name = "networkx" }, + { name = "nltk" }, { name = "openai" }, { name = "opencv-python-headless" }, { name = "pillow" }, @@ -5664,6 +5721,7 @@ dependencies = [ { name = "readability-lxml" }, { name = "redis" }, { name = "rich" }, + { name = "scikit-learn" }, { name = "sqlalchemy", extra = ["asyncio"] }, { name = "tabulate" }, { name = "tavily-python" }, @@ -5689,6 +5747,7 @@ requires-dist = [ { name = "aiosqlite", specifier = ">=0.20.0" }, { name = "argon2-cffi", specifier = ">=25.1.0" }, { name = "asyncpg", specifier = ">=0.30.0" }, + { name = "beautifulsoup4", specifier = ">=4.12.0" }, { name = "chardet", specifier = ">=5.0.0" }, { name = "colorlog", specifier = ">=6.9.0" }, { name = "dashscope", specifier = ">=1.23.2" }, @@ -5715,11 +5774,14 @@ requires-dist = [ { name = "llama-index", specifier = ">=0.14" }, { name = "llama-index-readers-file", specifier = ">=0.4.7" }, { name = "loguru", specifier = ">=0.7.3" }, + { name = "markdown-it-py", specifier = ">=3.0.0" }, { name = "markdownify", specifier = ">=1.1.0" }, { name = "mcp", specifier = ">=1.20" }, + { name = "mdit-py-plugins", specifier = ">=0.4.0" }, { name = "minio", specifier = ">=7.2.7" }, { name = "neo4j", specifier = ">=5.28.1" }, { name = "networkx", specifier = ">=3.5" }, + { name = "nltk", specifier = ">=3.8.1" }, { name = "openai", specifier = ">=1.109" }, { name = "opencv-python-headless", specifier = ">=4.11.0.86" }, { name = "pillow", specifier = ">=10.5.0" }, @@ -5737,6 +5799,7 @@ requires-dist = [ { name = "readability-lxml", specifier = ">=0.8.1" }, { name = "redis", specifier = ">=5.2.0" }, { name = "rich", specifier = ">=13.7.1" }, + { name = "scikit-learn", specifier = ">=1.3.0" }, { name = "sqlalchemy", extras = ["asyncio"], specifier = ">=2.0.0" }, { name = "tabulate", specifier = ">=0.9.0" }, { name = "tavily-python", specifier = ">=0.7.0" }, diff --git a/docker-compose.yml b/docker-compose.yml index 1f382df4..358259bd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -52,6 +52,7 @@ services: - ./.env:/app/.env - ${MODEL_DIR:-./models}:/models # 使用默认值处理未定义的环境变量 - /var/run/docker.sock:/var/run/docker.sock + - nltk_data:/root/nltk_data # 使用命名卷持久化NLTK数据，避免路径展开差异 ports: - "5050:5050" networks: @@ -430,6 +431,9 @@ services: - app-network restart: unless-stopped +volumes: + nltk_data: + networks: app-network: driver: bridge diff --git a/web/package.json b/web/package.json index bb8b62bc..9dc7dd78 100644 --- a/web/package.json +++ b/web/package.json @@ -2,6 +2,7 @@ "name": "yuxi-web", "version": "0.6.1", "private": true, + "type": "module", "scripts": { "dev": "vite", "server": "vite serve --host", diff --git a/web/src/utils/chunk_presets.js b/web/src/utils/chunk_presets.js index ebd6285f..642dddd1 100644 --- a/web/src/utils/chunk_presets.js +++ b/web/src/utils/chunk_presets.js @@ -18,6 +18,11 @@ export const CHUNK_PRESET_OPTIONS = [ value: 'laws', label: 'Laws', description: '法规分块：按法条层级组织与合并，适合法律法规、制度规范类文本。' + }, + { + value: 'semantic', + label: 'Semantic', + description: '语义分块：利用嵌入和聚类算法进行语义切分，并自动增强标题上下文。' } ]