From bba9bc974f64bf67a1ffd18effb368189c098ba3 Mon Sep 17 00:00:00 2001 From: Wenjie Zhang Date: Tue, 28 Apr 2026 20:30:02 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0=20separator=20?= =?UTF-8?q?=E8=A7=A3=E6=9E=90=E5=99=A8=E5=8F=8A=E7=9B=B8=E5=85=B3=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=EF=BC=8C=E6=94=AF=E6=8C=81=E4=B8=A5=E6=A0=BC=E5=88=86?= =?UTF-8?q?=E9=9A=94=E5=8A=9F=E8=83=BD=20Question:=20=E7=9F=A5=E8=AF=86?= =?UTF-8?q?=E5=BA=93=E5=AF=B9excel=E5=88=87=E5=88=86=E6=97=A0=E6=B3=95?= =?UTF-8?q?=E6=8C=89=E8=A1=8C=E5=88=87=E5=88=86=20Fixes=20#664?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../chunking/ragflow_like/dispatcher.py | 4 +- .../chunking/ragflow_like/parsers/__init__.py | 4 +- .../ragflow_like/parsers/separator.py | 86 +++++++++++++++++++ .../chunking/ragflow_like/presets.py | 19 ++++ web/src/utils/chunk_presets.js | 5 ++ 5 files changed, 115 insertions(+), 3 deletions(-) create mode 100644 backend/package/yuxi/knowledge/chunking/ragflow_like/parsers/separator.py diff --git a/backend/package/yuxi/knowledge/chunking/ragflow_like/dispatcher.py b/backend/package/yuxi/knowledge/chunking/ragflow_like/dispatcher.py index a9d5971a..f831e744 100644 --- a/backend/package/yuxi/knowledge/chunking/ragflow_like/dispatcher.py +++ b/backend/package/yuxi/knowledge/chunking/ragflow_like/dispatcher.py @@ -2,7 +2,7 @@ from __future__ import annotations from typing import Any -from yuxi.knowledge.chunking.ragflow_like.parsers import book, general, laws, qa, semantic +from yuxi.knowledge.chunking.ragflow_like.parsers import book, general, laws, qa, semantic, separator from yuxi.knowledge.chunking.ragflow_like.presets import map_to_internal_parser_id, normalize_chunk_preset_id @@ -44,6 +44,8 @@ def _dispatch_markdown_parser( return laws.chunk_markdown(filename, markdown_content, parser_config) if parser_id == "semantic": return semantic.chunk_markdown(markdown_content, parser_config) + if parser_id == "separator": + return separator.chunk_markdown(markdown_content, parser_config) return general.chunk_markdown(markdown_content, parser_config) diff --git a/backend/package/yuxi/knowledge/chunking/ragflow_like/parsers/__init__.py b/backend/package/yuxi/knowledge/chunking/ragflow_like/parsers/__init__.py index 91bd1683..317d2cf9 100644 --- a/backend/package/yuxi/knowledge/chunking/ragflow_like/parsers/__init__.py +++ b/backend/package/yuxi/knowledge/chunking/ragflow_like/parsers/__init__.py @@ -1,3 +1,3 @@ -from yuxi.knowledge.chunking.ragflow_like.parsers import book, general, laws, qa +from yuxi.knowledge.chunking.ragflow_like.parsers import book, general, laws, qa, separator -__all__ = ["general", "qa", "book", "laws"] +__all__ = ["general", "qa", "book", "laws", "separator"] diff --git a/backend/package/yuxi/knowledge/chunking/ragflow_like/parsers/separator.py b/backend/package/yuxi/knowledge/chunking/ragflow_like/parsers/separator.py new file mode 100644 index 00000000..dedc16e3 --- /dev/null +++ b/backend/package/yuxi/knowledge/chunking/ragflow_like/parsers/separator.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +from typing import Any + +from yuxi.knowledge.chunking.ragflow_like import nlp +from yuxi.knowledge.chunking.ragflow_like.parsers.general import _iter_sections, _unescape_delimiter + + +def _slice_text_by_tokens(text: str, max_tokens: int, overlap_tokens: int) -> list[str]: + if max_tokens <= 0: + return [text] if text.strip() else [] + + units = [part for part in text] + chunks: list[str] = [] + start = 0 + + while start < len(units): + current = "" + current_tokens = 0 + end = start + + while end < len(units): + next_text = current + units[end] + next_tokens = nlp.count_tokens(next_text) + if current and next_tokens > max_tokens: + break + current = next_text + current_tokens = next_tokens + end += 1 + if current_tokens >= max_tokens: + break + + chunk = current.strip() + if chunk: + chunks.append(chunk) + + if end >= len(units): + break + + if overlap_tokens <= 0: + start = end + continue + + backtrack = end + overlap_text = "" + while backtrack > start: + candidate = units[backtrack - 1] + overlap_text + if nlp.count_tokens(candidate) > overlap_tokens: + break + overlap_text = candidate + backtrack -= 1 + + start = backtrack if backtrack < end else end + + return chunks + + +def _split_section_with_overlap(section: str, chunk_token_num: int, overlapped_percent: int) -> list[str]: + overlap_tokens = 0 + if chunk_token_num > 0 and overlapped_percent > 0: + overlap_tokens = int(chunk_token_num * max(0, min(overlapped_percent, 99)) / 100) + return _slice_text_by_tokens(section, chunk_token_num, overlap_tokens) + + +def chunk_markdown(markdown_content: str, parser_config: dict[str, Any] | None = None) -> list[str]: + parser_config = parser_config or {} + + delimiter = _unescape_delimiter(str(parser_config.get("delimiter", "\n") or "\n")) + chunk_token_num = int(parser_config.get("chunk_token_num", 512) or 512) + overlapped_percent = int(parser_config.get("overlapped_percent", 0) or 0) + + sections = _iter_sections(markdown_content, delimiter) + chunks: list[str] = [] + + for section, _ in sections: + text = (section or "").strip() + if not text: + continue + + if chunk_token_num > 0 and nlp.count_tokens(text) > chunk_token_num: + chunks.extend(_split_section_with_overlap(text, chunk_token_num, overlapped_percent)) + continue + + chunks.append(text) + + return chunks diff --git a/backend/package/yuxi/knowledge/chunking/ragflow_like/presets.py b/backend/package/yuxi/knowledge/chunking/ragflow_like/presets.py index 2e3d7307..5c34a3f4 100644 --- a/backend/package/yuxi/knowledge/chunking/ragflow_like/presets.py +++ b/backend/package/yuxi/knowledge/chunking/ragflow_like/presets.py @@ -10,6 +10,7 @@ CHUNK_PRESET_QA = "qa" CHUNK_PRESET_BOOK = "book" CHUNK_PRESET_LAWS = "laws" CHUNK_PRESET_SEMANTIC = "semantic" +CHUNK_PRESET_SEPARATOR = "separator" CHUNK_PRESET_IDS = { CHUNK_PRESET_GENERAL, @@ -17,6 +18,7 @@ CHUNK_PRESET_IDS = { CHUNK_PRESET_BOOK, CHUNK_PRESET_LAWS, CHUNK_PRESET_SEMANTIC, + CHUNK_PRESET_SEPARATOR, } CHUNK_PRESET_DESCRIPTIONS: dict[str, str] = { @@ -25,6 +27,7 @@ CHUNK_PRESET_DESCRIPTIONS: dict[str, str] = { CHUNK_PRESET_BOOK: "书籍分块：强化章节标题识别并做层级合并，适合教材、手册、长章节文档。", CHUNK_PRESET_LAWS: "法规分块：按法条层级组织与合并，适合法律法规、制度规范类文本。", CHUNK_PRESET_SEMANTIC: "语义分块：利用嵌入和聚类算法进行语义切分，并自动增强标题上下文。", + CHUNK_PRESET_SEPARATOR: "严格分隔：命中分隔符即切分，仅超长片段内部继续按长度切分。", } CHUNK_ENGINE_VERSION = "ragflow_like_v1" @@ -74,6 +77,17 @@ _PRESET_DEFAULTS: dict[str, dict[str, Any] | None] = { "method": "light", }, }, + CHUNK_PRESET_SEPARATOR: { + "layout_recognize": "DeepDOC", + "chunk_token_num": 512, + "delimiter": "\n", + "auto_keywords": 0, + "auto_questions": 0, + "html4excel": False, + "topn_tags": 3, + "raptor": {"use_raptor": False}, + "graphrag": {"use_graphrag": False}, + }, } @@ -251,4 +265,9 @@ def get_chunk_preset_options() -> list[dict[str, str]]: "label": "Semantic", "description": CHUNK_PRESET_DESCRIPTIONS[CHUNK_PRESET_SEMANTIC], }, + { + "value": CHUNK_PRESET_SEPARATOR, + "label": "Separator", + "description": CHUNK_PRESET_DESCRIPTIONS[CHUNK_PRESET_SEPARATOR], + }, ] diff --git a/web/src/utils/chunk_presets.js b/web/src/utils/chunk_presets.js index 642dddd1..c2fb8480 100644 --- a/web/src/utils/chunk_presets.js +++ b/web/src/utils/chunk_presets.js @@ -23,6 +23,11 @@ export const CHUNK_PRESET_OPTIONS = [ value: 'semantic', label: 'Semantic', description: '语义分块：利用嵌入和聚类算法进行语义切分，并自动增强标题上下文。' + }, + { + value: 'separator', + label: 'Separator', + description: '严格分隔：命中分隔符即切分，仅超长片段内部继续按长度切分。' } ]