From 60bc408a3e99a387d9faa984b788cf3681db3f4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=82=96=E6=B3=BD=E6=B6=9B?= Date: Fri, 20 Feb 2026 19:35:43 +0800 Subject: [PATCH] feat(knowledge): add ragflow-like chunk preset pipeline --- server/routers/knowledge_router.py | 13 +- src/knowledge/base.py | 40 +- src/knowledge/chunking/__init__.py | 1 + .../chunking/ragflow_like/__init__.py | 3 + .../chunking/ragflow_like/dispatcher.py | 58 ++ src/knowledge/chunking/ragflow_like/nlp.py | 536 ++++++++++++++++++ .../chunking/ragflow_like/parsers/__init__.py | 3 + .../chunking/ragflow_like/parsers/book.py | 61 ++ .../chunking/ragflow_like/parsers/general.py | 46 ++ .../chunking/ragflow_like/parsers/laws.py | 85 +++ .../chunking/ragflow_like/parsers/qa.py | 261 +++++++++ .../chunking/ragflow_like/presets.py | 235 ++++++++ src/knowledge/implementations/lightrag.py | 76 ++- src/knowledge/implementations/milvus.py | 27 +- src/knowledge/manager.py | 25 +- test/api/test_knowledge_router.py | 53 ++ test/test_ragflow_like_chunking.py | 111 ++++ web/src/components/ChunkParamsConfig.vue | 81 ++- web/src/components/FileTable.vue | 50 +- web/src/components/FileUploadModal.vue | 54 +- web/src/components/KnowledgeBaseCard.vue | 43 +- web/src/utils/chunk_presets.js | 33 ++ web/src/views/DataBaseView.vue | 40 +- 23 files changed, 1864 insertions(+), 71 deletions(-) create mode 100644 src/knowledge/chunking/__init__.py create mode 100644 src/knowledge/chunking/ragflow_like/__init__.py create mode 100644 src/knowledge/chunking/ragflow_like/dispatcher.py create mode 100644 src/knowledge/chunking/ragflow_like/nlp.py create mode 100644 src/knowledge/chunking/ragflow_like/parsers/__init__.py create mode 100644 src/knowledge/chunking/ragflow_like/parsers/book.py create mode 100644 src/knowledge/chunking/ragflow_like/parsers/general.py create mode 100644 src/knowledge/chunking/ragflow_like/parsers/laws.py create mode 100644 src/knowledge/chunking/ragflow_like/parsers/qa.py create mode 100644 src/knowledge/chunking/ragflow_like/presets.py create mode 100644 test/test_ragflow_like_chunking.py create mode 100644 web/src/utils/chunk_presets.js diff --git a/server/routers/knowledge_router.py b/server/routers/knowledge_router.py index 1c748356..f79a6cb1 100644 --- a/server/routers/knowledge_router.py +++ b/server/routers/knowledge_router.py @@ -13,6 +13,7 @@ from starlette.responses import StreamingResponse from src.services.task_service import TaskContext, tasker from server.utils.auth_middleware import get_admin_user, get_required_user from src import config, knowledge_base +from src.knowledge.chunking.ragflow_like.presets import ensure_chunk_defaults_in_additional_params from src.knowledge.indexing import SUPPORTED_FILE_EXTENSIONS, is_supported_file_extension, process_file_to_markdown from src.knowledge.utils import calculate_content_hash from src.models.embed import test_all_embedding_models_status, test_embedding_model_status @@ -116,6 +117,7 @@ async def create_database( params.pop("reranker_config", None) remove_reranker_config(kb_type, additional_params) + additional_params = ensure_chunk_defaults_in_additional_params(additional_params) embed_info = config.embed_model_names[embed_model_name] # 将Pydantic模型转换为字典以便JSON序列化 @@ -180,7 +182,7 @@ async def update_database_info( name: str = Body(...), description: str = Body(...), llm_info: dict = Body(None), - additional_params: dict = Body({}), + additional_params: dict | None = Body(None), share_config: dict = Body(None), current_user: User = Depends(get_admin_user), ): @@ -190,6 +192,9 @@ async def update_database_info( f"additional_params={additional_params}, share_config={share_config}" ) try: + if additional_params is not None: + additional_params = ensure_chunk_defaults_in_additional_params(additional_params) + database = await knowledge_base.update_database( db_id, name, @@ -267,7 +272,13 @@ async def add_documents( "chunk_size": params.get("chunk_size", 1000), "chunk_overlap": params.get("chunk_overlap", 200), "qa_separator": params.get("qa_separator", ""), + "chunk_preset_id": params.get("chunk_preset_id"), + "chunk_parser_config": params.get("chunk_parser_config"), } + if not indexing_params.get("chunk_preset_id"): + indexing_params.pop("chunk_preset_id", None) + if not isinstance(indexing_params.get("chunk_parser_config"), dict): + indexing_params.pop("chunk_parser_config", None) # URL 解析与入库（需白名单验证） if content_type == "url": diff --git a/src/knowledge/base.py b/src/knowledge/base.py index c9cadd70..f773bfe6 100644 --- a/src/knowledge/base.py +++ b/src/knowledge/base.py @@ -3,6 +3,10 @@ import os from abc import ABC, abstractmethod from typing import Any +from src.knowledge.chunking.ragflow_like.presets import ( + ensure_chunk_defaults_in_additional_params, + resolve_chunk_processing_params, +) from src.utils import logger from src.utils.datetime_utils import coerce_any_to_utc_datetime, utc_isoformat @@ -76,6 +80,7 @@ class KnowledgeBase(ABC): self.databases_meta = {} for db_id, meta in global_databases_meta.items(): if meta.get("kb_type") == self.kb_type: + normalized_additional_params = ensure_chunk_defaults_in_additional_params(meta.get("additional_params")) self.databases_meta[db_id] = { "name": meta.get("name"), "description": meta.get("description"), @@ -83,7 +88,7 @@ class KnowledgeBase(ABC): "embed_info": meta.get("embed_info"), "llm_info": meta.get("llm_info"), "query_params": meta.get("query_params"), - "metadata": meta.get("additional_params", {}), + "metadata": normalized_additional_params, "created_at": meta.get("created_at"), } @@ -91,7 +96,14 @@ class KnowledgeBase(ABC): self.files_meta = {} for file_id, meta in files_meta.items(): if meta.get("database_id") in self.databases_meta: - self.files_meta[file_id] = meta + db_id = meta.get("database_id") + kb_additional_params = self.databases_meta.get(db_id, {}).get("metadata") or {} + normalized_meta = dict(meta) + normalized_meta["processing_params"] = resolve_chunk_processing_params( + kb_additional_params=kb_additional_params, + file_processing_params=meta.get("processing_params"), + ) + self.files_meta[file_id] = normalized_meta # 过滤评估基准 self.benchmarks_meta = {} @@ -199,6 +211,11 @@ class KnowledgeBase(ABC): # Prepare metadata metadata = await prepare_item_metadata(item, content_type, db_id, params=params) file_id = metadata["file_id"] + kb_additional_params = self.databases_meta.get(db_id, {}).get("metadata") or {} + metadata["processing_params"] = resolve_chunk_processing_params( + kb_additional_params=kb_additional_params, + file_processing_params=metadata.get("processing_params"), + ) # Initial status metadata["status"] = FileStatus.UPLOADED @@ -323,13 +340,16 @@ class KnowledgeBase(ABC): if not params: return - # Merge or overwrite? Usually merge is safer, or replace. - # User might want to change chunk size. current_params = self.files_meta[file_id].get("processing_params", {}) or {} + kb_additional_params = self.databases_meta.get(db_id, {}).get("metadata") or {} logger.debug(f"[update_file_params] file_id={file_id}, current_params={current_params}, new_params={params}") - current_params.update(params) + current_params = resolve_chunk_processing_params( + kb_additional_params=kb_additional_params, + file_processing_params=current_params, + request_params=params, + ) self.files_meta[file_id]["processing_params"] = current_params self.files_meta[file_id]["updated_at"] = utc_isoformat() @@ -414,6 +434,8 @@ class KnowledgeBase(ABC): """ from src.utils import hashstr + kwargs = ensure_chunk_defaults_in_additional_params(kwargs) + # 从 kwargs 中获取 is_private 配置 is_private = kwargs.get("is_private", False) prefix = "kb_private_" if is_private else "kb_" @@ -982,7 +1004,7 @@ class KnowledgeBase(ABC): "embed_info": kb.embed_info, "llm_info": kb.llm_info, "query_params": kb.query_params, - "metadata": kb.additional_params or {}, + "metadata": ensure_chunk_defaults_in_additional_params(kb.additional_params), "created_at": utc_isoformat(kb.created_at) if kb.created_at else utc_isoformat(), } for kb in databases @@ -990,6 +1012,7 @@ class KnowledgeBase(ABC): self.files_meta = {} for kb in databases: + kb_additional_params = self.databases_meta.get(kb.db_id, {}).get("metadata") or {} for record in await file_repo.list_by_db_id(kb.db_id): self.files_meta[record.file_id] = { "file_id": record.file_id, @@ -1003,7 +1026,10 @@ class KnowledgeBase(ABC): "content_hash": record.content_hash, "size": record.file_size, "content_type": record.content_type, - "processing_params": record.processing_params, + "processing_params": resolve_chunk_processing_params( + kb_additional_params=kb_additional_params, + file_processing_params=record.processing_params, + ), "is_folder": record.is_folder, "error": record.error_message, "created_by": record.created_by, diff --git a/src/knowledge/chunking/__init__.py b/src/knowledge/chunking/__init__.py new file mode 100644 index 00000000..a9a2c5b3 --- /dev/null +++ b/src/knowledge/chunking/__init__.py @@ -0,0 +1 @@ +__all__ = [] diff --git a/src/knowledge/chunking/ragflow_like/__init__.py b/src/knowledge/chunking/ragflow_like/__init__.py new file mode 100644 index 00000000..8200a554 --- /dev/null +++ b/src/knowledge/chunking/ragflow_like/__init__.py @@ -0,0 +1,3 @@ +from src.knowledge.chunking.ragflow_like.dispatcher import chunk_file, chunk_markdown + +__all__ = ["chunk_file", "chunk_markdown"] diff --git a/src/knowledge/chunking/ragflow_like/dispatcher.py b/src/knowledge/chunking/ragflow_like/dispatcher.py new file mode 100644 index 00000000..2c845077 --- /dev/null +++ b/src/knowledge/chunking/ragflow_like/dispatcher.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +from typing import Any + +from src.knowledge.chunking.ragflow_like.parsers import book, general, laws, qa +from src.knowledge.chunking.ragflow_like.presets import map_to_internal_parser_id, normalize_chunk_preset_id + + +def _build_chunk_records(text_chunks: list[str], file_id: str, filename: str) -> list[dict[str, Any]]: + records: list[dict[str, Any]] = [] + + for idx, chunk_content in enumerate(text_chunks): + text = (chunk_content or "").strip() + if not text: + continue + + records.append( + { + "id": f"{file_id}_chunk_{idx}", + "content": text, + "file_id": file_id, + "filename": filename, + "chunk_index": idx, + "source": filename, + "chunk_id": f"{file_id}_chunk_{idx}", + } + ) + + return records + + +def _dispatch_markdown_parser(preset_id: str, filename: str, markdown_content: str, parser_config: dict[str, Any]) -> list[str]: + parser_id = map_to_internal_parser_id(preset_id) + + if parser_id == "naive": + return general.chunk_markdown(markdown_content, parser_config) + if parser_id == "qa": + return qa.chunk_markdown(filename, markdown_content, parser_config) + if parser_id == "book": + return book.chunk_markdown(markdown_content, parser_config) + if parser_id == "laws": + return laws.chunk_markdown(filename, markdown_content, parser_config) + + return general.chunk_markdown(markdown_content, parser_config) + + +def chunk_markdown(markdown_content: str, file_id: str, filename: str, processing_params: dict[str, Any]) -> list[dict[str, Any]]: + params = dict(processing_params or {}) + preset_id = normalize_chunk_preset_id(params.get("chunk_preset_id")) + parser_config = params.get("chunk_parser_config") if isinstance(params.get("chunk_parser_config"), dict) else {} + + text_chunks = _dispatch_markdown_parser(preset_id, filename, markdown_content, parser_config) + return _build_chunk_records(text_chunks, file_id, filename) + + +def chunk_file(file_content: str, file_id: str, filename: str, processing_params: dict[str, Any]) -> list[dict[str, Any]]: + # 当前链路中入库前均已转换为 markdown，因此与 chunk_markdown 保持同实现。 + return chunk_markdown(file_content, file_id, filename, processing_params) diff --git a/src/knowledge/chunking/ragflow_like/nlp.py b/src/knowledge/chunking/ragflow_like/nlp.py new file mode 100644 index 00000000..dd3196e2 --- /dev/null +++ b/src/knowledge/chunking/ragflow_like/nlp.py @@ -0,0 +1,536 @@ +from __future__ import annotations + +import random +import re +from collections import Counter +from dataclasses import dataclass, field + + +BULLET_PATTERN = [ + [ + r"第[零一二三四五六七八九十百0-9]+(分?编|部分)", + r"第[零一二三四五六七八九十百0-9]+章", + r"第[零一二三四五六七八九十百0-9]+节", + r"第[零一二三四五六七八九十百0-9]+条", + r"[\(（][零一二三四五六七八九十百]+[\)）]", + ], + [ + r"第[0-9]+章", + r"第[0-9]+节", + r"[0-9]{,2}[\. 、]", + r"[0-9]{,2}\.[0-9]{,2}[^a-zA-Z/%~-]", + r"[0-9]{,2}\.[0-9]{,2}\.[0-9]{,2}", + r"[0-9]{,2}\.[0-9]{,2}\.[0-9]{,2}\.[0-9]{,2}", + ], + [ + r"第[零一二三四五六七八九十百0-9]+章", + r"第[零一二三四五六七八九十百0-9]+节", + r"[零一二三四五六七八九十百]+[ 、]", + r"[\(（][零一二三四五六七八九十百]+[\)）]", + r"[\(（][0-9]{,2}[\)）]", + ], + [ + r"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)", + r"Chapter (I+V?|VI*|XI|IX|X)", + r"Section [0-9]+", + r"Article [0-9]+", + ], + [ + r"^#[^#]", + r"^##[^#]", + r"^###.*", + r"^####.*", + r"^#####.*", + r"^######.*", + ], +] + +MARKDOWN_BULLET_GROUP_INDEX = 4 + + +def count_tokens(text: str) -> int: + """近似 token 计数，避免引入额外依赖。""" + if not text: + return 0 + # 英文单词 + 数字 + CJK 单字 + parts = re.findall(r"[A-Za-z0-9_]+|[\u4e00-\u9fff]", text) + return max(1, len(parts)) if text.strip() else 0 + + +def random_choices(arr: list[str], k: int) -> list[str]: + if not arr: + return [] + return random.choices(arr, k=min(len(arr), k)) + + +def is_english(texts: str | list[str]) -> bool: + if not texts: + return False + + patt = re.compile(r"[`a-zA-Z0-9\s.,':;/\"?<>!\(\)\-]+") + if isinstance(texts, str): + seq = [texts] + else: + seq = [t for t in texts if isinstance(t, str) and t.strip()] + + if not seq: + return False + + hits = sum(1 for t in seq if patt.fullmatch(t.strip())) + return (hits / len(seq)) > 0.8 + + +def not_bullet(line: str) -> bool: + patt = [ + r"0", + r"[0-9]+ +[0-9~个只-]", + r"[0-9]+\.{2,}", + ] + return any(re.match(p, line) for p in patt) + + +def is_probable_heading_line(line: str) -> bool: + text = (line or "").strip() + if not text: + return False + + if re.match(r"^#{1,6}\s+\S", text): + return True + + # 表格/HTML 残留通常不是标题。 + if re.search(r"]*>", text, flags=re.IGNORECASE): + return False + + # 超长行基本是正文或条款，不是章节标题。 + if len(text) > 96: + return False + if count_tokens(text) > 72: + return False + + # 标题前段通常不会出现明显句号/逗号；出现则大概率是正文。 + if re.search(r"[，。；！？!?:：]", text[:24]): + return False + + if text.endswith(("。", "；", "！", "!", "？", "?")) and len(text) > 20: + return False + + return True + + +def _is_mid_sentence_bullet(line: str) -> bool: + text = (line or "").strip() + if not text: + return False + + if re.match(r"^#{1,6}\s+\S", text): + return False + + marker = re.search( + r"([一二三四五六七八九十百]+、|[\(（][一二三四五六七八九十百]+[\)）]|[0-9]{1,2}[\.、])", + text, + ) + if not marker: + return False + + if marker.start() == 0: + return False + + prev = text[marker.start() - 1] + return prev not in {"#", "\n"} + + +def bullets_category(sections: list[str]) -> int: + hits: list[float] = [0.0] * len(BULLET_PATTERN) + + def bullet_weight(group_idx: int, line: str) -> float: + # 对 markdown 标题候选增加权重，避免“正文里的 一、/（一）”压过真正的 # 标题层级。 + if group_idx != MARKDOWN_BULLET_GROUP_INDEX: + return 1.0 + + heading = line.strip() + if not re.match(r"^#{1,6}\s+\S", heading): + return 1.0 + + level = len(heading) - len(heading.lstrip("#")) + if level <= 2: + return 4.0 + if level <= 4: + return 3.0 + return 2.0 + + for i, pro in enumerate(BULLET_PATTERN): + for sec in sections: + sec = sec.strip() + for p in pro: + if re.match(p, sec) and not not_bullet(sec): + w = bullet_weight(i, sec) + if _is_mid_sentence_bullet(sec): + w *= 0.1 + if i != MARKDOWN_BULLET_GROUP_INDEX and not is_probable_heading_line(sec): + w *= 0.2 + hits[i] += w + break + maximum = 0 + res = -1 + for i, hit in enumerate(hits): + if hit <= maximum: + continue + res = i + maximum = hit + return res + + +def _get_text(section: str | tuple[str, str]) -> str: + if isinstance(section, str): + return section.strip() + return (section[0] or "").strip() + + +def remove_contents_table(sections: list[str] | list[tuple[str, str]], eng: bool = False) -> None: + i = 0 + while i < len(sections): + line = re.sub(r"( |　|\u3000)+", "", _get_text(sections[i]).split("@@")[0], flags=re.IGNORECASE) + if not re.match(r"(contents|目录|目次|tableofcontents|致谢|acknowledge)$", line, flags=re.IGNORECASE): + i += 1 + continue + + sections.pop(i) + if i >= len(sections): + break + + prefix = _get_text(sections[i])[:3] if not eng else " ".join(_get_text(sections[i]).split()[:2]) + while not prefix and i < len(sections): + sections.pop(i) + if i >= len(sections): + break + prefix = _get_text(sections[i])[:3] if not eng else " ".join(_get_text(sections[i]).split()[:2]) + + if i >= len(sections) or not prefix: + break + + sections.pop(i) + if i >= len(sections): + break + + for j in range(i, min(i + 128, len(sections))): + if not re.match(re.escape(prefix), _get_text(sections[j])): + continue + for _ in range(i, j): + sections.pop(i) + break + + +def make_colon_as_title(sections: list[str] | list[tuple[str, str]]) -> list[str] | list[tuple[str, str]]: + if not sections: + return sections + if isinstance(sections[0], str): + return sections + + i = 0 + while i < len(sections): + text, layout = sections[i] + i += 1 + text = text.split("@")[0].strip() + if not text or text[-1] not in ":：": + continue + + rev = text[::-1] + arr = re.split(r"([。？！!?;；]| \.)", rev) + if len(arr) < 2 or len(arr[1]) < 32: + continue + + sections.insert(i - 1, (arr[0][::-1], "title")) + i += 1 + + return sections + + +def not_title(text: str) -> bool: + if re.match(r"第[零一二三四五六七八九十百0-9]+条", text): + return False + if len(text.split()) > 12 or (" " not in text and len(text) >= 32): + return True + return bool(re.search(r"[,;，。；！!]", text)) + + +def tree_merge(bull: int, sections: list[str] | list[tuple[str, str]], depth: int) -> list[str]: + if not sections or bull < 0: + return [s if isinstance(s, str) else s[0] for s in sections] + + if isinstance(sections[0], str): + typed_sections: list[tuple[str, str]] = [(s, "") for s in sections] + else: + typed_sections = sections # type: ignore[assignment] + + typed_sections = [ + (t, o) + for t, o in typed_sections + if t and len(t.split("@")[0].strip()) > 1 and not re.match(r"[0-9]+$", t.split("@")[0].strip()) + ] + + def get_level(section: tuple[str, str]) -> tuple[int, str]: + text, layout = section + text = re.sub(r"\u3000", " ", text).strip() + + for i, patt in enumerate(BULLET_PATTERN[bull]): + if re.match(patt, text) and is_probable_heading_line(text): + return i + 1, text + + if re.search(r"(title|head)", layout) and not not_title(text): + return len(BULLET_PATTERN[bull]) + 1, text + + return len(BULLET_PATTERN[bull]) + 2, text + + lines: list[tuple[int, str]] = [] + level_set: set[int] = set() + for section in typed_sections: + level, text = get_level(section) + if not text.strip("\n"): + continue + lines.append((level, text)) + level_set.add(level) + + if not lines: + return [] + + sorted_levels = sorted(level_set) + target_level = sorted_levels[depth - 1] if depth <= len(sorted_levels) else sorted_levels[-1] + + max_body_level = len(BULLET_PATTERN[bull]) + 2 + if target_level == max_body_level: + target_level = sorted_levels[-2] if len(sorted_levels) > 1 else sorted_levels[0] + + root = Node(level=0, depth=target_level, texts=[]) + root.build_tree(lines) + return [item for item in root.get_tree() if item] + + +def hierarchical_merge(bull: int, sections: list[str] | list[tuple[str, str]], depth: int) -> list[list[str]]: + if not sections or bull < 0: + return [] + + if isinstance(sections[0], str): + typed_sections: list[tuple[str, str]] = [(s, "") for s in sections] + else: + typed_sections = sections # type: ignore[assignment] + + typed_sections = [ + (t, o) + for t, o in typed_sections + if t and len(t.split("@")[0].strip()) > 1 and not re.match(r"[0-9]+$", t.split("@")[0].strip()) + ] + + bullets_size = len(BULLET_PATTERN[bull]) + levels: list[list[int]] = [[] for _ in range(bullets_size + 2)] + + for i, (text, layout) in enumerate(typed_sections): + for j, patt in enumerate(BULLET_PATTERN[bull]): + if re.match(patt, text.strip()) and is_probable_heading_line(text): + levels[j].append(i) + break + else: + if re.search(r"(title|head)", layout) and not not_title(text): + levels[bullets_size].append(i) + else: + levels[bullets_size + 1].append(i) + + pure_sections = [t for t, _ in typed_sections] + + def binary_search(arr: list[int], target: int) -> int: + if not arr: + return -1 + if target > arr[-1]: + return len(arr) - 1 + if target < arr[0]: + return -1 + + s, e = 0, len(arr) + while e - s > 1: + mid = (e + s) // 2 + if target > arr[mid]: + s = mid + elif target < arr[mid]: + e = mid + else: + return mid + return s + + cks: list[list[int]] = [] + readed = [False] * len(pure_sections) + levels = list(reversed(levels)) + for i, arr in enumerate(levels[:depth]): + for j in arr: + if readed[j]: + continue + readed[j] = True + cks.append([j]) + if i + 1 == len(levels) - 1: + continue + + for ii in range(i + 1, len(levels)): + jj = binary_search(levels[ii], j) + if jj < 0: + continue + if levels[ii][jj] > cks[-1][-1]: + cks[-1].pop(-1) + cks[-1].append(levels[ii][jj]) + + for ii in cks[-1]: + readed[ii] = True + + if not cks: + return [] + + for i in range(len(cks)): + cks[i] = [pure_sections[j] for j in reversed(cks[i])] + + res: list[list[str]] = [[]] + num = [0] + for ck in cks: + if len(ck) == 1: + n = count_tokens(re.sub(r"@@[0-9]+.*", "", ck[0])) + if n + num[-1] < 218: + res[-1].append(ck[0]) + num[-1] += n + continue + res.append(ck) + num.append(n) + continue + res.append(ck) + num.append(218) + + return [chunk for chunk in res if chunk] + + +def _remove_pdf_tags(text: str) -> str: + return re.sub(r"@@[0-9-]+\t[0-9.\t]+##", "", text or "") + + +def _extract_custom_delimiters(delimiter: str) -> list[str]: + return [m.group(1) for m in re.finditer(r"`([^`]+)`", delimiter or "")] + + +def naive_merge( + sections: str | list[str] | list[tuple[str, str]], + chunk_token_num: int = 128, + delimiter: str = "\n。；！？", + overlapped_percent: int = 0, +) -> list[str]: + if not sections: + return [] + + if isinstance(sections, str): + typed_sections: list[tuple[str, str]] = [(sections, "")] + elif isinstance(sections[0], str): + typed_sections = [(s, "") for s in sections] # type: ignore[index] + else: + typed_sections = sections # type: ignore[assignment] + + chunk_token_num = max(int(chunk_token_num or 0), 0) + overlap = max(0, min(int(overlapped_percent or 0), 99)) + + custom_delimiters = _extract_custom_delimiters(delimiter) + if custom_delimiters: + pattern = "|".join(re.escape(t) for t in sorted(set(custom_delimiters), key=len, reverse=True)) + chunks: list[str] = [] + for sec, pos in typed_sections: + split_sec = re.split(rf"({pattern})", sec, flags=re.DOTALL) + for sub in split_sec: + if re.fullmatch(pattern, sub or ""): + continue + text = "\n" + sub + local_pos = pos if count_tokens(text) >= 8 else "" + if local_pos and local_pos not in text: + text += local_pos + if text.strip(): + chunks.append(text) + return chunks + + if chunk_token_num <= 0: + merged = "\n".join(sec for sec, _ in typed_sections if sec and sec.strip()) + return [merged] if merged.strip() else [] + + chunks = [""] + token_nums = [0] + + def add_chunk(text: str, pos: str) -> None: + tnum = count_tokens(text) + local_pos = pos or "" + if tnum < 8: + local_pos = "" + + threshold = chunk_token_num * (100 - overlap) / 100.0 + if chunks[-1] == "" or token_nums[-1] > threshold: + if chunks: + prev = _remove_pdf_tags(chunks[-1]) + start = int(len(prev) * (100 - overlap) / 100.0) + text = prev[start:] + text + if local_pos and local_pos not in text: + text += local_pos + chunks.append(text) + token_nums.append(tnum) + else: + if local_pos and local_pos not in chunks[-1]: + text += local_pos + chunks[-1] += text + token_nums[-1] += tnum + + for sec, pos in typed_sections: + if not sec: + continue + add_chunk("\n" + sec, pos) + + return [chunk for chunk in chunks if chunk.strip()] + + +@dataclass +class Node: + level: int + depth: int = -1 + texts: list[str] = field(default_factory=list) + children: list["Node"] = field(default_factory=list) + + def add_child(self, child_node: "Node") -> None: + self.children.append(child_node) + + def add_text(self, text: str) -> None: + self.texts.append(text) + + def build_tree(self, lines: list[tuple[int, str]]) -> "Node": + stack: list[Node] = [self] + for level, text in lines: + if self.depth != -1 and level > self.depth: + stack[-1].add_text(text) + continue + + while len(stack) > 1 and level <= stack[-1].level: + stack.pop() + + node = Node(level=level, texts=[text]) + stack[-1].add_child(node) + stack.append(node) + + return self + + def get_tree(self) -> list[str]: + tree_list: list[str] = [] + self._dfs(self, tree_list, []) + return tree_list + + def _dfs(self, node: "Node", tree_list: list[str], titles: list[str]) -> None: + level = node.level + texts = node.texts + child = node.children + + if level == 0 and texts: + tree_list.append("\n".join(titles + texts)) + + path_titles = titles + texts if 1 <= level <= self.depth else titles + + if level > self.depth and texts: + tree_list.append("\n".join(path_titles + texts)) + elif not child and (1 <= level <= self.depth): + tree_list.append("\n".join(path_titles)) + + for c in child: + self._dfs(c, tree_list, path_titles) diff --git a/src/knowledge/chunking/ragflow_like/parsers/__init__.py b/src/knowledge/chunking/ragflow_like/parsers/__init__.py new file mode 100644 index 00000000..9f250470 --- /dev/null +++ b/src/knowledge/chunking/ragflow_like/parsers/__init__.py @@ -0,0 +1,3 @@ +from src.knowledge.chunking.ragflow_like.parsers import book, general, laws, qa + +__all__ = ["general", "qa", "book", "laws"] diff --git a/src/knowledge/chunking/ragflow_like/parsers/book.py b/src/knowledge/chunking/ragflow_like/parsers/book.py new file mode 100644 index 00000000..389e8ca5 --- /dev/null +++ b/src/knowledge/chunking/ragflow_like/parsers/book.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +from typing import Any + +from src.knowledge.chunking.ragflow_like import nlp + + +def _unescape_delimiter(delimiter: str) -> str: + return delimiter.replace("\\n", "\n").replace("\\r", "\r").replace("\\t", "\t").replace("\\\\", "\\") + + +def _iter_sections(markdown_content: str) -> list[tuple[str, str]]: + sections: list[tuple[str, str]] = [] + for line in (markdown_content or "").splitlines(): + text = line.strip() + if not text: + continue + sections.append((text, "")) + + if not sections and markdown_content and markdown_content.strip(): + sections.append((markdown_content.strip(), "")) + + return sections + + +def chunk_markdown(markdown_content: str, parser_config: dict[str, Any] | None = None) -> list[str]: + parser_config = parser_config or {} + + delimiter = _unescape_delimiter(str(parser_config.get("delimiter", "\n") or "\n")) + chunk_token_num = int(parser_config.get("chunk_token_num", 512) or 512) + overlapped_percent = int(parser_config.get("overlapped_percent", 0) or 0) + + sections = _iter_sections(markdown_content) + if not sections: + return [] + + section_texts = [text for text, _ in sections] + nlp.remove_contents_table(sections, eng=nlp.is_english(nlp.random_choices(section_texts, k=200))) + nlp.make_colon_as_title(sections) + + bull = nlp.bullets_category([t for t in nlp.random_choices([t for t, _ in sections], k=100)]) + + if bull >= 0: + chunks = ["\n".join(ck) for ck in nlp.hierarchical_merge(bull, sections, depth=5)] + else: + chunks = nlp.naive_merge( + sections, + chunk_token_num=chunk_token_num, + delimiter=delimiter, + overlapped_percent=overlapped_percent, + ) + + if chunks: + return chunks + + return nlp.naive_merge( + sections, + chunk_token_num=chunk_token_num, + delimiter=delimiter, + overlapped_percent=overlapped_percent, + ) diff --git a/src/knowledge/chunking/ragflow_like/parsers/general.py b/src/knowledge/chunking/ragflow_like/parsers/general.py new file mode 100644 index 00000000..0ed01d63 --- /dev/null +++ b/src/knowledge/chunking/ragflow_like/parsers/general.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +from typing import Any + +from src.knowledge.chunking.ragflow_like import nlp + + +def _unescape_delimiter(delimiter: str) -> str: + return delimiter.replace("\\n", "\n").replace("\\r", "\r").replace("\\t", "\t").replace("\\\\", "\\") + + +def _iter_sections(markdown_content: str, delimiter: str) -> list[tuple[str, str]]: + sections: list[tuple[str, str]] = [] + text = markdown_content or "" + if delimiter and delimiter not in {"\n", "\r\n"} and "`" not in delimiter: + for part in text.split(delimiter): + block = part.strip() + if block: + sections.append((block, "")) + else: + for line in text.splitlines(): + block = line.strip() + if not block: + continue + sections.append((block, "")) + + if not sections and text.strip(): + sections.append((text.strip(), "")) + + return sections + + +def chunk_markdown(markdown_content: str, parser_config: dict[str, Any] | None = None) -> list[str]: + parser_config = parser_config or {} + + delimiter = _unescape_delimiter(str(parser_config.get("delimiter", "\n") or "\n")) + chunk_token_num = int(parser_config.get("chunk_token_num", 512) or 512) + overlapped_percent = int(parser_config.get("overlapped_percent", 0) or 0) + + sections = _iter_sections(markdown_content, delimiter) + return nlp.naive_merge( + sections, + chunk_token_num=chunk_token_num, + delimiter=delimiter, + overlapped_percent=overlapped_percent, + ) diff --git a/src/knowledge/chunking/ragflow_like/parsers/laws.py b/src/knowledge/chunking/ragflow_like/parsers/laws.py new file mode 100644 index 00000000..513ff7ae --- /dev/null +++ b/src/knowledge/chunking/ragflow_like/parsers/laws.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +import re +from typing import Any + +from src.knowledge.chunking.ragflow_like import nlp + + +def _unescape_delimiter(delimiter: str) -> str: + return delimiter.replace("\\n", "\n").replace("\\r", "\r").replace("\\t", "\t").replace("\\\\", "\\") + + +def _iter_lines(markdown_content: str) -> list[str]: + return [line.strip() for line in (markdown_content or "").splitlines() if line.strip()] + + +def _docx_heading_tree(markdown_content: str) -> list[str]: + lines: list[tuple[int, str]] = [] + level_set: set[int] = set() + + for raw in (markdown_content or "").splitlines(): + text = raw.strip() + if not text: + continue + + heading_match = re.match(r"^(#{1,6})\s+(.*)$", text) + if heading_match: + level = len(heading_match.group(1)) + value = heading_match.group(2).strip() + else: + level = 99 + value = text + + if not value: + continue + + lines.append((level, value)) + level_set.add(level) + + if not lines: + return [] + + sorted_levels = sorted(level_set) + h2_level = sorted_levels[1] if len(sorted_levels) > 1 else 1 + h2_level = sorted_levels[-2] if h2_level == sorted_levels[-1] and len(sorted_levels) > 2 else h2_level + + root = nlp.Node(level=0, depth=h2_level, texts=[]) + root.build_tree(lines) + return [element for element in root.get_tree() if element] + + +def chunk_markdown(filename: str, markdown_content: str, parser_config: dict[str, Any] | None = None) -> list[str]: + parser_config = parser_config or {} + + delimiter = _unescape_delimiter(str(parser_config.get("delimiter", "\n") or "\n")) + chunk_token_num = int(parser_config.get("chunk_token_num", 512) or 512) + overlapped_percent = int(parser_config.get("overlapped_percent", 0) or 0) + + if re.search(r"\.docx$", filename or "", re.IGNORECASE): + chunks = _docx_heading_tree(markdown_content) + if chunks: + return chunks + + sections = _iter_lines(markdown_content) + if not sections: + return [] + + eng = nlp.is_english(sections) + nlp.remove_contents_table(sections, eng=eng) + + typed_sections = [(s, "") for s in sections] + nlp.make_colon_as_title(typed_sections) + + bull = nlp.bullets_category([s for s, _ in typed_sections]) + merged = nlp.tree_merge(bull, typed_sections, depth=2) + + if merged: + return merged + + return nlp.naive_merge( + typed_sections, + chunk_token_num=chunk_token_num, + delimiter=delimiter, + overlapped_percent=overlapped_percent, + ) diff --git a/src/knowledge/chunking/ragflow_like/parsers/qa.py b/src/knowledge/chunking/ragflow_like/parsers/qa.py new file mode 100644 index 00000000..cdfb4272 --- /dev/null +++ b/src/knowledge/chunking/ragflow_like/parsers/qa.py @@ -0,0 +1,261 @@ +from __future__ import annotations + +import csv +import re +from io import StringIO +from typing import Any + + +def _rm_prefix(text: str) -> str: + return re.sub( + r"^(问题|答案|回答|user|assistant|Q|A|Question|Answer|问|答)[\t:： ]+", + "", + (text or "").strip(), + flags=re.IGNORECASE, + ) + + +def _to_qa_chunk(question: str, answer: str, eng: bool = False) -> str: + qprefix = "Question: " if eng else "问题：" + aprefix = "Answer: " if eng else "回答：" + return "\t".join([qprefix + _rm_prefix(question), aprefix + _rm_prefix(answer)]) + + +def _guess_delimiter(lines: list[str]) -> str: + comma = 0 + tab = 0 + for line in lines: + if len(line.split(",")) == 2: + comma += 1 + if len(line.split("\t")) == 2: + tab += 1 + return "\t" if tab >= comma else "," + + +def _extract_pairs_with_delimiter(lines: list[str], delimiter: str) -> list[tuple[str, str]]: + pairs: list[tuple[str, str]] = [] + question = "" + answer = "" + + for line in lines: + arr = line.split(delimiter) + if len(arr) != 2: + if question: + answer += "\n" + line + continue + + if question and answer: + pairs.append((question, answer)) + question, answer = arr + + if question: + pairs.append((question, answer)) + + return [(q.strip(), a.strip()) for q, a in pairs if q.strip()] + + +def _extract_pairs_from_csv(lines: list[str], delimiter: str) -> list[tuple[str, str]]: + pairs: list[tuple[str, str]] = [] + question = "" + answer = "" + + reader = csv.reader(lines, delimiter=delimiter) + for row, raw_line in zip(reader, lines, strict=False): + if len(row) != 2: + if question: + answer += "\n" + raw_line + continue + + if question and answer: + pairs.append((question, answer)) + question, answer = row + + if question: + pairs.append((question, answer)) + + return [(q.strip(), a.strip()) for q, a in pairs if q.strip()] + + +def _parse_markdown_table_row(line: str) -> list[str] | None: + if "|" not in line: + return None + + text = line.strip() + if not text: + return None + + if text.startswith("|"): + text = text[1:] + if text.endswith("|"): + text = text[:-1] + + cells = [cell.strip() for cell in text.split("|")] + if not cells: + return None + + if all(re.fullmatch(r":?-{3,}:?", c.replace(" ", "")) for c in cells if c): + return None + + return cells + + +def _extract_pairs_from_markdown_tables(markdown_content: str) -> list[tuple[str, str]]: + pairs: list[tuple[str, str]] = [] + + for line in (markdown_content or "").splitlines(): + cells = _parse_markdown_table_row(line) + if not cells or len(cells) < 2: + continue + + question = cells[0] + answer = cells[1] + if question and answer: + pairs.append((question, answer)) + + return pairs + + +def _md_question_level(line: str) -> tuple[int, str]: + match = re.match(r"^#*", line) + if not match: + return 0, line + return len(match.group(0)), line.lstrip("#").lstrip() + + +def _extract_pairs_from_markdown_headings(markdown_content: str) -> list[tuple[str, str]]: + lines = (markdown_content or "").splitlines() + if not lines: + return [] + + pairs: list[tuple[str, str]] = [] + last_answer = "" + question_stack: list[str] = [] + level_stack: list[int] = [] + code_block = False + + for line in lines: + if line.strip().startswith("```"): + code_block = not code_block + + question_level = 0 + question = "" + if not code_block: + question_level, question = _md_question_level(line) + + if not question_level or question_level > 6: + last_answer = f"{last_answer}\n{line}" + continue + + if last_answer.strip(): + sum_question = "\n".join(question_stack) + if sum_question: + pairs.append((sum_question, last_answer.strip())) + last_answer = "" + + while question_stack and question_level <= level_stack[-1]: + question_stack.pop() + level_stack.pop() + + question_stack.append(question) + level_stack.append(question_level) + + if last_answer.strip(): + sum_question = "\n".join(question_stack) + if sum_question: + pairs.append((sum_question, last_answer.strip())) + + return pairs + + +def _extract_pairs_by_prefix(lines: list[str]) -> list[tuple[str, str]]: + pairs: list[tuple[str, str]] = [] + question = "" + answer_lines: list[str] = [] + + for line in lines: + if re.match(r"^(Q|Question|问|问题)\s*[:：]", line, flags=re.IGNORECASE): + if question: + pairs.append((question, "\n".join(answer_lines))) + question = re.sub(r"^(Q|Question|问|问题)\s*[:：]", "", line, flags=re.IGNORECASE).strip() + answer_lines = [] + continue + + if re.match(r"^(A|Answer|答|回答)\s*[:：]", line, flags=re.IGNORECASE): + answer_lines.append(re.sub(r"^(A|Answer|答|回答)\s*[:：]", "", line, flags=re.IGNORECASE).strip()) + continue + + if question: + answer_lines.append(line) + + if question: + pairs.append((question, "\n".join(answer_lines))) + + return [(q.strip(), a.strip()) for q, a in pairs if q.strip() and a.strip()] + + +def _dedupe_pairs(pairs: list[tuple[str, str]]) -> list[tuple[str, str]]: + res: list[tuple[str, str]] = [] + seen: set[tuple[str, str]] = set() + + for question, answer in pairs: + q = question.strip() + a = answer.strip() + if not q or not a: + continue + key = (q, a) + if key in seen: + continue + seen.add(key) + res.append((q, a)) + + return res + + +def chunk_markdown(filename: str, markdown_content: str, parser_config: dict[str, Any] | None = None) -> list[str]: + parser_config = parser_config or {} + eng = str(parser_config.get("language", "Chinese")).lower() == "english" + + suffix = "" + if filename and "." in filename: + suffix = "." + filename.lower().split(".")[-1] + + lines = [line for line in (markdown_content or "").splitlines() if line.strip()] + pairs: list[tuple[str, str]] = [] + + if suffix in {".xlsx", ".xls"}: + pairs.extend(_extract_pairs_from_markdown_tables(markdown_content)) + if not pairs: + delimiter = _guess_delimiter(lines) + pairs.extend(_extract_pairs_with_delimiter(lines, delimiter)) + elif suffix == ".csv": + pairs.extend(_extract_pairs_from_markdown_tables(markdown_content)) + delimiter = "\t" if any("\t" in line for line in lines) else "," + pairs.extend(_extract_pairs_from_csv(lines, delimiter)) + elif suffix == ".txt": + delimiter = _guess_delimiter(lines) + pairs.extend(_extract_pairs_with_delimiter(lines, delimiter)) + elif suffix in {".md", ".markdown", ".mdx"}: + pairs.extend(_extract_pairs_from_markdown_headings(markdown_content)) + pairs.extend(_extract_pairs_from_markdown_tables(markdown_content)) + elif suffix == ".docx": + pairs.extend(_extract_pairs_from_markdown_headings(markdown_content)) + pairs.extend(_extract_pairs_from_markdown_tables(markdown_content)) + else: + pairs.extend(_extract_pairs_from_markdown_headings(markdown_content)) + pairs.extend(_extract_pairs_from_markdown_tables(markdown_content)) + pairs.extend(_extract_pairs_by_prefix(lines)) + if not pairs: + delimiter = _guess_delimiter(lines) + pairs.extend(_extract_pairs_with_delimiter(lines, delimiter)) + + pairs = _dedupe_pairs(pairs) + + if not pairs and lines: + # 最后兜底：把内容按 2 行一组构成问答 + for i in range(0, len(lines), 2): + q = lines[i] + a = lines[i + 1] if i + 1 < len(lines) else "" + if q.strip() and a.strip(): + pairs.append((q, a)) + + return [_to_qa_chunk(q, a, eng=eng) for q, a in pairs] diff --git a/src/knowledge/chunking/ragflow_like/presets.py b/src/knowledge/chunking/ragflow_like/presets.py new file mode 100644 index 00000000..53dfe917 --- /dev/null +++ b/src/knowledge/chunking/ragflow_like/presets.py @@ -0,0 +1,235 @@ +from __future__ import annotations + +from copy import deepcopy +from typing import Any + +from src.utils import logger + +CHUNK_PRESET_GENERAL = "general" +CHUNK_PRESET_QA = "qa" +CHUNK_PRESET_BOOK = "book" +CHUNK_PRESET_LAWS = "laws" + +CHUNK_PRESET_IDS = { + CHUNK_PRESET_GENERAL, + CHUNK_PRESET_QA, + CHUNK_PRESET_BOOK, + CHUNK_PRESET_LAWS, +} + +CHUNK_PRESET_DESCRIPTIONS: dict[str, str] = { + CHUNK_PRESET_GENERAL: "通用分块：按分隔符和长度切分，适合大多数普通文档。", + CHUNK_PRESET_QA: "问答分块：优先抽取问题-回答结构，适合 FAQ、题库、问答手册。", + CHUNK_PRESET_BOOK: "书籍分块：强化章节标题识别并做层级合并，适合教材、手册、长章节文档。", + CHUNK_PRESET_LAWS: "法规分块：按法条层级组织与合并，适合法律法规、制度规范类文本。", +} + +CHUNK_ENGINE_VERSION = "ragflow_like_v1" +GENERAL_INTERNAL_PARSER_ID = "naive" + +_BASE_DEFAULTS: dict[str, Any] = { + "table_context_size": 0, + "image_context_size": 0, +} + +_PRESET_DEFAULTS: dict[str, dict[str, Any] | None] = { + CHUNK_PRESET_GENERAL: { + "layout_recognize": "DeepDOC", + "chunk_token_num": 512, + "delimiter": "\n", + "auto_keywords": 0, + "auto_questions": 0, + "html4excel": False, + "topn_tags": 3, + "raptor": { + "use_raptor": True, + "prompt": "Please summarize the following paragraphs. Be careful with the numbers, do not make things up. Paragraphs as following:\n {cluster_content}\nThe above is the content you need to summarize.", + "max_token": 256, + "threshold": 0.1, + "max_cluster": 64, + "random_seed": 0, + }, + "graphrag": { + "use_graphrag": True, + "entity_types": ["organization", "person", "geo", "event", "category"], + "method": "light", + }, + }, + CHUNK_PRESET_QA: {"raptor": {"use_raptor": False}, "graphrag": {"use_graphrag": False}}, + CHUNK_PRESET_BOOK: {"raptor": {"use_raptor": False}, "graphrag": {"use_graphrag": False}}, + CHUNK_PRESET_LAWS: {"raptor": {"use_raptor": False}, "graphrag": {"use_graphrag": False}}, +} + + +def deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]: + result = deepcopy(base) + for key, value in (override or {}).items(): + if isinstance(value, dict) and isinstance(result.get(key), dict): + result[key] = deep_merge(result[key], value) + else: + result[key] = value + return result + + +def normalize_chunk_preset_id(value: str | None) -> str: + if not value: + return CHUNK_PRESET_GENERAL + + normalized = str(value).strip().lower() + if normalized == GENERAL_INTERNAL_PARSER_ID: + return CHUNK_PRESET_GENERAL + + if normalized in CHUNK_PRESET_IDS: + return normalized + + logger.warning(f"Unknown chunk preset id '{value}', fallback to general") + return CHUNK_PRESET_GENERAL + + +def map_to_internal_parser_id(preset_id: str) -> str: + normalized = normalize_chunk_preset_id(preset_id) + if normalized == CHUNK_PRESET_GENERAL: + return GENERAL_INTERNAL_PARSER_ID + return normalized + + +def get_default_chunk_parser_config(preset_id: str) -> dict[str, Any]: + normalized = normalize_chunk_preset_id(preset_id) + default_config = deepcopy(_PRESET_DEFAULTS.get(normalized) or {}) + return deep_merge(_BASE_DEFAULTS, default_config) + + +def _safe_int(value: Any) -> int | None: + if value is None: + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _legacy_params_to_parser_config(params: dict[str, Any] | None) -> dict[str, Any]: + if not isinstance(params, dict): + return {} + + parser_config: dict[str, Any] = {} + + chunk_size = _safe_int(params.get("chunk_size")) + chunk_overlap = _safe_int(params.get("chunk_overlap")) + + if chunk_size and chunk_size > 0: + parser_config["chunk_token_num"] = chunk_size + if chunk_size and chunk_size > 0 and chunk_overlap is not None: + overlap_percent = round(max(0, min(chunk_overlap, chunk_size - 1)) * 100 / chunk_size) + parser_config["overlapped_percent"] = max(0, min(overlap_percent, 99)) + + if isinstance(params.get("qa_separator"), str) and params.get("qa_separator"): + parser_config["delimiter"] = params["qa_separator"] + + if isinstance(params.get("delimiter"), str) and params.get("delimiter"): + parser_config["delimiter"] = params["delimiter"] + + if "chunk_token_num" in params: + normalized_chunk_token_num = _safe_int(params.get("chunk_token_num")) + if normalized_chunk_token_num is not None: + parser_config["chunk_token_num"] = normalized_chunk_token_num + + if "overlapped_percent" in params: + normalized_overlapped_percent = _safe_int(params.get("overlapped_percent")) + if normalized_overlapped_percent is not None: + parser_config["overlapped_percent"] = max(0, min(normalized_overlapped_percent, 99)) + + return parser_config + + +def ensure_chunk_defaults_in_additional_params(additional_params: dict[str, Any] | None) -> dict[str, Any]: + params = dict(additional_params or {}) + params["chunk_preset_id"] = normalize_chunk_preset_id(params.get("chunk_preset_id")) + + if "chunk_parser_config" in params and not isinstance(params.get("chunk_parser_config"), dict): + logger.warning("Invalid chunk_parser_config in additional_params, fallback to empty dict") + params["chunk_parser_config"] = {} + + return params + + +def resolve_chunk_processing_params( + kb_additional_params: dict[str, Any] | None, + file_processing_params: dict[str, Any] | None, + request_params: dict[str, Any] | None = None, +) -> dict[str, Any]: + kb_additional = ensure_chunk_defaults_in_additional_params(kb_additional_params) + file_params = dict(file_processing_params or {}) + request = dict(request_params or {}) + + preset_id = normalize_chunk_preset_id( + request.get("chunk_preset_id") + or file_params.get("chunk_preset_id") + or kb_additional.get("chunk_preset_id") + ) + + parser_config = get_default_chunk_parser_config(preset_id) + + kb_parser_config = kb_additional.get("chunk_parser_config") + if isinstance(kb_parser_config, dict): + parser_config = deep_merge(parser_config, kb_parser_config) + + file_parser_config = file_params.get("chunk_parser_config") + if isinstance(file_parser_config, dict): + parser_config = deep_merge(parser_config, file_parser_config) + + req_parser_config = request.get("chunk_parser_config") + if isinstance(req_parser_config, dict): + parser_config = deep_merge(parser_config, req_parser_config) + + merged_legacy = {} + merged_legacy.update(file_params) + merged_legacy.update(request) + parser_config = deep_merge(parser_config, _legacy_params_to_parser_config(merged_legacy)) + + # Build processing params snapshot (keep existing + request overrides for non-chunk fields) + snapshot: dict[str, Any] = {} + snapshot.update(file_params) + snapshot.update(request) + snapshot["chunk_preset_id"] = preset_id + snapshot["chunk_parser_config"] = parser_config + snapshot["chunk_engine_version"] = CHUNK_ENGINE_VERSION + + # Keep backward-compatible fields for current UI + if "chunk_size" not in snapshot and isinstance(parser_config.get("chunk_token_num"), int): + snapshot["chunk_size"] = parser_config["chunk_token_num"] + + if "chunk_overlap" not in snapshot and isinstance(parser_config.get("overlapped_percent"), int): + token_num = parser_config.get("chunk_token_num") + if isinstance(token_num, int) and token_num > 0: + snapshot["chunk_overlap"] = int(token_num * parser_config["overlapped_percent"] / 100) + + if "qa_separator" not in snapshot and isinstance(parser_config.get("delimiter"), str): + snapshot["qa_separator"] = parser_config["delimiter"] + + return snapshot + + +def get_chunk_preset_options() -> list[dict[str, str]]: + return [ + { + "value": CHUNK_PRESET_GENERAL, + "label": "General", + "description": CHUNK_PRESET_DESCRIPTIONS[CHUNK_PRESET_GENERAL], + }, + { + "value": CHUNK_PRESET_QA, + "label": "QA", + "description": CHUNK_PRESET_DESCRIPTIONS[CHUNK_PRESET_QA], + }, + { + "value": CHUNK_PRESET_BOOK, + "label": "Book", + "description": CHUNK_PRESET_DESCRIPTIONS[CHUNK_PRESET_BOOK], + }, + { + "value": CHUNK_PRESET_LAWS, + "label": "Laws", + "description": CHUNK_PRESET_DESCRIPTIONS[CHUNK_PRESET_LAWS], + }, + ] diff --git a/src/knowledge/implementations/lightrag.py b/src/knowledge/implementations/lightrag.py index 0c73f307..e88e3ea1 100644 --- a/src/knowledge/implementations/lightrag.py +++ b/src/knowledge/implementations/lightrag.py @@ -11,6 +11,8 @@ from pymilvus import connections, utility from src import config from src.knowledge.base import FileStatus, KnowledgeBase +from src.knowledge.chunking.ragflow_like.dispatcher import chunk_markdown +from src.knowledge.chunking.ragflow_like.presets import resolve_chunk_processing_params from src.knowledge.indexing import process_file_to_markdown from src.knowledge.utils.kb_utils import get_embedding_config from src.utils import hashstr, logger @@ -40,6 +42,18 @@ class LightRagKB(KnowledgeBase): """知识库类型标识""" return "lightrag" + @staticmethod + def _prepare_lightrag_insert_payload(chunks: list[dict]) -> tuple[str, str | None, bool]: + if not chunks: + return "", None, False + + if len(chunks) == 1: + return chunks[0]["content"], None, False + + delimiter = "\n<|YUXI_CHUNK_DELIM|>\n" + payload = delimiter.join(chunk["content"] for chunk in chunks if chunk.get("content")) + return payload, delimiter, True + def delete_database(self, db_id: str) -> dict: """删除数据库，同时清除Milvus和Neo4j中的数据""" # Drop Milvus collection @@ -137,6 +151,21 @@ class LightRagKB(KnowledgeBase): await instance.initialize_storages() await initialize_pipeline_status() + @staticmethod + async def _ensure_doc_processed(rag: LightRAG, file_id: str) -> None: + """确保 LightRAG 文档处理成功，否则抛出异常。""" + status_doc = await rag.doc_status.get_by_id(file_id) + if not status_doc: + raise ValueError(f"LightRAG 文档状态缺失: {file_id}") + + status = status_doc.get("status") + status_value = status.value if hasattr(status, "value") else status + if status_value not in {"processed", "preprocessed"}: + error_msg = status_doc.get("error_msg") or "unknown error" + raise ValueError( + f"LightRAG 实体关系抽取失败: file_id={file_id}, status={status_value}, error={error_msg}" + ) + async def _get_lightrag_instance(self, db_id: str) -> LightRAG | None: """获取或创建 LightRAG 实例""" if db_id in self.instances: @@ -293,14 +322,36 @@ class LightRagKB(KnowledgeBase): # Read markdown markdown_content = await self._read_markdown_from_minio(file_meta["markdown_file"]) file_path = file_meta.get("path") + filename = file_meta.get("filename") or file_id + processing_params = resolve_chunk_processing_params( + kb_additional_params=self.databases_meta.get(db_id, {}).get("metadata"), + file_processing_params=file_meta.get("processing_params"), + ) + self.files_meta[file_id]["processing_params"] = processing_params + await self._save_metadata() + + chunks = chunk_markdown(markdown_content, file_id, filename, processing_params) + chunk_input, split_by_character, split_by_character_only = self._prepare_lightrag_insert_payload(chunks) + if not chunk_input: + chunk_input = markdown_content # Clean up existing chunks if any (for re-indexing) await self.delete_file_chunks_only(db_id, file_id) # Insert - await rag.ainsert(input=markdown_content, ids=file_id, file_paths=file_path) + await rag.ainsert( + input=chunk_input, + ids=file_id, + file_paths=file_path, + split_by_character=split_by_character, + split_by_character_only=split_by_character_only, + ) + await self._ensure_doc_processed(rag, file_id) - logger.info(f"Indexed file {file_id} into LightRAG") + logger.info( + f"Indexed file {file_id} into LightRAG with {len(chunks)} chunks, " + f"chunk_preset_id={processing_params.get('chunk_preset_id')}" + ) # Update status self.files_meta[file_id]["status"] = FileStatus.INDEXED @@ -358,7 +409,12 @@ class LightRagKB(KnowledgeBase): try: # 更新状态为处理中 - self.files_meta[file_id]["processing_params"] = params.copy() + resolved_params = resolve_chunk_processing_params( + kb_additional_params=self.databases_meta.get(db_id, {}).get("metadata"), + file_processing_params=self.files_meta[file_id].get("processing_params"), + request_params=params, + ) + self.files_meta[file_id]["processing_params"] = resolved_params self.files_meta[file_id]["status"] = "processing" await self._save_metadata() @@ -368,12 +424,24 @@ class LightRagKB(KnowledgeBase): markdown_content = await process_file_to_markdown(file_path, params=params) markdown_content_lines = markdown_content[:100].replace("\n", " ") logger.info(f"Markdown content: {markdown_content_lines}...") + filename = file_meta.get("filename") or file_id + chunks = chunk_markdown(markdown_content, file_id, filename, resolved_params) + chunk_input, split_by_character, split_by_character_only = self._prepare_lightrag_insert_payload(chunks) + if not chunk_input: + chunk_input = markdown_content # 先删除现有的 LightRAG 数据（仅删除chunks，保留元数据） await self.delete_file_chunks_only(db_id, file_id) # 使用 LightRAG 重新插入内容 - await rag.ainsert(input=markdown_content, ids=file_id, file_paths=file_path) + await rag.ainsert( + input=chunk_input, + ids=file_id, + file_paths=file_path, + split_by_character=split_by_character, + split_by_character_only=split_by_character_only, + ) + await self._ensure_doc_processed(rag, file_id) logger.info(f"Updated {content_type} {file_path} in LightRAG. Done.") diff --git a/src/knowledge/implementations/milvus.py b/src/knowledge/implementations/milvus.py index 1e01d14c..5f97ba40 100644 --- a/src/knowledge/implementations/milvus.py +++ b/src/knowledge/implementations/milvus.py @@ -10,11 +10,10 @@ from pymilvus import Collection, CollectionSchema, DataType, FieldSchema, connec from src import config from src.knowledge.base import FileStatus, KnowledgeBase +from src.knowledge.chunking.ragflow_like.dispatcher import chunk_markdown +from src.knowledge.chunking.ragflow_like.presets import resolve_chunk_processing_params from src.knowledge.indexing import process_file_to_markdown -from src.knowledge.utils.kb_utils import ( - get_embedding_config, - split_text_into_chunks, -) +from src.knowledge.utils.kb_utils import get_embedding_config from src.models.embed import OtherEmbedding from src.utils import hashstr, logger from src.utils.datetime_utils import utc_isoformat @@ -222,7 +221,7 @@ class MilvusKB(KnowledgeBase): def _split_text_into_chunks(self, text: str, file_id: str, filename: str, params: dict) -> list[dict]: """将文本分割成块""" - return split_text_into_chunks(text, file_id, filename, params) + return chunk_markdown(text, file_id, filename, params) async def index_file(self, db_id: str, file_id: str, operator_id: str | None = None) -> dict: """ @@ -281,10 +280,14 @@ class MilvusKB(KnowledgeBase): self.files_meta[file_id]["updated_at"] = utc_isoformat() if operator_id: self.files_meta[file_id]["updated_by"] = operator_id - await self._save_metadata() # Read processing params inside lock to ensure we get the latest values - params = file_meta.get("processing_params", {}) or {} + params = resolve_chunk_processing_params( + kb_additional_params=self.databases_meta.get(db_id, {}).get("metadata"), + file_processing_params=file_meta.get("processing_params"), + ) + self.files_meta[file_id]["processing_params"] = params + await self._save_metadata() logger.debug(f"[index_file] file_id={file_id}, processing_params={params}") # Add to processing queue @@ -299,6 +302,7 @@ class MilvusKB(KnowledgeBase): chunks = self._split_text_into_chunks(markdown_content, file_id, filename, params) logger.info( f"Split {filename} into {len(chunks)} chunks with params: " + f"chunk_preset_id={params.get('chunk_preset_id')}, " f"chunk_size={params.get('chunk_size')}, " f"chunk_overlap={params.get('chunk_overlap')}, " f"qa_separator={params.get('qa_separator')}" @@ -391,7 +395,12 @@ class MilvusKB(KnowledgeBase): try: # 更新状态为处理中 async with self._metadata_lock: - self.files_meta[file_id]["processing_params"] = params.copy() + resolved_params = resolve_chunk_processing_params( + kb_additional_params=self.databases_meta.get(db_id, {}).get("metadata"), + file_processing_params=self.files_meta[file_id].get("processing_params"), + request_params=params, + ) + self.files_meta[file_id]["processing_params"] = resolved_params self.files_meta[file_id]["status"] = "processing" await self._save_metadata() @@ -404,7 +413,7 @@ class MilvusKB(KnowledgeBase): await self.delete_file_chunks_only(db_id, file_id) # 重新生成 chunks - chunks = self._split_text_into_chunks(markdown_content, file_id, filename, params) + chunks = self._split_text_into_chunks(markdown_content, file_id, filename, resolved_params) logger.info(f"Split {filename} into {len(chunks)} chunks") if chunks: diff --git a/src/knowledge/manager.py b/src/knowledge/manager.py index 1f262bc9..a74192c4 100644 --- a/src/knowledge/manager.py +++ b/src/knowledge/manager.py @@ -2,6 +2,10 @@ import asyncio import os from src.knowledge.base import KBNotFoundError, KnowledgeBase +from src.knowledge.chunking.ragflow_like.presets import ( + deep_merge, + ensure_chunk_defaults_in_additional_params, +) from src.knowledge.factory import KnowledgeBaseFactory from src.utils import logger from src.utils.datetime_utils import utc_isoformat @@ -179,7 +183,7 @@ class KnowledgeBaseManager: if db_info: # 补充 share_config 和 additional_params db_info["share_config"] = row.share_config or {"is_shared": True, "accessible_departments": []} - db_info["additional_params"] = row.additional_params or {} + db_info["additional_params"] = ensure_chunk_defaults_in_additional_params(row.additional_params) all_databases.append(db_info) return {"databases": all_databases} @@ -310,6 +314,8 @@ class KnowledgeBaseManager: if share_config is None: share_config = {"is_shared": True, "accessible_departments": []} + kwargs = ensure_chunk_defaults_in_additional_params(kwargs) + kb_instance = self._get_or_create_kb_instance(kb_type) db_info = await kb_instance.create_database(database_name, description, embed_info, **kwargs) db_id = db_info["db_id"] @@ -414,7 +420,7 @@ class KnowledgeBaseManager: } # 添加数据库中的附加字段 - db_info["additional_params"] = kb.additional_params or {} + db_info["additional_params"] = ensure_chunk_defaults_in_additional_params(kb.additional_params) db_info["share_config"] = kb.share_config or {"is_shared": True, "accessible_departments": []} db_info["mindmap"] = kb.mindmap db_info["sample_questions"] = kb.sample_questions or [] @@ -566,6 +572,11 @@ class KnowledgeBaseManager: """更新数据库""" from src.repositories.knowledge_base_repository import KnowledgeBaseRepository + kb_repo = KnowledgeBaseRepository() + kb = await kb_repo.get_by_id(db_id) + if kb is None: + raise ValueError(f"数据库 {db_id} 不存在") + kb_instance = await self._get_kb_for_database(db_id) kb_instance.update_database(db_id, name, description, llm_info) @@ -576,13 +587,19 @@ class KnowledgeBaseManager: } if llm_info is not None: update_data["llm_info"] = llm_info + if additional_params is not None: - update_data["additional_params"] = additional_params + merged_additional_params = ensure_chunk_defaults_in_additional_params( + deep_merge(kb.additional_params or {}, additional_params) + ) + update_data["additional_params"] = merged_additional_params + if db_id in kb_instance.databases_meta: + kb_instance.databases_meta[db_id]["metadata"] = merged_additional_params + if share_config is not None: update_data["share_config"] = share_config # 保存到数据库 - kb_repo = KnowledgeBaseRepository() await kb_repo.update(db_id, update_data) return await self.get_database_info(db_id) diff --git a/test/api/test_knowledge_router.py b/test/api/test_knowledge_router.py index 29fa366f..51a41bba 100644 --- a/test/api/test_knowledge_router.py +++ b/test/api/test_knowledge_router.py @@ -40,6 +40,59 @@ async def test_admin_can_manage_knowledge_databases(test_client, admin_headers, assert update_response.json()["database"]["description"] == "Updated by pytest" +async def test_create_database_with_chunk_preset(test_client, admin_headers): + db_name = f"pytest_chunk_preset_{uuid.uuid4().hex[:6]}" + payload = { + "database_name": db_name, + "description": "Chunk preset create test", + "embed_model_name": "siliconflow/BAAI/bge-m3", + "kb_type": "milvus", + "additional_params": {"chunk_preset_id": "book"}, + } + + create_response = await test_client.post("/api/knowledge/databases", json=payload, headers=admin_headers) + assert create_response.status_code == 200, create_response.text + db_id = create_response.json()["db_id"] + + info_response = await test_client.get(f"/api/knowledge/databases/{db_id}", headers=admin_headers) + assert info_response.status_code == 200, info_response.text + assert info_response.json()["additional_params"]["chunk_preset_id"] == "book" + + delete_response = await test_client.delete(f"/api/knowledge/databases/{db_id}", headers=admin_headers) + assert delete_response.status_code == 200, delete_response.text + + +async def test_update_database_additional_params_merge_keeps_chunk_preset( + test_client, admin_headers, knowledge_database +): + db_id = knowledge_database["db_id"] + + first_update = await test_client.put( + f"/api/knowledge/databases/{db_id}", + json={ + "name": knowledge_database["name"], + "description": "update with chunk preset", + "additional_params": {"chunk_preset_id": "qa"}, + }, + headers=admin_headers, + ) + assert first_update.status_code == 200, first_update.text + + second_update = await test_client.put( + f"/api/knowledge/databases/{db_id}", + json={ + "name": knowledge_database["name"], + "description": "update without additional params", + }, + headers=admin_headers, + ) + assert second_update.status_code == 200, second_update.text + + info_response = await test_client.get(f"/api/knowledge/databases/{db_id}", headers=admin_headers) + assert info_response.status_code == 200, info_response.text + assert info_response.json()["additional_params"]["chunk_preset_id"] == "qa" + + async def test_knowledge_routes_enforce_permissions(test_client, standard_user, knowledge_database): db_id = knowledge_database["db_id"] diff --git a/test/test_ragflow_like_chunking.py b/test/test_ragflow_like_chunking.py new file mode 100644 index 00000000..ad270655 --- /dev/null +++ b/test/test_ragflow_like_chunking.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +import os +import sys + +sys.path.append(os.getcwd()) + +from src.knowledge.chunking.ragflow_like.dispatcher import chunk_markdown +from src.knowledge.chunking.ragflow_like.nlp import bullets_category +from src.knowledge.chunking.ragflow_like.presets import ( + CHUNK_ENGINE_VERSION, + get_chunk_preset_options, + map_to_internal_parser_id, + resolve_chunk_processing_params, +) + + +def test_general_maps_to_naive() -> None: + assert map_to_internal_parser_id("general") == "naive" + + +def test_resolve_chunk_processing_params_priority() -> None: + resolved = resolve_chunk_processing_params( + kb_additional_params={ + "chunk_preset_id": "book", + "chunk_parser_config": {"chunk_token_num": 300, "delimiter": "\\n"}, + }, + file_processing_params={ + "chunk_preset_id": "qa", + "chunk_parser_config": {"delimiter": "###"}, + }, + request_params={ + "chunk_preset_id": "laws", + "chunk_parser_config": {"chunk_token_num": 666}, + "chunk_size": 777, + }, + ) + + assert resolved["chunk_preset_id"] == "laws" + assert resolved["chunk_engine_version"] == CHUNK_ENGINE_VERSION + # legacy chunk_size 在当前实现里会映射为 chunk_token_num + assert resolved["chunk_parser_config"]["chunk_token_num"] == 777 + assert resolved["chunk_parser_config"]["delimiter"] == "###" + + +def test_qa_chunking_from_markdown_headings() -> None: + content = """ +# 问题一 +这是答案一。 + +## 子问题 +这是答案二。 +""".strip() + + chunks = chunk_markdown( + markdown_content=content, + file_id="file_1", + filename="faq.md", + processing_params={"chunk_preset_id": "qa", "chunk_parser_config": {}}, + ) + + assert len(chunks) >= 1 + assert "问题：" in chunks[0]["content"] + assert "回答：" in chunks[0]["content"] + + +def test_book_chunking_hierarchical_merge() -> None: + content = """ +第一章 总则 +第一节 适用范围 +本规范适用于测试场景。 +第二节 基本原则 +应当遵循最小改动原则。 +""".strip() + + chunks = chunk_markdown( + markdown_content=content, + file_id="file_2", + filename="book.txt", + processing_params={"chunk_preset_id": "book", "chunk_parser_config": {"chunk_token_num": 256}}, + ) + + assert len(chunks) >= 1 + assert any("第一章" in ck["content"] for ck in chunks) + + +def test_markdown_heading_has_higher_weight_in_bullet_category() -> None: + sections = [ + "# 3.2 个人所得项目及计税、申报方式概括", + "一、关于季节工、临时工等费用税前扣除问题，以下规定继续执行。", + "二、根据现行规定，补贴收入应并入工资薪金所得。", + "（一）从超出国家规定比例支付的补贴，不属于免税福利费。", + ] + + # 命中 markdown 标题模式（BULLET_PATTERN 下标 4）时，应该优先选中该组。 + assert bullets_category(sections) == 4 + + +def test_mid_sentence_bullet_marker_should_not_be_treated_as_heading() -> None: + sections = [ + "根据前述规则：一、这里是句中枚举，不是章节标题，不能被当成层级。", + "延续上文：（二）这里同样是正文中的枚举表达，不是独立标题。", + "## 3.4 交通补贴的个税处理", + ] + assert bullets_category(sections) == 4 + + +def test_chunk_preset_options_include_description() -> None: + options = get_chunk_preset_options() + assert len(options) == 4 + assert all(isinstance(option.get("description"), str) and option["description"] for option in options) diff --git a/web/src/components/ChunkParamsConfig.vue b/web/src/components/ChunkParamsConfig.vue index 9617e9c8..30e72be5 100644 --- a/web/src/components/ChunkParamsConfig.vue +++ b/web/src/components/ChunkParamsConfig.vue @@ -4,7 +4,27 @@

调整分块参数可以控制文本的切分方式，影响检索质量和文档加载效率。

-
+ + + +

+ 预设策略对齐 RAGFlow：General、QA、Book、Laws。 + 留空时沿用知识库默认策略。 +

+ + +
diff --git a/web/src/components/FileTable.vue b/web/src/components/FileTable.vue index c8645954..f2a84494 100644 --- a/web/src/components/FileTable.vue +++ b/web/src/components/FileTable.vue @@ -189,7 +189,14 @@ 确定
- +
@@ -704,8 +711,25 @@ const indexConfigModalTitle = ref('入库参数配置') const indexParams = ref({ chunk_size: 1000, chunk_overlap: 200, - qa_separator: '' + qa_separator: '', + chunk_preset_id: '' }) +const buildIndexParamsPayload = () => { + const payload = {} + if (indexParams.value.chunk_preset_id) { + payload.chunk_preset_id = indexParams.value.chunk_preset_id + } + + if (isLightRAG.value) { + payload.qa_separator = indexParams.value.qa_separator || '' + return payload + } + + return { + ...indexParams.value, + ...payload + } +} const currentIndexFileIds = ref([]) const isBatchIndexOperation = ref(false) @@ -1088,12 +1112,6 @@ const handleBatchIndex = async () => { return } - if (isLightRAG.value) { - await store.indexFiles(validKeys) - selectedRowKeys.value = [] - return - } - currentIndexFileIds.value = [...validKeys] isBatchIndexOperation.value = true indexConfigModalTitle.value = '批量入库参数配置' @@ -1172,11 +1190,6 @@ const handleParseFile = async (record) => { const handleIndexFile = async (record) => { closePopover(record.file_id) - if (isLightRAG.value) { - await store.indexFiles([record.file_id]) - return - } - // 打开参数配置弹窗 currentIndexFileIds.value = [record.file_id] isBatchIndexOperation.value = false @@ -1189,7 +1202,8 @@ const handleIndexFile = async (record) => { Object.assign(indexParams.value, { chunk_size: 1000, chunk_overlap: 200, - qa_separator: '' + qa_separator: '', + chunk_preset_id: '' }) } @@ -1214,7 +1228,7 @@ const handleReindexFile = async (record) => { const handleIndexConfigConfirm = async () => { try { // 调用 indexFiles 接口 (支持 params) - const result = await store.indexFiles(currentIndexFileIds.value, indexParams.value) + const result = await store.indexFiles(currentIndexFileIds.value, buildIndexParamsPayload()) if (result) { currentIndexFileIds.value = [] // 清空选择 @@ -1228,7 +1242,8 @@ const handleIndexConfigConfirm = async () => { Object.assign(indexParams.value, { chunk_size: 1000, chunk_overlap: 200, - qa_separator: '' + qa_separator: '', + chunk_preset_id: '' }) } else { // message.error(`入库失败: ${result.message}`); // store already shows message @@ -1249,7 +1264,8 @@ const handleIndexConfigCancel = () => { Object.assign(indexParams.value, { chunk_size: 1000, chunk_overlap: 200, - qa_separator: '' + qa_separator: '', + chunk_preset_id: '' }) } diff --git a/web/src/components/FileUploadModal.vue b/web/src/components/FileUploadModal.vue index ddd85e19..bf0e6f9c 100644 --- a/web/src/components/FileUploadModal.vue +++ b/web/src/components/FileUploadModal.vue @@ -108,15 +108,17 @@
入库参数配置
- - + +

+ LightRAG 按分隔符预切分，超长片段仍会按 token 大小继续切分。 +

@@ -564,9 +566,26 @@ const autoIndex = ref(false) const indexParams = ref({ chunk_size: 1000, chunk_overlap: 200, - qa_separator: '' + qa_separator: '', + chunk_preset_id: '' }) +const buildAutoIndexParams = () => { + const payload = {} + if (indexParams.value.chunk_preset_id) { + payload.chunk_preset_id = indexParams.value.chunk_preset_id + } + + if (isGraphBased.value) { + payload.qa_separator = indexParams.value.qa_separator || '' + return payload + } + return { + ...indexParams.value, + ...payload + } +} + // 计算属性：是否支持QA分割 const isQaSplitSupported = computed(() => { const type = kbType.value?.toLowerCase() @@ -990,7 +1009,7 @@ const chunkData = async () => { const params = { ...chunkParams.value } if (autoIndex.value) { params.auto_index = true - Object.assign(params, indexParams.value) + Object.assign(params, buildAutoIndexParams()) } // 构造 _preprocessed_map 和 items (minio urls) @@ -1068,7 +1087,7 @@ const chunkData = async () => { const params = { ...chunkParams.value, content_hashes } if (autoIndex.value) { params.auto_index = true - Object.assign(params, indexParams.value) + Object.assign(params, buildAutoIndexParams()) } await store.addFiles({ @@ -1604,17 +1623,6 @@ const chunkData = async () => { border-radius: 6px; } -.lightrag-tip { - display: flex; - align-items: center; - margin-top: 8px; - padding: 8px 12px; - background: var(--main-50); - border-radius: 6px; - font-size: 13px; - color: var(--gray-600); -} - .setting-label .ant-checkbox { margin-right: 8px; } diff --git a/web/src/components/KnowledgeBaseCard.vue b/web/src/components/KnowledgeBaseCard.vue index 67f2a733..833e516a 100644 --- a/web/src/components/KnowledgeBaseCard.vue +++ b/web/src/components/KnowledgeBaseCard.vue @@ -40,6 +40,9 @@ {{ getKbTypeLabel(database.kb_type || 'lightrag') }} {{ database.embed_info?.name || 'N/A' }} + {{ + chunkPresetLabelMap[database.additional_params?.chunk_preset_id || 'general'] || 'General' + }}
@@ -81,6 +84,18 @@ > + + + + + ({ label, value })) +const chunkPresetLabelMap = CHUNK_PRESET_LABEL_MAP +const editPresetDescription = computed(() => getChunkPresetDescription(editForm.chunk_preset_id)) + const rules = { name: [{ required: true, message: '请输入知识库名称' }] } @@ -238,6 +263,7 @@ const showEditModal = () => { editForm.description = database.value.description || '' editForm.auto_generate_questions = database.value.additional_params?.auto_generate_questions || false + editForm.chunk_preset_id = database.value.additional_params?.chunk_preset_id || 'general' // 如果是 LightRAG 类型，加载当前的 LLM 配置 if (database.value.kb_type === 'lightrag') { @@ -283,7 +309,8 @@ const handleEditSubmit = () => { name: editForm.name, description: editForm.description, additional_params: { - auto_generate_questions: editForm.auto_generate_questions + auto_generate_questions: editForm.auto_generate_questions, + chunk_preset_id: editForm.chunk_preset_id || 'general' }, share_config: { is_shared: finalIsShared, @@ -428,4 +455,16 @@ const deleteDatabase = () => { align-items: center; flex-wrap: wrap; } + +.chunk-preset-label { + display: inline-flex; + align-items: center; + gap: 6px; +} + +.chunk-preset-help-icon { + color: var(--gray-500); + cursor: help; + font-size: 14px; +} diff --git a/web/src/utils/chunk_presets.js b/web/src/utils/chunk_presets.js new file mode 100644 index 00000000..ebd6285f --- /dev/null +++ b/web/src/utils/chunk_presets.js @@ -0,0 +1,33 @@ +export const CHUNK_PRESET_OPTIONS = [ + { + value: 'general', + label: 'General', + description: '通用分块：按分隔符和长度切分，适合大多数普通文档。' + }, + { + value: 'qa', + label: 'QA', + description: '问答分块：优先抽取问题-回答结构，适合 FAQ、题库、问答手册。' + }, + { + value: 'book', + label: 'Book', + description: '书籍分块：强化章节标题识别并做层级合并，适合教材、手册、长章节文档。' + }, + { + value: 'laws', + label: 'Laws', + description: '法规分块：按法条层级组织与合并，适合法律法规、制度规范类文本。' + } +] + +export const CHUNK_PRESET_LABEL_MAP = Object.fromEntries( + CHUNK_PRESET_OPTIONS.map((item) => [item.value, item.label]) +) + +export const CHUNK_PRESET_DESCRIPTION_MAP = Object.fromEntries( + CHUNK_PRESET_OPTIONS.map((item) => [item.value, item.description]) +) + +export const getChunkPresetDescription = (presetId) => + CHUNK_PRESET_DESCRIPTION_MAP[presetId] || CHUNK_PRESET_DESCRIPTION_MAP.general diff --git a/web/src/views/DataBaseView.vue b/web/src/views/DataBaseView.vue index 1497f138..87294cb0 100644 --- a/web/src/views/DataBaseView.vue +++ b/web/src/views/DataBaseView.vue @@ -56,6 +56,19 @@ placeholder="请选择嵌入模型" /> +
+

分块策略

+ + + +
+ +

语言

@@ -197,7 +210,7 @@ import { useRouter, useRoute } from 'vue-router' import { storeToRefs } from 'pinia' import { useConfigStore } from '@/stores/config' import { useDatabaseStore } from '@/stores/database' -import { LockOutlined, InfoCircleOutlined, PlusOutlined } from '@ant-design/icons-vue' +import { LockOutlined, PlusOutlined, QuestionCircleOutlined } from '@ant-design/icons-vue' import { typeApi } from '@/apis/knowledge_api' import HeaderComponent from '@/components/HeaderComponent.vue' import ModelSelectorComponent from '@/components/ModelSelectorComponent.vue' @@ -206,6 +219,7 @@ import ShareConfigForm from '@/components/ShareConfigForm.vue' import dayjs, { parseToShanghai } from '@/utils/time' import AiTextarea from '@/components/AiTextarea.vue' import { getKbTypeLabel, getKbTypeIcon, getKbTypeColor } from '@/utils/kb_utils' +import { CHUNK_PRESET_OPTIONS, getChunkPresetDescription } from '@/utils/chunk_presets' const route = useRoute() const router = useRouter() @@ -240,6 +254,8 @@ const languageOptions = [ { label: '印地语 Hindi', value: 'Hindi' } ] +const chunkPresetOptions = CHUNK_PRESET_OPTIONS.map(({ label, value }) => ({ label, value })) + const createEmptyDatabaseForm = () => ({ name: '', description: '', @@ -247,6 +263,7 @@ const createEmptyDatabaseForm = () => ({ kb_type: 'milvus', is_private: false, storage: '', + chunk_preset_id: 'general', language: 'Chinese', llm_info: { provider: '', @@ -256,6 +273,10 @@ const createEmptyDatabaseForm = () => ({ const newDatabase = reactive(createEmptyDatabaseForm()) +const selectedPresetDescription = computed(() => + getChunkPresetDescription(newDatabase.chunk_preset_id) +) + const llmModelSpec = computed(() => { const provider = newDatabase.llm_info?.provider || '' const modelName = newDatabase.llm_info?.model_name || '' @@ -364,7 +385,8 @@ const buildRequestData = () => { embed_model_name: newDatabase.embed_model_name || configStore.config.embed_model, kb_type: newDatabase.kb_type, additional_params: { - is_private: newDatabase.is_private || false + is_private: newDatabase.is_private || false, + chunk_preset_id: newDatabase.chunk_preset_id || 'general' } } @@ -429,6 +451,20 @@ onMounted(() => {