feat(knowledge): add ragflow-like chunk preset pipeline

This commit is contained in:
肖泽涛 2026-02-20 19:35:43 +08:00
parent 4bd77fce12
commit 60bc408a3e
23 changed files with 1864 additions and 71 deletions

View File

@ -13,6 +13,7 @@ from starlette.responses import StreamingResponse
from src.services.task_service import TaskContext, tasker
from server.utils.auth_middleware import get_admin_user, get_required_user
from src import config, knowledge_base
from src.knowledge.chunking.ragflow_like.presets import ensure_chunk_defaults_in_additional_params
from src.knowledge.indexing import SUPPORTED_FILE_EXTENSIONS, is_supported_file_extension, process_file_to_markdown
from src.knowledge.utils import calculate_content_hash
from src.models.embed import test_all_embedding_models_status, test_embedding_model_status
@ -116,6 +117,7 @@ async def create_database(
params.pop("reranker_config", None)
remove_reranker_config(kb_type, additional_params)
additional_params = ensure_chunk_defaults_in_additional_params(additional_params)
embed_info = config.embed_model_names[embed_model_name]
# 将Pydantic模型转换为字典以便JSON序列化
@ -180,7 +182,7 @@ async def update_database_info(
name: str = Body(...),
description: str = Body(...),
llm_info: dict = Body(None),
additional_params: dict = Body({}),
additional_params: dict | None = Body(None),
share_config: dict = Body(None),
current_user: User = Depends(get_admin_user),
):
@ -190,6 +192,9 @@ async def update_database_info(
f"additional_params={additional_params}, share_config={share_config}"
)
try:
if additional_params is not None:
additional_params = ensure_chunk_defaults_in_additional_params(additional_params)
database = await knowledge_base.update_database(
db_id,
name,
@ -267,7 +272,13 @@ async def add_documents(
"chunk_size": params.get("chunk_size", 1000),
"chunk_overlap": params.get("chunk_overlap", 200),
"qa_separator": params.get("qa_separator", ""),
"chunk_preset_id": params.get("chunk_preset_id"),
"chunk_parser_config": params.get("chunk_parser_config"),
}
if not indexing_params.get("chunk_preset_id"):
indexing_params.pop("chunk_preset_id", None)
if not isinstance(indexing_params.get("chunk_parser_config"), dict):
indexing_params.pop("chunk_parser_config", None)
# URL 解析与入库(需白名单验证)
if content_type == "url":

View File

@ -3,6 +3,10 @@ import os
from abc import ABC, abstractmethod
from typing import Any
from src.knowledge.chunking.ragflow_like.presets import (
ensure_chunk_defaults_in_additional_params,
resolve_chunk_processing_params,
)
from src.utils import logger
from src.utils.datetime_utils import coerce_any_to_utc_datetime, utc_isoformat
@ -76,6 +80,7 @@ class KnowledgeBase(ABC):
self.databases_meta = {}
for db_id, meta in global_databases_meta.items():
if meta.get("kb_type") == self.kb_type:
normalized_additional_params = ensure_chunk_defaults_in_additional_params(meta.get("additional_params"))
self.databases_meta[db_id] = {
"name": meta.get("name"),
"description": meta.get("description"),
@ -83,7 +88,7 @@ class KnowledgeBase(ABC):
"embed_info": meta.get("embed_info"),
"llm_info": meta.get("llm_info"),
"query_params": meta.get("query_params"),
"metadata": meta.get("additional_params", {}),
"metadata": normalized_additional_params,
"created_at": meta.get("created_at"),
}
@ -91,7 +96,14 @@ class KnowledgeBase(ABC):
self.files_meta = {}
for file_id, meta in files_meta.items():
if meta.get("database_id") in self.databases_meta:
self.files_meta[file_id] = meta
db_id = meta.get("database_id")
kb_additional_params = self.databases_meta.get(db_id, {}).get("metadata") or {}
normalized_meta = dict(meta)
normalized_meta["processing_params"] = resolve_chunk_processing_params(
kb_additional_params=kb_additional_params,
file_processing_params=meta.get("processing_params"),
)
self.files_meta[file_id] = normalized_meta
# 过滤评估基准
self.benchmarks_meta = {}
@ -199,6 +211,11 @@ class KnowledgeBase(ABC):
# Prepare metadata
metadata = await prepare_item_metadata(item, content_type, db_id, params=params)
file_id = metadata["file_id"]
kb_additional_params = self.databases_meta.get(db_id, {}).get("metadata") or {}
metadata["processing_params"] = resolve_chunk_processing_params(
kb_additional_params=kb_additional_params,
file_processing_params=metadata.get("processing_params"),
)
# Initial status
metadata["status"] = FileStatus.UPLOADED
@ -323,13 +340,16 @@ class KnowledgeBase(ABC):
if not params:
return
# Merge or overwrite? Usually merge is safer, or replace.
# User might want to change chunk size.
current_params = self.files_meta[file_id].get("processing_params", {}) or {}
kb_additional_params = self.databases_meta.get(db_id, {}).get("metadata") or {}
logger.debug(f"[update_file_params] file_id={file_id}, current_params={current_params}, new_params={params}")
current_params.update(params)
current_params = resolve_chunk_processing_params(
kb_additional_params=kb_additional_params,
file_processing_params=current_params,
request_params=params,
)
self.files_meta[file_id]["processing_params"] = current_params
self.files_meta[file_id]["updated_at"] = utc_isoformat()
@ -414,6 +434,8 @@ class KnowledgeBase(ABC):
"""
from src.utils import hashstr
kwargs = ensure_chunk_defaults_in_additional_params(kwargs)
# 从 kwargs 中获取 is_private 配置
is_private = kwargs.get("is_private", False)
prefix = "kb_private_" if is_private else "kb_"
@ -982,7 +1004,7 @@ class KnowledgeBase(ABC):
"embed_info": kb.embed_info,
"llm_info": kb.llm_info,
"query_params": kb.query_params,
"metadata": kb.additional_params or {},
"metadata": ensure_chunk_defaults_in_additional_params(kb.additional_params),
"created_at": utc_isoformat(kb.created_at) if kb.created_at else utc_isoformat(),
}
for kb in databases
@ -990,6 +1012,7 @@ class KnowledgeBase(ABC):
self.files_meta = {}
for kb in databases:
kb_additional_params = self.databases_meta.get(kb.db_id, {}).get("metadata") or {}
for record in await file_repo.list_by_db_id(kb.db_id):
self.files_meta[record.file_id] = {
"file_id": record.file_id,
@ -1003,7 +1026,10 @@ class KnowledgeBase(ABC):
"content_hash": record.content_hash,
"size": record.file_size,
"content_type": record.content_type,
"processing_params": record.processing_params,
"processing_params": resolve_chunk_processing_params(
kb_additional_params=kb_additional_params,
file_processing_params=record.processing_params,
),
"is_folder": record.is_folder,
"error": record.error_message,
"created_by": record.created_by,

View File

@ -0,0 +1 @@
__all__ = []

View File

@ -0,0 +1,3 @@
from src.knowledge.chunking.ragflow_like.dispatcher import chunk_file, chunk_markdown
__all__ = ["chunk_file", "chunk_markdown"]

View File

@ -0,0 +1,58 @@
from __future__ import annotations
from typing import Any
from src.knowledge.chunking.ragflow_like.parsers import book, general, laws, qa
from src.knowledge.chunking.ragflow_like.presets import map_to_internal_parser_id, normalize_chunk_preset_id
def _build_chunk_records(text_chunks: list[str], file_id: str, filename: str) -> list[dict[str, Any]]:
records: list[dict[str, Any]] = []
for idx, chunk_content in enumerate(text_chunks):
text = (chunk_content or "").strip()
if not text:
continue
records.append(
{
"id": f"{file_id}_chunk_{idx}",
"content": text,
"file_id": file_id,
"filename": filename,
"chunk_index": idx,
"source": filename,
"chunk_id": f"{file_id}_chunk_{idx}",
}
)
return records
def _dispatch_markdown_parser(preset_id: str, filename: str, markdown_content: str, parser_config: dict[str, Any]) -> list[str]:
parser_id = map_to_internal_parser_id(preset_id)
if parser_id == "naive":
return general.chunk_markdown(markdown_content, parser_config)
if parser_id == "qa":
return qa.chunk_markdown(filename, markdown_content, parser_config)
if parser_id == "book":
return book.chunk_markdown(markdown_content, parser_config)
if parser_id == "laws":
return laws.chunk_markdown(filename, markdown_content, parser_config)
return general.chunk_markdown(markdown_content, parser_config)
def chunk_markdown(markdown_content: str, file_id: str, filename: str, processing_params: dict[str, Any]) -> list[dict[str, Any]]:
params = dict(processing_params or {})
preset_id = normalize_chunk_preset_id(params.get("chunk_preset_id"))
parser_config = params.get("chunk_parser_config") if isinstance(params.get("chunk_parser_config"), dict) else {}
text_chunks = _dispatch_markdown_parser(preset_id, filename, markdown_content, parser_config)
return _build_chunk_records(text_chunks, file_id, filename)
def chunk_file(file_content: str, file_id: str, filename: str, processing_params: dict[str, Any]) -> list[dict[str, Any]]:
# 当前链路中入库前均已转换为 markdown因此与 chunk_markdown 保持同实现。
return chunk_markdown(file_content, file_id, filename, processing_params)

View File

@ -0,0 +1,536 @@
from __future__ import annotations
import random
import re
from collections import Counter
from dataclasses import dataclass, field
BULLET_PATTERN = [
[
r"第[零一二三四五六七八九十百0-9]+(分?编|部分)",
r"第[零一二三四五六七八九十百0-9]+章",
r"第[零一二三四五六七八九十百0-9]+节",
r"第[零一二三四五六七八九十百0-9]+条",
r"[\(][零一二三四五六七八九十百]+[\)]",
],
[
r"第[0-9]+章",
r"第[0-9]+节",
r"[0-9]{,2}[\. 、]",
r"[0-9]{,2}\.[0-9]{,2}[^a-zA-Z/%~-]",
r"[0-9]{,2}\.[0-9]{,2}\.[0-9]{,2}",
r"[0-9]{,2}\.[0-9]{,2}\.[0-9]{,2}\.[0-9]{,2}",
],
[
r"第[零一二三四五六七八九十百0-9]+章",
r"第[零一二三四五六七八九十百0-9]+节",
r"[零一二三四五六七八九十百]+[ 、]",
r"[\(][零一二三四五六七八九十百]+[\)]",
r"[\(][0-9]{,2}[\)]",
],
[
r"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)",
r"Chapter (I+V?|VI*|XI|IX|X)",
r"Section [0-9]+",
r"Article [0-9]+",
],
[
r"^#[^#]",
r"^##[^#]",
r"^###.*",
r"^####.*",
r"^#####.*",
r"^######.*",
],
]
MARKDOWN_BULLET_GROUP_INDEX = 4
def count_tokens(text: str) -> int:
"""近似 token 计数,避免引入额外依赖。"""
if not text:
return 0
# 英文单词 + 数字 + CJK 单字
parts = re.findall(r"[A-Za-z0-9_]+|[\u4e00-\u9fff]", text)
return max(1, len(parts)) if text.strip() else 0
def random_choices(arr: list[str], k: int) -> list[str]:
if not arr:
return []
return random.choices(arr, k=min(len(arr), k))
def is_english(texts: str | list[str]) -> bool:
if not texts:
return False
patt = re.compile(r"[`a-zA-Z0-9\s.,':;/\"?<>!\(\)\-]+")
if isinstance(texts, str):
seq = [texts]
else:
seq = [t for t in texts if isinstance(t, str) and t.strip()]
if not seq:
return False
hits = sum(1 for t in seq if patt.fullmatch(t.strip()))
return (hits / len(seq)) > 0.8
def not_bullet(line: str) -> bool:
patt = [
r"0",
r"[0-9]+ +[0-9~个只-]",
r"[0-9]+\.{2,}",
]
return any(re.match(p, line) for p in patt)
def is_probable_heading_line(line: str) -> bool:
text = (line or "").strip()
if not text:
return False
if re.match(r"^#{1,6}\s+\S", text):
return True
# 表格/HTML 残留通常不是标题。
if re.search(r"</?(table|tr|td|th|caption|tbody|thead)[^>]*>", text, flags=re.IGNORECASE):
return False
# 超长行基本是正文或条款,不是章节标题。
if len(text) > 96:
return False
if count_tokens(text) > 72:
return False
# 标题前段通常不会出现明显句号/逗号;出现则大概率是正文。
if re.search(r"[,。;!?!?:]", text[:24]):
return False
if text.endswith(("", "", "", "!", "", "?")) and len(text) > 20:
return False
return True
def _is_mid_sentence_bullet(line: str) -> bool:
text = (line or "").strip()
if not text:
return False
if re.match(r"^#{1,6}\s+\S", text):
return False
marker = re.search(
r"([一二三四五六七八九十百]+、|[\(][一二三四五六七八九十百]+[\)]|[0-9]{1,2}[\.、])",
text,
)
if not marker:
return False
if marker.start() == 0:
return False
prev = text[marker.start() - 1]
return prev not in {"#", "\n"}
def bullets_category(sections: list[str]) -> int:
hits: list[float] = [0.0] * len(BULLET_PATTERN)
def bullet_weight(group_idx: int, line: str) -> float:
# 对 markdown 标题候选增加权重,避免“正文里的 一、/(一)”压过真正的 # 标题层级。
if group_idx != MARKDOWN_BULLET_GROUP_INDEX:
return 1.0
heading = line.strip()
if not re.match(r"^#{1,6}\s+\S", heading):
return 1.0
level = len(heading) - len(heading.lstrip("#"))
if level <= 2:
return 4.0
if level <= 4:
return 3.0
return 2.0
for i, pro in enumerate(BULLET_PATTERN):
for sec in sections:
sec = sec.strip()
for p in pro:
if re.match(p, sec) and not not_bullet(sec):
w = bullet_weight(i, sec)
if _is_mid_sentence_bullet(sec):
w *= 0.1
if i != MARKDOWN_BULLET_GROUP_INDEX and not is_probable_heading_line(sec):
w *= 0.2
hits[i] += w
break
maximum = 0
res = -1
for i, hit in enumerate(hits):
if hit <= maximum:
continue
res = i
maximum = hit
return res
def _get_text(section: str | tuple[str, str]) -> str:
if isinstance(section, str):
return section.strip()
return (section[0] or "").strip()
def remove_contents_table(sections: list[str] | list[tuple[str, str]], eng: bool = False) -> None:
i = 0
while i < len(sections):
line = re.sub(r"( | |\u3000)+", "", _get_text(sections[i]).split("@@")[0], flags=re.IGNORECASE)
if not re.match(r"(contents|目录|目次|tableofcontents|致谢|acknowledge)$", line, flags=re.IGNORECASE):
i += 1
continue
sections.pop(i)
if i >= len(sections):
break
prefix = _get_text(sections[i])[:3] if not eng else " ".join(_get_text(sections[i]).split()[:2])
while not prefix and i < len(sections):
sections.pop(i)
if i >= len(sections):
break
prefix = _get_text(sections[i])[:3] if not eng else " ".join(_get_text(sections[i]).split()[:2])
if i >= len(sections) or not prefix:
break
sections.pop(i)
if i >= len(sections):
break
for j in range(i, min(i + 128, len(sections))):
if not re.match(re.escape(prefix), _get_text(sections[j])):
continue
for _ in range(i, j):
sections.pop(i)
break
def make_colon_as_title(sections: list[str] | list[tuple[str, str]]) -> list[str] | list[tuple[str, str]]:
if not sections:
return sections
if isinstance(sections[0], str):
return sections
i = 0
while i < len(sections):
text, layout = sections[i]
i += 1
text = text.split("@")[0].strip()
if not text or text[-1] not in ":":
continue
rev = text[::-1]
arr = re.split(r"([。?!!?;]| \.)", rev)
if len(arr) < 2 or len(arr[1]) < 32:
continue
sections.insert(i - 1, (arr[0][::-1], "title"))
i += 1
return sections
def not_title(text: str) -> bool:
if re.match(r"第[零一二三四五六七八九十百0-9]+条", text):
return False
if len(text.split()) > 12 or (" " not in text and len(text) >= 32):
return True
return bool(re.search(r"[,;,。;!!]", text))
def tree_merge(bull: int, sections: list[str] | list[tuple[str, str]], depth: int) -> list[str]:
if not sections or bull < 0:
return [s if isinstance(s, str) else s[0] for s in sections]
if isinstance(sections[0], str):
typed_sections: list[tuple[str, str]] = [(s, "") for s in sections]
else:
typed_sections = sections # type: ignore[assignment]
typed_sections = [
(t, o)
for t, o in typed_sections
if t and len(t.split("@")[0].strip()) > 1 and not re.match(r"[0-9]+$", t.split("@")[0].strip())
]
def get_level(section: tuple[str, str]) -> tuple[int, str]:
text, layout = section
text = re.sub(r"\u3000", " ", text).strip()
for i, patt in enumerate(BULLET_PATTERN[bull]):
if re.match(patt, text) and is_probable_heading_line(text):
return i + 1, text
if re.search(r"(title|head)", layout) and not not_title(text):
return len(BULLET_PATTERN[bull]) + 1, text
return len(BULLET_PATTERN[bull]) + 2, text
lines: list[tuple[int, str]] = []
level_set: set[int] = set()
for section in typed_sections:
level, text = get_level(section)
if not text.strip("\n"):
continue
lines.append((level, text))
level_set.add(level)
if not lines:
return []
sorted_levels = sorted(level_set)
target_level = sorted_levels[depth - 1] if depth <= len(sorted_levels) else sorted_levels[-1]
max_body_level = len(BULLET_PATTERN[bull]) + 2
if target_level == max_body_level:
target_level = sorted_levels[-2] if len(sorted_levels) > 1 else sorted_levels[0]
root = Node(level=0, depth=target_level, texts=[])
root.build_tree(lines)
return [item for item in root.get_tree() if item]
def hierarchical_merge(bull: int, sections: list[str] | list[tuple[str, str]], depth: int) -> list[list[str]]:
if not sections or bull < 0:
return []
if isinstance(sections[0], str):
typed_sections: list[tuple[str, str]] = [(s, "") for s in sections]
else:
typed_sections = sections # type: ignore[assignment]
typed_sections = [
(t, o)
for t, o in typed_sections
if t and len(t.split("@")[0].strip()) > 1 and not re.match(r"[0-9]+$", t.split("@")[0].strip())
]
bullets_size = len(BULLET_PATTERN[bull])
levels: list[list[int]] = [[] for _ in range(bullets_size + 2)]
for i, (text, layout) in enumerate(typed_sections):
for j, patt in enumerate(BULLET_PATTERN[bull]):
if re.match(patt, text.strip()) and is_probable_heading_line(text):
levels[j].append(i)
break
else:
if re.search(r"(title|head)", layout) and not not_title(text):
levels[bullets_size].append(i)
else:
levels[bullets_size + 1].append(i)
pure_sections = [t for t, _ in typed_sections]
def binary_search(arr: list[int], target: int) -> int:
if not arr:
return -1
if target > arr[-1]:
return len(arr) - 1
if target < arr[0]:
return -1
s, e = 0, len(arr)
while e - s > 1:
mid = (e + s) // 2
if target > arr[mid]:
s = mid
elif target < arr[mid]:
e = mid
else:
return mid
return s
cks: list[list[int]] = []
readed = [False] * len(pure_sections)
levels = list(reversed(levels))
for i, arr in enumerate(levels[:depth]):
for j in arr:
if readed[j]:
continue
readed[j] = True
cks.append([j])
if i + 1 == len(levels) - 1:
continue
for ii in range(i + 1, len(levels)):
jj = binary_search(levels[ii], j)
if jj < 0:
continue
if levels[ii][jj] > cks[-1][-1]:
cks[-1].pop(-1)
cks[-1].append(levels[ii][jj])
for ii in cks[-1]:
readed[ii] = True
if not cks:
return []
for i in range(len(cks)):
cks[i] = [pure_sections[j] for j in reversed(cks[i])]
res: list[list[str]] = [[]]
num = [0]
for ck in cks:
if len(ck) == 1:
n = count_tokens(re.sub(r"@@[0-9]+.*", "", ck[0]))
if n + num[-1] < 218:
res[-1].append(ck[0])
num[-1] += n
continue
res.append(ck)
num.append(n)
continue
res.append(ck)
num.append(218)
return [chunk for chunk in res if chunk]
def _remove_pdf_tags(text: str) -> str:
return re.sub(r"@@[0-9-]+\t[0-9.\t]+##", "", text or "")
def _extract_custom_delimiters(delimiter: str) -> list[str]:
return [m.group(1) for m in re.finditer(r"`([^`]+)`", delimiter or "")]
def naive_merge(
sections: str | list[str] | list[tuple[str, str]],
chunk_token_num: int = 128,
delimiter: str = "\n。;!?",
overlapped_percent: int = 0,
) -> list[str]:
if not sections:
return []
if isinstance(sections, str):
typed_sections: list[tuple[str, str]] = [(sections, "")]
elif isinstance(sections[0], str):
typed_sections = [(s, "") for s in sections] # type: ignore[index]
else:
typed_sections = sections # type: ignore[assignment]
chunk_token_num = max(int(chunk_token_num or 0), 0)
overlap = max(0, min(int(overlapped_percent or 0), 99))
custom_delimiters = _extract_custom_delimiters(delimiter)
if custom_delimiters:
pattern = "|".join(re.escape(t) for t in sorted(set(custom_delimiters), key=len, reverse=True))
chunks: list[str] = []
for sec, pos in typed_sections:
split_sec = re.split(rf"({pattern})", sec, flags=re.DOTALL)
for sub in split_sec:
if re.fullmatch(pattern, sub or ""):
continue
text = "\n" + sub
local_pos = pos if count_tokens(text) >= 8 else ""
if local_pos and local_pos not in text:
text += local_pos
if text.strip():
chunks.append(text)
return chunks
if chunk_token_num <= 0:
merged = "\n".join(sec for sec, _ in typed_sections if sec and sec.strip())
return [merged] if merged.strip() else []
chunks = [""]
token_nums = [0]
def add_chunk(text: str, pos: str) -> None:
tnum = count_tokens(text)
local_pos = pos or ""
if tnum < 8:
local_pos = ""
threshold = chunk_token_num * (100 - overlap) / 100.0
if chunks[-1] == "" or token_nums[-1] > threshold:
if chunks:
prev = _remove_pdf_tags(chunks[-1])
start = int(len(prev) * (100 - overlap) / 100.0)
text = prev[start:] + text
if local_pos and local_pos not in text:
text += local_pos
chunks.append(text)
token_nums.append(tnum)
else:
if local_pos and local_pos not in chunks[-1]:
text += local_pos
chunks[-1] += text
token_nums[-1] += tnum
for sec, pos in typed_sections:
if not sec:
continue
add_chunk("\n" + sec, pos)
return [chunk for chunk in chunks if chunk.strip()]
@dataclass
class Node:
level: int
depth: int = -1
texts: list[str] = field(default_factory=list)
children: list["Node"] = field(default_factory=list)
def add_child(self, child_node: "Node") -> None:
self.children.append(child_node)
def add_text(self, text: str) -> None:
self.texts.append(text)
def build_tree(self, lines: list[tuple[int, str]]) -> "Node":
stack: list[Node] = [self]
for level, text in lines:
if self.depth != -1 and level > self.depth:
stack[-1].add_text(text)
continue
while len(stack) > 1 and level <= stack[-1].level:
stack.pop()
node = Node(level=level, texts=[text])
stack[-1].add_child(node)
stack.append(node)
return self
def get_tree(self) -> list[str]:
tree_list: list[str] = []
self._dfs(self, tree_list, [])
return tree_list
def _dfs(self, node: "Node", tree_list: list[str], titles: list[str]) -> None:
level = node.level
texts = node.texts
child = node.children
if level == 0 and texts:
tree_list.append("\n".join(titles + texts))
path_titles = titles + texts if 1 <= level <= self.depth else titles
if level > self.depth and texts:
tree_list.append("\n".join(path_titles + texts))
elif not child and (1 <= level <= self.depth):
tree_list.append("\n".join(path_titles))
for c in child:
self._dfs(c, tree_list, path_titles)

View File

@ -0,0 +1,3 @@
from src.knowledge.chunking.ragflow_like.parsers import book, general, laws, qa
__all__ = ["general", "qa", "book", "laws"]

View File

@ -0,0 +1,61 @@
from __future__ import annotations
from typing import Any
from src.knowledge.chunking.ragflow_like import nlp
def _unescape_delimiter(delimiter: str) -> str:
return delimiter.replace("\\n", "\n").replace("\\r", "\r").replace("\\t", "\t").replace("\\\\", "\\")
def _iter_sections(markdown_content: str) -> list[tuple[str, str]]:
sections: list[tuple[str, str]] = []
for line in (markdown_content or "").splitlines():
text = line.strip()
if not text:
continue
sections.append((text, ""))
if not sections and markdown_content and markdown_content.strip():
sections.append((markdown_content.strip(), ""))
return sections
def chunk_markdown(markdown_content: str, parser_config: dict[str, Any] | None = None) -> list[str]:
parser_config = parser_config or {}
delimiter = _unescape_delimiter(str(parser_config.get("delimiter", "\n") or "\n"))
chunk_token_num = int(parser_config.get("chunk_token_num", 512) or 512)
overlapped_percent = int(parser_config.get("overlapped_percent", 0) or 0)
sections = _iter_sections(markdown_content)
if not sections:
return []
section_texts = [text for text, _ in sections]
nlp.remove_contents_table(sections, eng=nlp.is_english(nlp.random_choices(section_texts, k=200)))
nlp.make_colon_as_title(sections)
bull = nlp.bullets_category([t for t in nlp.random_choices([t for t, _ in sections], k=100)])
if bull >= 0:
chunks = ["\n".join(ck) for ck in nlp.hierarchical_merge(bull, sections, depth=5)]
else:
chunks = nlp.naive_merge(
sections,
chunk_token_num=chunk_token_num,
delimiter=delimiter,
overlapped_percent=overlapped_percent,
)
if chunks:
return chunks
return nlp.naive_merge(
sections,
chunk_token_num=chunk_token_num,
delimiter=delimiter,
overlapped_percent=overlapped_percent,
)

View File

@ -0,0 +1,46 @@
from __future__ import annotations
from typing import Any
from src.knowledge.chunking.ragflow_like import nlp
def _unescape_delimiter(delimiter: str) -> str:
return delimiter.replace("\\n", "\n").replace("\\r", "\r").replace("\\t", "\t").replace("\\\\", "\\")
def _iter_sections(markdown_content: str, delimiter: str) -> list[tuple[str, str]]:
sections: list[tuple[str, str]] = []
text = markdown_content or ""
if delimiter and delimiter not in {"\n", "\r\n"} and "`" not in delimiter:
for part in text.split(delimiter):
block = part.strip()
if block:
sections.append((block, ""))
else:
for line in text.splitlines():
block = line.strip()
if not block:
continue
sections.append((block, ""))
if not sections and text.strip():
sections.append((text.strip(), ""))
return sections
def chunk_markdown(markdown_content: str, parser_config: dict[str, Any] | None = None) -> list[str]:
parser_config = parser_config or {}
delimiter = _unescape_delimiter(str(parser_config.get("delimiter", "\n") or "\n"))
chunk_token_num = int(parser_config.get("chunk_token_num", 512) or 512)
overlapped_percent = int(parser_config.get("overlapped_percent", 0) or 0)
sections = _iter_sections(markdown_content, delimiter)
return nlp.naive_merge(
sections,
chunk_token_num=chunk_token_num,
delimiter=delimiter,
overlapped_percent=overlapped_percent,
)

View File

@ -0,0 +1,85 @@
from __future__ import annotations
import re
from typing import Any
from src.knowledge.chunking.ragflow_like import nlp
def _unescape_delimiter(delimiter: str) -> str:
return delimiter.replace("\\n", "\n").replace("\\r", "\r").replace("\\t", "\t").replace("\\\\", "\\")
def _iter_lines(markdown_content: str) -> list[str]:
return [line.strip() for line in (markdown_content or "").splitlines() if line.strip()]
def _docx_heading_tree(markdown_content: str) -> list[str]:
lines: list[tuple[int, str]] = []
level_set: set[int] = set()
for raw in (markdown_content or "").splitlines():
text = raw.strip()
if not text:
continue
heading_match = re.match(r"^(#{1,6})\s+(.*)$", text)
if heading_match:
level = len(heading_match.group(1))
value = heading_match.group(2).strip()
else:
level = 99
value = text
if not value:
continue
lines.append((level, value))
level_set.add(level)
if not lines:
return []
sorted_levels = sorted(level_set)
h2_level = sorted_levels[1] if len(sorted_levels) > 1 else 1
h2_level = sorted_levels[-2] if h2_level == sorted_levels[-1] and len(sorted_levels) > 2 else h2_level
root = nlp.Node(level=0, depth=h2_level, texts=[])
root.build_tree(lines)
return [element for element in root.get_tree() if element]
def chunk_markdown(filename: str, markdown_content: str, parser_config: dict[str, Any] | None = None) -> list[str]:
parser_config = parser_config or {}
delimiter = _unescape_delimiter(str(parser_config.get("delimiter", "\n") or "\n"))
chunk_token_num = int(parser_config.get("chunk_token_num", 512) or 512)
overlapped_percent = int(parser_config.get("overlapped_percent", 0) or 0)
if re.search(r"\.docx$", filename or "", re.IGNORECASE):
chunks = _docx_heading_tree(markdown_content)
if chunks:
return chunks
sections = _iter_lines(markdown_content)
if not sections:
return []
eng = nlp.is_english(sections)
nlp.remove_contents_table(sections, eng=eng)
typed_sections = [(s, "") for s in sections]
nlp.make_colon_as_title(typed_sections)
bull = nlp.bullets_category([s for s, _ in typed_sections])
merged = nlp.tree_merge(bull, typed_sections, depth=2)
if merged:
return merged
return nlp.naive_merge(
typed_sections,
chunk_token_num=chunk_token_num,
delimiter=delimiter,
overlapped_percent=overlapped_percent,
)

View File

@ -0,0 +1,261 @@
from __future__ import annotations
import csv
import re
from io import StringIO
from typing import Any
def _rm_prefix(text: str) -> str:
return re.sub(
r"^(问题|答案|回答|user|assistant|Q|A|Question|Answer|问|答)[\t: ]+",
"",
(text or "").strip(),
flags=re.IGNORECASE,
)
def _to_qa_chunk(question: str, answer: str, eng: bool = False) -> str:
qprefix = "Question: " if eng else "问题:"
aprefix = "Answer: " if eng else "回答:"
return "\t".join([qprefix + _rm_prefix(question), aprefix + _rm_prefix(answer)])
def _guess_delimiter(lines: list[str]) -> str:
comma = 0
tab = 0
for line in lines:
if len(line.split(",")) == 2:
comma += 1
if len(line.split("\t")) == 2:
tab += 1
return "\t" if tab >= comma else ","
def _extract_pairs_with_delimiter(lines: list[str], delimiter: str) -> list[tuple[str, str]]:
pairs: list[tuple[str, str]] = []
question = ""
answer = ""
for line in lines:
arr = line.split(delimiter)
if len(arr) != 2:
if question:
answer += "\n" + line
continue
if question and answer:
pairs.append((question, answer))
question, answer = arr
if question:
pairs.append((question, answer))
return [(q.strip(), a.strip()) for q, a in pairs if q.strip()]
def _extract_pairs_from_csv(lines: list[str], delimiter: str) -> list[tuple[str, str]]:
pairs: list[tuple[str, str]] = []
question = ""
answer = ""
reader = csv.reader(lines, delimiter=delimiter)
for row, raw_line in zip(reader, lines, strict=False):
if len(row) != 2:
if question:
answer += "\n" + raw_line
continue
if question and answer:
pairs.append((question, answer))
question, answer = row
if question:
pairs.append((question, answer))
return [(q.strip(), a.strip()) for q, a in pairs if q.strip()]
def _parse_markdown_table_row(line: str) -> list[str] | None:
if "|" not in line:
return None
text = line.strip()
if not text:
return None
if text.startswith("|"):
text = text[1:]
if text.endswith("|"):
text = text[:-1]
cells = [cell.strip() for cell in text.split("|")]
if not cells:
return None
if all(re.fullmatch(r":?-{3,}:?", c.replace(" ", "")) for c in cells if c):
return None
return cells
def _extract_pairs_from_markdown_tables(markdown_content: str) -> list[tuple[str, str]]:
pairs: list[tuple[str, str]] = []
for line in (markdown_content or "").splitlines():
cells = _parse_markdown_table_row(line)
if not cells or len(cells) < 2:
continue
question = cells[0]
answer = cells[1]
if question and answer:
pairs.append((question, answer))
return pairs
def _md_question_level(line: str) -> tuple[int, str]:
match = re.match(r"^#*", line)
if not match:
return 0, line
return len(match.group(0)), line.lstrip("#").lstrip()
def _extract_pairs_from_markdown_headings(markdown_content: str) -> list[tuple[str, str]]:
lines = (markdown_content or "").splitlines()
if not lines:
return []
pairs: list[tuple[str, str]] = []
last_answer = ""
question_stack: list[str] = []
level_stack: list[int] = []
code_block = False
for line in lines:
if line.strip().startswith("```"):
code_block = not code_block
question_level = 0
question = ""
if not code_block:
question_level, question = _md_question_level(line)
if not question_level or question_level > 6:
last_answer = f"{last_answer}\n{line}"
continue
if last_answer.strip():
sum_question = "\n".join(question_stack)
if sum_question:
pairs.append((sum_question, last_answer.strip()))
last_answer = ""
while question_stack and question_level <= level_stack[-1]:
question_stack.pop()
level_stack.pop()
question_stack.append(question)
level_stack.append(question_level)
if last_answer.strip():
sum_question = "\n".join(question_stack)
if sum_question:
pairs.append((sum_question, last_answer.strip()))
return pairs
def _extract_pairs_by_prefix(lines: list[str]) -> list[tuple[str, str]]:
pairs: list[tuple[str, str]] = []
question = ""
answer_lines: list[str] = []
for line in lines:
if re.match(r"^(Q|Question|问|问题)\s*[:]", line, flags=re.IGNORECASE):
if question:
pairs.append((question, "\n".join(answer_lines)))
question = re.sub(r"^(Q|Question|问|问题)\s*[:]", "", line, flags=re.IGNORECASE).strip()
answer_lines = []
continue
if re.match(r"^(A|Answer|答|回答)\s*[:]", line, flags=re.IGNORECASE):
answer_lines.append(re.sub(r"^(A|Answer|答|回答)\s*[:]", "", line, flags=re.IGNORECASE).strip())
continue
if question:
answer_lines.append(line)
if question:
pairs.append((question, "\n".join(answer_lines)))
return [(q.strip(), a.strip()) for q, a in pairs if q.strip() and a.strip()]
def _dedupe_pairs(pairs: list[tuple[str, str]]) -> list[tuple[str, str]]:
res: list[tuple[str, str]] = []
seen: set[tuple[str, str]] = set()
for question, answer in pairs:
q = question.strip()
a = answer.strip()
if not q or not a:
continue
key = (q, a)
if key in seen:
continue
seen.add(key)
res.append((q, a))
return res
def chunk_markdown(filename: str, markdown_content: str, parser_config: dict[str, Any] | None = None) -> list[str]:
parser_config = parser_config or {}
eng = str(parser_config.get("language", "Chinese")).lower() == "english"
suffix = ""
if filename and "." in filename:
suffix = "." + filename.lower().split(".")[-1]
lines = [line for line in (markdown_content or "").splitlines() if line.strip()]
pairs: list[tuple[str, str]] = []
if suffix in {".xlsx", ".xls"}:
pairs.extend(_extract_pairs_from_markdown_tables(markdown_content))
if not pairs:
delimiter = _guess_delimiter(lines)
pairs.extend(_extract_pairs_with_delimiter(lines, delimiter))
elif suffix == ".csv":
pairs.extend(_extract_pairs_from_markdown_tables(markdown_content))
delimiter = "\t" if any("\t" in line for line in lines) else ","
pairs.extend(_extract_pairs_from_csv(lines, delimiter))
elif suffix == ".txt":
delimiter = _guess_delimiter(lines)
pairs.extend(_extract_pairs_with_delimiter(lines, delimiter))
elif suffix in {".md", ".markdown", ".mdx"}:
pairs.extend(_extract_pairs_from_markdown_headings(markdown_content))
pairs.extend(_extract_pairs_from_markdown_tables(markdown_content))
elif suffix == ".docx":
pairs.extend(_extract_pairs_from_markdown_headings(markdown_content))
pairs.extend(_extract_pairs_from_markdown_tables(markdown_content))
else:
pairs.extend(_extract_pairs_from_markdown_headings(markdown_content))
pairs.extend(_extract_pairs_from_markdown_tables(markdown_content))
pairs.extend(_extract_pairs_by_prefix(lines))
if not pairs:
delimiter = _guess_delimiter(lines)
pairs.extend(_extract_pairs_with_delimiter(lines, delimiter))
pairs = _dedupe_pairs(pairs)
if not pairs and lines:
# 最后兜底:把内容按 2 行一组构成问答
for i in range(0, len(lines), 2):
q = lines[i]
a = lines[i + 1] if i + 1 < len(lines) else ""
if q.strip() and a.strip():
pairs.append((q, a))
return [_to_qa_chunk(q, a, eng=eng) for q, a in pairs]

View File

@ -0,0 +1,235 @@
from __future__ import annotations
from copy import deepcopy
from typing import Any
from src.utils import logger
CHUNK_PRESET_GENERAL = "general"
CHUNK_PRESET_QA = "qa"
CHUNK_PRESET_BOOK = "book"
CHUNK_PRESET_LAWS = "laws"
CHUNK_PRESET_IDS = {
CHUNK_PRESET_GENERAL,
CHUNK_PRESET_QA,
CHUNK_PRESET_BOOK,
CHUNK_PRESET_LAWS,
}
CHUNK_PRESET_DESCRIPTIONS: dict[str, str] = {
CHUNK_PRESET_GENERAL: "通用分块:按分隔符和长度切分,适合大多数普通文档。",
CHUNK_PRESET_QA: "问答分块:优先抽取问题-回答结构,适合 FAQ、题库、问答手册。",
CHUNK_PRESET_BOOK: "书籍分块:强化章节标题识别并做层级合并,适合教材、手册、长章节文档。",
CHUNK_PRESET_LAWS: "法规分块:按法条层级组织与合并,适合法律法规、制度规范类文本。",
}
CHUNK_ENGINE_VERSION = "ragflow_like_v1"
GENERAL_INTERNAL_PARSER_ID = "naive"
_BASE_DEFAULTS: dict[str, Any] = {
"table_context_size": 0,
"image_context_size": 0,
}
_PRESET_DEFAULTS: dict[str, dict[str, Any] | None] = {
CHUNK_PRESET_GENERAL: {
"layout_recognize": "DeepDOC",
"chunk_token_num": 512,
"delimiter": "\n",
"auto_keywords": 0,
"auto_questions": 0,
"html4excel": False,
"topn_tags": 3,
"raptor": {
"use_raptor": True,
"prompt": "Please summarize the following paragraphs. Be careful with the numbers, do not make things up. Paragraphs as following:\n {cluster_content}\nThe above is the content you need to summarize.",
"max_token": 256,
"threshold": 0.1,
"max_cluster": 64,
"random_seed": 0,
},
"graphrag": {
"use_graphrag": True,
"entity_types": ["organization", "person", "geo", "event", "category"],
"method": "light",
},
},
CHUNK_PRESET_QA: {"raptor": {"use_raptor": False}, "graphrag": {"use_graphrag": False}},
CHUNK_PRESET_BOOK: {"raptor": {"use_raptor": False}, "graphrag": {"use_graphrag": False}},
CHUNK_PRESET_LAWS: {"raptor": {"use_raptor": False}, "graphrag": {"use_graphrag": False}},
}
def deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]:
result = deepcopy(base)
for key, value in (override or {}).items():
if isinstance(value, dict) and isinstance(result.get(key), dict):
result[key] = deep_merge(result[key], value)
else:
result[key] = value
return result
def normalize_chunk_preset_id(value: str | None) -> str:
if not value:
return CHUNK_PRESET_GENERAL
normalized = str(value).strip().lower()
if normalized == GENERAL_INTERNAL_PARSER_ID:
return CHUNK_PRESET_GENERAL
if normalized in CHUNK_PRESET_IDS:
return normalized
logger.warning(f"Unknown chunk preset id '{value}', fallback to general")
return CHUNK_PRESET_GENERAL
def map_to_internal_parser_id(preset_id: str) -> str:
normalized = normalize_chunk_preset_id(preset_id)
if normalized == CHUNK_PRESET_GENERAL:
return GENERAL_INTERNAL_PARSER_ID
return normalized
def get_default_chunk_parser_config(preset_id: str) -> dict[str, Any]:
normalized = normalize_chunk_preset_id(preset_id)
default_config = deepcopy(_PRESET_DEFAULTS.get(normalized) or {})
return deep_merge(_BASE_DEFAULTS, default_config)
def _safe_int(value: Any) -> int | None:
if value is None:
return None
try:
return int(value)
except (TypeError, ValueError):
return None
def _legacy_params_to_parser_config(params: dict[str, Any] | None) -> dict[str, Any]:
if not isinstance(params, dict):
return {}
parser_config: dict[str, Any] = {}
chunk_size = _safe_int(params.get("chunk_size"))
chunk_overlap = _safe_int(params.get("chunk_overlap"))
if chunk_size and chunk_size > 0:
parser_config["chunk_token_num"] = chunk_size
if chunk_size and chunk_size > 0 and chunk_overlap is not None:
overlap_percent = round(max(0, min(chunk_overlap, chunk_size - 1)) * 100 / chunk_size)
parser_config["overlapped_percent"] = max(0, min(overlap_percent, 99))
if isinstance(params.get("qa_separator"), str) and params.get("qa_separator"):
parser_config["delimiter"] = params["qa_separator"]
if isinstance(params.get("delimiter"), str) and params.get("delimiter"):
parser_config["delimiter"] = params["delimiter"]
if "chunk_token_num" in params:
normalized_chunk_token_num = _safe_int(params.get("chunk_token_num"))
if normalized_chunk_token_num is not None:
parser_config["chunk_token_num"] = normalized_chunk_token_num
if "overlapped_percent" in params:
normalized_overlapped_percent = _safe_int(params.get("overlapped_percent"))
if normalized_overlapped_percent is not None:
parser_config["overlapped_percent"] = max(0, min(normalized_overlapped_percent, 99))
return parser_config
def ensure_chunk_defaults_in_additional_params(additional_params: dict[str, Any] | None) -> dict[str, Any]:
params = dict(additional_params or {})
params["chunk_preset_id"] = normalize_chunk_preset_id(params.get("chunk_preset_id"))
if "chunk_parser_config" in params and not isinstance(params.get("chunk_parser_config"), dict):
logger.warning("Invalid chunk_parser_config in additional_params, fallback to empty dict")
params["chunk_parser_config"] = {}
return params
def resolve_chunk_processing_params(
kb_additional_params: dict[str, Any] | None,
file_processing_params: dict[str, Any] | None,
request_params: dict[str, Any] | None = None,
) -> dict[str, Any]:
kb_additional = ensure_chunk_defaults_in_additional_params(kb_additional_params)
file_params = dict(file_processing_params or {})
request = dict(request_params or {})
preset_id = normalize_chunk_preset_id(
request.get("chunk_preset_id")
or file_params.get("chunk_preset_id")
or kb_additional.get("chunk_preset_id")
)
parser_config = get_default_chunk_parser_config(preset_id)
kb_parser_config = kb_additional.get("chunk_parser_config")
if isinstance(kb_parser_config, dict):
parser_config = deep_merge(parser_config, kb_parser_config)
file_parser_config = file_params.get("chunk_parser_config")
if isinstance(file_parser_config, dict):
parser_config = deep_merge(parser_config, file_parser_config)
req_parser_config = request.get("chunk_parser_config")
if isinstance(req_parser_config, dict):
parser_config = deep_merge(parser_config, req_parser_config)
merged_legacy = {}
merged_legacy.update(file_params)
merged_legacy.update(request)
parser_config = deep_merge(parser_config, _legacy_params_to_parser_config(merged_legacy))
# Build processing params snapshot (keep existing + request overrides for non-chunk fields)
snapshot: dict[str, Any] = {}
snapshot.update(file_params)
snapshot.update(request)
snapshot["chunk_preset_id"] = preset_id
snapshot["chunk_parser_config"] = parser_config
snapshot["chunk_engine_version"] = CHUNK_ENGINE_VERSION
# Keep backward-compatible fields for current UI
if "chunk_size" not in snapshot and isinstance(parser_config.get("chunk_token_num"), int):
snapshot["chunk_size"] = parser_config["chunk_token_num"]
if "chunk_overlap" not in snapshot and isinstance(parser_config.get("overlapped_percent"), int):
token_num = parser_config.get("chunk_token_num")
if isinstance(token_num, int) and token_num > 0:
snapshot["chunk_overlap"] = int(token_num * parser_config["overlapped_percent"] / 100)
if "qa_separator" not in snapshot and isinstance(parser_config.get("delimiter"), str):
snapshot["qa_separator"] = parser_config["delimiter"]
return snapshot
def get_chunk_preset_options() -> list[dict[str, str]]:
return [
{
"value": CHUNK_PRESET_GENERAL,
"label": "General",
"description": CHUNK_PRESET_DESCRIPTIONS[CHUNK_PRESET_GENERAL],
},
{
"value": CHUNK_PRESET_QA,
"label": "QA",
"description": CHUNK_PRESET_DESCRIPTIONS[CHUNK_PRESET_QA],
},
{
"value": CHUNK_PRESET_BOOK,
"label": "Book",
"description": CHUNK_PRESET_DESCRIPTIONS[CHUNK_PRESET_BOOK],
},
{
"value": CHUNK_PRESET_LAWS,
"label": "Laws",
"description": CHUNK_PRESET_DESCRIPTIONS[CHUNK_PRESET_LAWS],
},
]

View File

@ -11,6 +11,8 @@ from pymilvus import connections, utility
from src import config
from src.knowledge.base import FileStatus, KnowledgeBase
from src.knowledge.chunking.ragflow_like.dispatcher import chunk_markdown
from src.knowledge.chunking.ragflow_like.presets import resolve_chunk_processing_params
from src.knowledge.indexing import process_file_to_markdown
from src.knowledge.utils.kb_utils import get_embedding_config
from src.utils import hashstr, logger
@ -40,6 +42,18 @@ class LightRagKB(KnowledgeBase):
"""知识库类型标识"""
return "lightrag"
@staticmethod
def _prepare_lightrag_insert_payload(chunks: list[dict]) -> tuple[str, str | None, bool]:
if not chunks:
return "", None, False
if len(chunks) == 1:
return chunks[0]["content"], None, False
delimiter = "\n<|YUXI_CHUNK_DELIM|>\n"
payload = delimiter.join(chunk["content"] for chunk in chunks if chunk.get("content"))
return payload, delimiter, True
def delete_database(self, db_id: str) -> dict:
"""删除数据库同时清除Milvus和Neo4j中的数据"""
# Drop Milvus collection
@ -137,6 +151,21 @@ class LightRagKB(KnowledgeBase):
await instance.initialize_storages()
await initialize_pipeline_status()
@staticmethod
async def _ensure_doc_processed(rag: LightRAG, file_id: str) -> None:
"""确保 LightRAG 文档处理成功,否则抛出异常。"""
status_doc = await rag.doc_status.get_by_id(file_id)
if not status_doc:
raise ValueError(f"LightRAG 文档状态缺失: {file_id}")
status = status_doc.get("status")
status_value = status.value if hasattr(status, "value") else status
if status_value not in {"processed", "preprocessed"}:
error_msg = status_doc.get("error_msg") or "unknown error"
raise ValueError(
f"LightRAG 实体关系抽取失败: file_id={file_id}, status={status_value}, error={error_msg}"
)
async def _get_lightrag_instance(self, db_id: str) -> LightRAG | None:
"""获取或创建 LightRAG 实例"""
if db_id in self.instances:
@ -293,14 +322,36 @@ class LightRagKB(KnowledgeBase):
# Read markdown
markdown_content = await self._read_markdown_from_minio(file_meta["markdown_file"])
file_path = file_meta.get("path")
filename = file_meta.get("filename") or file_id
processing_params = resolve_chunk_processing_params(
kb_additional_params=self.databases_meta.get(db_id, {}).get("metadata"),
file_processing_params=file_meta.get("processing_params"),
)
self.files_meta[file_id]["processing_params"] = processing_params
await self._save_metadata()
chunks = chunk_markdown(markdown_content, file_id, filename, processing_params)
chunk_input, split_by_character, split_by_character_only = self._prepare_lightrag_insert_payload(chunks)
if not chunk_input:
chunk_input = markdown_content
# Clean up existing chunks if any (for re-indexing)
await self.delete_file_chunks_only(db_id, file_id)
# Insert
await rag.ainsert(input=markdown_content, ids=file_id, file_paths=file_path)
await rag.ainsert(
input=chunk_input,
ids=file_id,
file_paths=file_path,
split_by_character=split_by_character,
split_by_character_only=split_by_character_only,
)
await self._ensure_doc_processed(rag, file_id)
logger.info(f"Indexed file {file_id} into LightRAG")
logger.info(
f"Indexed file {file_id} into LightRAG with {len(chunks)} chunks, "
f"chunk_preset_id={processing_params.get('chunk_preset_id')}"
)
# Update status
self.files_meta[file_id]["status"] = FileStatus.INDEXED
@ -358,7 +409,12 @@ class LightRagKB(KnowledgeBase):
try:
# 更新状态为处理中
self.files_meta[file_id]["processing_params"] = params.copy()
resolved_params = resolve_chunk_processing_params(
kb_additional_params=self.databases_meta.get(db_id, {}).get("metadata"),
file_processing_params=self.files_meta[file_id].get("processing_params"),
request_params=params,
)
self.files_meta[file_id]["processing_params"] = resolved_params
self.files_meta[file_id]["status"] = "processing"
await self._save_metadata()
@ -368,12 +424,24 @@ class LightRagKB(KnowledgeBase):
markdown_content = await process_file_to_markdown(file_path, params=params)
markdown_content_lines = markdown_content[:100].replace("\n", " ")
logger.info(f"Markdown content: {markdown_content_lines}...")
filename = file_meta.get("filename") or file_id
chunks = chunk_markdown(markdown_content, file_id, filename, resolved_params)
chunk_input, split_by_character, split_by_character_only = self._prepare_lightrag_insert_payload(chunks)
if not chunk_input:
chunk_input = markdown_content
# 先删除现有的 LightRAG 数据仅删除chunks保留元数据
await self.delete_file_chunks_only(db_id, file_id)
# 使用 LightRAG 重新插入内容
await rag.ainsert(input=markdown_content, ids=file_id, file_paths=file_path)
await rag.ainsert(
input=chunk_input,
ids=file_id,
file_paths=file_path,
split_by_character=split_by_character,
split_by_character_only=split_by_character_only,
)
await self._ensure_doc_processed(rag, file_id)
logger.info(f"Updated {content_type} {file_path} in LightRAG. Done.")

View File

@ -10,11 +10,10 @@ from pymilvus import Collection, CollectionSchema, DataType, FieldSchema, connec
from src import config
from src.knowledge.base import FileStatus, KnowledgeBase
from src.knowledge.chunking.ragflow_like.dispatcher import chunk_markdown
from src.knowledge.chunking.ragflow_like.presets import resolve_chunk_processing_params
from src.knowledge.indexing import process_file_to_markdown
from src.knowledge.utils.kb_utils import (
get_embedding_config,
split_text_into_chunks,
)
from src.knowledge.utils.kb_utils import get_embedding_config
from src.models.embed import OtherEmbedding
from src.utils import hashstr, logger
from src.utils.datetime_utils import utc_isoformat
@ -222,7 +221,7 @@ class MilvusKB(KnowledgeBase):
def _split_text_into_chunks(self, text: str, file_id: str, filename: str, params: dict) -> list[dict]:
"""将文本分割成块"""
return split_text_into_chunks(text, file_id, filename, params)
return chunk_markdown(text, file_id, filename, params)
async def index_file(self, db_id: str, file_id: str, operator_id: str | None = None) -> dict:
"""
@ -281,10 +280,14 @@ class MilvusKB(KnowledgeBase):
self.files_meta[file_id]["updated_at"] = utc_isoformat()
if operator_id:
self.files_meta[file_id]["updated_by"] = operator_id
await self._save_metadata()
# Read processing params inside lock to ensure we get the latest values
params = file_meta.get("processing_params", {}) or {}
params = resolve_chunk_processing_params(
kb_additional_params=self.databases_meta.get(db_id, {}).get("metadata"),
file_processing_params=file_meta.get("processing_params"),
)
self.files_meta[file_id]["processing_params"] = params
await self._save_metadata()
logger.debug(f"[index_file] file_id={file_id}, processing_params={params}")
# Add to processing queue
@ -299,6 +302,7 @@ class MilvusKB(KnowledgeBase):
chunks = self._split_text_into_chunks(markdown_content, file_id, filename, params)
logger.info(
f"Split {filename} into {len(chunks)} chunks with params: "
f"chunk_preset_id={params.get('chunk_preset_id')}, "
f"chunk_size={params.get('chunk_size')}, "
f"chunk_overlap={params.get('chunk_overlap')}, "
f"qa_separator={params.get('qa_separator')}"
@ -391,7 +395,12 @@ class MilvusKB(KnowledgeBase):
try:
# 更新状态为处理中
async with self._metadata_lock:
self.files_meta[file_id]["processing_params"] = params.copy()
resolved_params = resolve_chunk_processing_params(
kb_additional_params=self.databases_meta.get(db_id, {}).get("metadata"),
file_processing_params=self.files_meta[file_id].get("processing_params"),
request_params=params,
)
self.files_meta[file_id]["processing_params"] = resolved_params
self.files_meta[file_id]["status"] = "processing"
await self._save_metadata()
@ -404,7 +413,7 @@ class MilvusKB(KnowledgeBase):
await self.delete_file_chunks_only(db_id, file_id)
# 重新生成 chunks
chunks = self._split_text_into_chunks(markdown_content, file_id, filename, params)
chunks = self._split_text_into_chunks(markdown_content, file_id, filename, resolved_params)
logger.info(f"Split {filename} into {len(chunks)} chunks")
if chunks:

View File

@ -2,6 +2,10 @@ import asyncio
import os
from src.knowledge.base import KBNotFoundError, KnowledgeBase
from src.knowledge.chunking.ragflow_like.presets import (
deep_merge,
ensure_chunk_defaults_in_additional_params,
)
from src.knowledge.factory import KnowledgeBaseFactory
from src.utils import logger
from src.utils.datetime_utils import utc_isoformat
@ -179,7 +183,7 @@ class KnowledgeBaseManager:
if db_info:
# 补充 share_config 和 additional_params
db_info["share_config"] = row.share_config or {"is_shared": True, "accessible_departments": []}
db_info["additional_params"] = row.additional_params or {}
db_info["additional_params"] = ensure_chunk_defaults_in_additional_params(row.additional_params)
all_databases.append(db_info)
return {"databases": all_databases}
@ -310,6 +314,8 @@ class KnowledgeBaseManager:
if share_config is None:
share_config = {"is_shared": True, "accessible_departments": []}
kwargs = ensure_chunk_defaults_in_additional_params(kwargs)
kb_instance = self._get_or_create_kb_instance(kb_type)
db_info = await kb_instance.create_database(database_name, description, embed_info, **kwargs)
db_id = db_info["db_id"]
@ -414,7 +420,7 @@ class KnowledgeBaseManager:
}
# 添加数据库中的附加字段
db_info["additional_params"] = kb.additional_params or {}
db_info["additional_params"] = ensure_chunk_defaults_in_additional_params(kb.additional_params)
db_info["share_config"] = kb.share_config or {"is_shared": True, "accessible_departments": []}
db_info["mindmap"] = kb.mindmap
db_info["sample_questions"] = kb.sample_questions or []
@ -566,6 +572,11 @@ class KnowledgeBaseManager:
"""更新数据库"""
from src.repositories.knowledge_base_repository import KnowledgeBaseRepository
kb_repo = KnowledgeBaseRepository()
kb = await kb_repo.get_by_id(db_id)
if kb is None:
raise ValueError(f"数据库 {db_id} 不存在")
kb_instance = await self._get_kb_for_database(db_id)
kb_instance.update_database(db_id, name, description, llm_info)
@ -576,13 +587,19 @@ class KnowledgeBaseManager:
}
if llm_info is not None:
update_data["llm_info"] = llm_info
if additional_params is not None:
update_data["additional_params"] = additional_params
merged_additional_params = ensure_chunk_defaults_in_additional_params(
deep_merge(kb.additional_params or {}, additional_params)
)
update_data["additional_params"] = merged_additional_params
if db_id in kb_instance.databases_meta:
kb_instance.databases_meta[db_id]["metadata"] = merged_additional_params
if share_config is not None:
update_data["share_config"] = share_config
# 保存到数据库
kb_repo = KnowledgeBaseRepository()
await kb_repo.update(db_id, update_data)
return await self.get_database_info(db_id)

View File

@ -40,6 +40,59 @@ async def test_admin_can_manage_knowledge_databases(test_client, admin_headers,
assert update_response.json()["database"]["description"] == "Updated by pytest"
async def test_create_database_with_chunk_preset(test_client, admin_headers):
db_name = f"pytest_chunk_preset_{uuid.uuid4().hex[:6]}"
payload = {
"database_name": db_name,
"description": "Chunk preset create test",
"embed_model_name": "siliconflow/BAAI/bge-m3",
"kb_type": "milvus",
"additional_params": {"chunk_preset_id": "book"},
}
create_response = await test_client.post("/api/knowledge/databases", json=payload, headers=admin_headers)
assert create_response.status_code == 200, create_response.text
db_id = create_response.json()["db_id"]
info_response = await test_client.get(f"/api/knowledge/databases/{db_id}", headers=admin_headers)
assert info_response.status_code == 200, info_response.text
assert info_response.json()["additional_params"]["chunk_preset_id"] == "book"
delete_response = await test_client.delete(f"/api/knowledge/databases/{db_id}", headers=admin_headers)
assert delete_response.status_code == 200, delete_response.text
async def test_update_database_additional_params_merge_keeps_chunk_preset(
test_client, admin_headers, knowledge_database
):
db_id = knowledge_database["db_id"]
first_update = await test_client.put(
f"/api/knowledge/databases/{db_id}",
json={
"name": knowledge_database["name"],
"description": "update with chunk preset",
"additional_params": {"chunk_preset_id": "qa"},
},
headers=admin_headers,
)
assert first_update.status_code == 200, first_update.text
second_update = await test_client.put(
f"/api/knowledge/databases/{db_id}",
json={
"name": knowledge_database["name"],
"description": "update without additional params",
},
headers=admin_headers,
)
assert second_update.status_code == 200, second_update.text
info_response = await test_client.get(f"/api/knowledge/databases/{db_id}", headers=admin_headers)
assert info_response.status_code == 200, info_response.text
assert info_response.json()["additional_params"]["chunk_preset_id"] == "qa"
async def test_knowledge_routes_enforce_permissions(test_client, standard_user, knowledge_database):
db_id = knowledge_database["db_id"]

View File

@ -0,0 +1,111 @@
from __future__ import annotations
import os
import sys
sys.path.append(os.getcwd())
from src.knowledge.chunking.ragflow_like.dispatcher import chunk_markdown
from src.knowledge.chunking.ragflow_like.nlp import bullets_category
from src.knowledge.chunking.ragflow_like.presets import (
CHUNK_ENGINE_VERSION,
get_chunk_preset_options,
map_to_internal_parser_id,
resolve_chunk_processing_params,
)
def test_general_maps_to_naive() -> None:
assert map_to_internal_parser_id("general") == "naive"
def test_resolve_chunk_processing_params_priority() -> None:
resolved = resolve_chunk_processing_params(
kb_additional_params={
"chunk_preset_id": "book",
"chunk_parser_config": {"chunk_token_num": 300, "delimiter": "\\n"},
},
file_processing_params={
"chunk_preset_id": "qa",
"chunk_parser_config": {"delimiter": "###"},
},
request_params={
"chunk_preset_id": "laws",
"chunk_parser_config": {"chunk_token_num": 666},
"chunk_size": 777,
},
)
assert resolved["chunk_preset_id"] == "laws"
assert resolved["chunk_engine_version"] == CHUNK_ENGINE_VERSION
# legacy chunk_size 在当前实现里会映射为 chunk_token_num
assert resolved["chunk_parser_config"]["chunk_token_num"] == 777
assert resolved["chunk_parser_config"]["delimiter"] == "###"
def test_qa_chunking_from_markdown_headings() -> None:
content = """
# 问题一
这是答案一
## 子问题
这是答案二
""".strip()
chunks = chunk_markdown(
markdown_content=content,
file_id="file_1",
filename="faq.md",
processing_params={"chunk_preset_id": "qa", "chunk_parser_config": {}},
)
assert len(chunks) >= 1
assert "问题:" in chunks[0]["content"]
assert "回答:" in chunks[0]["content"]
def test_book_chunking_hierarchical_merge() -> None:
content = """
第一章 总则
第一节 适用范围
本规范适用于测试场景
第二节 基本原则
应当遵循最小改动原则
""".strip()
chunks = chunk_markdown(
markdown_content=content,
file_id="file_2",
filename="book.txt",
processing_params={"chunk_preset_id": "book", "chunk_parser_config": {"chunk_token_num": 256}},
)
assert len(chunks) >= 1
assert any("第一章" in ck["content"] for ck in chunks)
def test_markdown_heading_has_higher_weight_in_bullet_category() -> None:
sections = [
"# 3.2 个人所得项目及计税、申报方式概括",
"一、关于季节工、临时工等费用税前扣除问题,以下规定继续执行。",
"二、根据现行规定,补贴收入应并入工资薪金所得。",
"(一)从超出国家规定比例支付的补贴,不属于免税福利费。",
]
# 命中 markdown 标题模式BULLET_PATTERN 下标 4应该优先选中该组。
assert bullets_category(sections) == 4
def test_mid_sentence_bullet_marker_should_not_be_treated_as_heading() -> None:
sections = [
"根据前述规则:一、这里是句中枚举,不是章节标题,不能被当成层级。",
"延续上文:(二)这里同样是正文中的枚举表达,不是独立标题。",
"## 3.4 交通补贴的个税处理",
]
assert bullets_category(sections) == 4
def test_chunk_preset_options_include_description() -> None:
options = get_chunk_preset_options()
assert len(options) == 4
assert all(isinstance(option.get("description"), str) and option["description"] for option in options)

View File

@ -4,7 +4,27 @@
<p>调整分块参数可以控制文本的切分方式影响检索质量和文档加载效率</p>
</div>
<a-form :model="tempChunkParams" name="chunkConfig" autocomplete="off" layout="vertical">
<div class="chunk-row">
<a-form-item v-if="showPreset" name="chunk_preset_id">
<template #label>
<span class="chunk-preset-label">
分块策略
<a-tooltip :title="presetDescription">
<QuestionCircleOutlined class="chunk-preset-help-icon" />
</a-tooltip>
</span>
</template>
<a-select
v-model:value="tempChunkParams.chunk_preset_id"
:options="presetOptions"
style="width: 100%"
/>
<p class="param-description">
预设策略对齐 RAGFlowGeneralQABookLaws
<span v-if="allowPresetFollowDefault">留空时沿用知识库默认策略</span>
</p>
</a-form-item>
<div class="chunk-row" v-if="showChunkSizeOverlap">
<a-form-item label="Chunk Size" name="chunk_size">
<a-input-number
v-model:value="tempChunkParams.chunk_size"
@ -42,7 +62,15 @@
</template>
<script setup>
defineProps({
import { computed } from 'vue'
import { QuestionCircleOutlined } from '@ant-design/icons-vue'
import {
CHUNK_PRESET_OPTIONS,
CHUNK_PRESET_LABEL_MAP,
getChunkPresetDescription
} from '@/utils/chunk_presets'
const props = defineProps({
tempChunkParams: {
type: Object,
required: true
@ -50,8 +78,45 @@ defineProps({
showQaSplit: {
type: Boolean,
default: true
},
showChunkSizeOverlap: {
type: Boolean,
default: true
},
showPreset: {
type: Boolean,
default: true
},
allowPresetFollowDefault: {
type: Boolean,
default: false
},
databasePresetId: {
type: String,
default: 'general'
}
})
const presetOptions = computed(() => {
const options = []
const defaultPresetLabel = CHUNK_PRESET_LABEL_MAP[props.databasePresetId] || 'General'
if (props.allowPresetFollowDefault) {
options.push({
value: '',
label: `沿用知识库默认(${defaultPresetLabel}`
})
}
options.push(...CHUNK_PRESET_OPTIONS.map(({ value, label }) => ({ value, label })))
return options
})
const effectivePresetId = computed(
() => props.tempChunkParams.chunk_preset_id || props.databasePresetId || 'general'
)
const presetDescription = computed(() => getChunkPresetDescription(effectivePresetId.value))
</script>
<style scoped>
@ -92,4 +157,16 @@ defineProps({
margin: 4px 0 0 0;
line-height: 1.4;
}
.chunk-preset-label {
display: inline-flex;
align-items: center;
gap: 6px;
}
.chunk-preset-help-icon {
color: var(--gray-500);
cursor: help;
font-size: 14px;
}
</style>

View File

@ -189,7 +189,14 @@
<a-button key="submit" type="primary" @click="handleIndexConfigConfirm">确定</a-button>
</template>
<div class="index-params">
<ChunkParamsConfig :temp-chunk-params="indexParams" :show-qa-split="true" />
<ChunkParamsConfig
:temp-chunk-params="indexParams"
:show-qa-split="true"
:show-chunk-size-overlap="!isLightRAG"
:show-preset="true"
:allow-preset-follow-default="true"
:database-preset-id="store.database?.additional_params?.chunk_preset_id || 'general'"
/>
</div>
</a-modal>
@ -704,8 +711,25 @@ const indexConfigModalTitle = ref('入库参数配置')
const indexParams = ref({
chunk_size: 1000,
chunk_overlap: 200,
qa_separator: ''
qa_separator: '',
chunk_preset_id: ''
})
const buildIndexParamsPayload = () => {
const payload = {}
if (indexParams.value.chunk_preset_id) {
payload.chunk_preset_id = indexParams.value.chunk_preset_id
}
if (isLightRAG.value) {
payload.qa_separator = indexParams.value.qa_separator || ''
return payload
}
return {
...indexParams.value,
...payload
}
}
const currentIndexFileIds = ref([])
const isBatchIndexOperation = ref(false)
@ -1088,12 +1112,6 @@ const handleBatchIndex = async () => {
return
}
if (isLightRAG.value) {
await store.indexFiles(validKeys)
selectedRowKeys.value = []
return
}
currentIndexFileIds.value = [...validKeys]
isBatchIndexOperation.value = true
indexConfigModalTitle.value = '批量入库参数配置'
@ -1172,11 +1190,6 @@ const handleParseFile = async (record) => {
const handleIndexFile = async (record) => {
closePopover(record.file_id)
if (isLightRAG.value) {
await store.indexFiles([record.file_id])
return
}
//
currentIndexFileIds.value = [record.file_id]
isBatchIndexOperation.value = false
@ -1189,7 +1202,8 @@ const handleIndexFile = async (record) => {
Object.assign(indexParams.value, {
chunk_size: 1000,
chunk_overlap: 200,
qa_separator: ''
qa_separator: '',
chunk_preset_id: ''
})
}
@ -1214,7 +1228,7 @@ const handleReindexFile = async (record) => {
const handleIndexConfigConfirm = async () => {
try {
// indexFiles ( params)
const result = await store.indexFiles(currentIndexFileIds.value, indexParams.value)
const result = await store.indexFiles(currentIndexFileIds.value, buildIndexParamsPayload())
if (result) {
currentIndexFileIds.value = []
//
@ -1228,7 +1242,8 @@ const handleIndexConfigConfirm = async () => {
Object.assign(indexParams.value, {
chunk_size: 1000,
chunk_overlap: 200,
qa_separator: ''
qa_separator: '',
chunk_preset_id: ''
})
} else {
// message.error(`: ${result.message}`); // store already shows message
@ -1249,7 +1264,8 @@ const handleIndexConfigCancel = () => {
Object.assign(indexParams.value, {
chunk_size: 1000,
chunk_overlap: 200,
qa_separator: ''
qa_separator: '',
chunk_preset_id: ''
})
}

View File

@ -108,15 +108,17 @@
<div class="col-item">
<div class="setting-label">入库参数配置</div>
<div class="setting-content">
<template v-if="!isGraphBased">
<ChunkParamsConfig :temp-chunk-params="indexParams" :show-qa-split="true" />
</template>
<template v-else>
<div class="lightrag-tip">
<Info :size="14" style="margin-right: 6px" />
<span>LightRAG 将使用默认参数自动入库</span>
</div>
</template>
<ChunkParamsConfig
:temp-chunk-params="indexParams"
:show-qa-split="true"
:show-chunk-size-overlap="!isGraphBased"
:show-preset="true"
:allow-preset-follow-default="true"
:database-preset-id="store.database?.additional_params?.chunk_preset_id || 'general'"
/>
<p v-if="isGraphBased" class="param-description">
LightRAG 按分隔符预切分超长片段仍会按 token 大小继续切分
</p>
</div>
</div>
</div>
@ -564,9 +566,26 @@ const autoIndex = ref(false)
const indexParams = ref({
chunk_size: 1000,
chunk_overlap: 200,
qa_separator: ''
qa_separator: '',
chunk_preset_id: ''
})
const buildAutoIndexParams = () => {
const payload = {}
if (indexParams.value.chunk_preset_id) {
payload.chunk_preset_id = indexParams.value.chunk_preset_id
}
if (isGraphBased.value) {
payload.qa_separator = indexParams.value.qa_separator || ''
return payload
}
return {
...indexParams.value,
...payload
}
}
// QA
const isQaSplitSupported = computed(() => {
const type = kbType.value?.toLowerCase()
@ -990,7 +1009,7 @@ const chunkData = async () => {
const params = { ...chunkParams.value }
if (autoIndex.value) {
params.auto_index = true
Object.assign(params, indexParams.value)
Object.assign(params, buildAutoIndexParams())
}
// _preprocessed_map items (minio urls)
@ -1068,7 +1087,7 @@ const chunkData = async () => {
const params = { ...chunkParams.value, content_hashes }
if (autoIndex.value) {
params.auto_index = true
Object.assign(params, indexParams.value)
Object.assign(params, buildAutoIndexParams())
}
await store.addFiles({
@ -1604,17 +1623,6 @@ const chunkData = async () => {
border-radius: 6px;
}
.lightrag-tip {
display: flex;
align-items: center;
margin-top: 8px;
padding: 8px 12px;
background: var(--main-50);
border-radius: 6px;
font-size: 13px;
color: var(--gray-600);
}
.setting-label .ant-checkbox {
margin-right: 8px;
}

View File

@ -40,6 +40,9 @@
{{ getKbTypeLabel(database.kb_type || 'lightrag') }}
</a-tag>
<a-tag color="blue" size="small">{{ database.embed_info?.name || 'N/A' }}</a-tag>
<a-tag color="cyan" size="small">{{
chunkPresetLabelMap[database.additional_params?.chunk_preset_id || 'general'] || 'General'
}}</a-tag>
</div>
</div>
</div>
@ -81,6 +84,18 @@
>
</a-form-item>
<a-form-item name="chunk_preset_id">
<template #label>
<span class="chunk-preset-label">
分块策略
<a-tooltip :title="editPresetDescription">
<QuestionCircleOutlined class="chunk-preset-help-icon" />
</a-tooltip>
</span>
</template>
<a-select v-model:value="editForm.chunk_preset_id" :options="chunkPresetOptions" />
</a-form-item>
<!-- 仅对 LightRAG 类型显示 LLM 配置 -->
<a-form-item v-if="database.kb_type === 'lightrag'" label="语言模型 (LLM)" name="llm_info">
<ModelSelectorComponent
@ -122,8 +137,13 @@ import { useRouter } from 'vue-router'
import { useDatabaseStore } from '@/stores/database'
import { useUserStore } from '@/stores/user'
import { getKbTypeLabel, getKbTypeColor } from '@/utils/kb_utils'
import {
CHUNK_PRESET_OPTIONS,
CHUNK_PRESET_LABEL_MAP,
getChunkPresetDescription
} from '@/utils/chunk_presets'
import { message } from 'ant-design-vue'
import { LeftOutlined } from '@ant-design/icons-vue'
import { LeftOutlined, QuestionCircleOutlined } from '@ant-design/icons-vue'
import { Pencil, Trash2, Copy } from 'lucide-vue-next'
import { departmentApi } from '@/apis/department_api'
import ModelSelectorComponent from '@/components/ModelSelectorComponent.vue'
@ -220,12 +240,17 @@ const editForm = reactive({
name: '',
description: '',
auto_generate_questions: false,
chunk_preset_id: 'general',
llm_info: {
provider: '',
model_name: ''
}
})
const chunkPresetOptions = CHUNK_PRESET_OPTIONS.map(({ label, value }) => ({ label, value }))
const chunkPresetLabelMap = CHUNK_PRESET_LABEL_MAP
const editPresetDescription = computed(() => getChunkPresetDescription(editForm.chunk_preset_id))
const rules = {
name: [{ required: true, message: '请输入知识库名称' }]
}
@ -238,6 +263,7 @@ const showEditModal = () => {
editForm.description = database.value.description || ''
editForm.auto_generate_questions =
database.value.additional_params?.auto_generate_questions || false
editForm.chunk_preset_id = database.value.additional_params?.chunk_preset_id || 'general'
// LightRAG LLM
if (database.value.kb_type === 'lightrag') {
@ -283,7 +309,8 @@ const handleEditSubmit = () => {
name: editForm.name,
description: editForm.description,
additional_params: {
auto_generate_questions: editForm.auto_generate_questions
auto_generate_questions: editForm.auto_generate_questions,
chunk_preset_id: editForm.chunk_preset_id || 'general'
},
share_config: {
is_shared: finalIsShared,
@ -428,4 +455,16 @@ const deleteDatabase = () => {
align-items: center;
flex-wrap: wrap;
}
.chunk-preset-label {
display: inline-flex;
align-items: center;
gap: 6px;
}
.chunk-preset-help-icon {
color: var(--gray-500);
cursor: help;
font-size: 14px;
}
</style>

View File

@ -0,0 +1,33 @@
export const CHUNK_PRESET_OPTIONS = [
{
value: 'general',
label: 'General',
description: '通用分块:按分隔符和长度切分,适合大多数普通文档。'
},
{
value: 'qa',
label: 'QA',
description: '问答分块:优先抽取问题-回答结构,适合 FAQ、题库、问答手册。'
},
{
value: 'book',
label: 'Book',
description: '书籍分块:强化章节标题识别并做层级合并,适合教材、手册、长章节文档。'
},
{
value: 'laws',
label: 'Laws',
description: '法规分块:按法条层级组织与合并,适合法律法规、制度规范类文本。'
}
]
export const CHUNK_PRESET_LABEL_MAP = Object.fromEntries(
CHUNK_PRESET_OPTIONS.map((item) => [item.value, item.label])
)
export const CHUNK_PRESET_DESCRIPTION_MAP = Object.fromEntries(
CHUNK_PRESET_OPTIONS.map((item) => [item.value, item.description])
)
export const getChunkPresetDescription = (presetId) =>
CHUNK_PRESET_DESCRIPTION_MAP[presetId] || CHUNK_PRESET_DESCRIPTION_MAP.general

View File

@ -56,6 +56,19 @@
placeholder="请选择嵌入模型"
/>
<div class="chunk-preset-title-row">
<h3 style="margin: 0">分块策略</h3>
<a-tooltip :title="selectedPresetDescription">
<QuestionCircleOutlined class="chunk-preset-help-icon" />
</a-tooltip>
</div>
<a-select
v-model:value="newDatabase.chunk_preset_id"
:options="chunkPresetOptions"
style="width: 100%"
size="large"
/>
<!-- 仅对 LightRAG 提供语言选择和LLM选择 -->
<div v-if="newDatabase.kb_type === 'lightrag'">
<h3 style="margin-top: 20px">语言</h3>
@ -197,7 +210,7 @@ import { useRouter, useRoute } from 'vue-router'
import { storeToRefs } from 'pinia'
import { useConfigStore } from '@/stores/config'
import { useDatabaseStore } from '@/stores/database'
import { LockOutlined, InfoCircleOutlined, PlusOutlined } from '@ant-design/icons-vue'
import { LockOutlined, PlusOutlined, QuestionCircleOutlined } from '@ant-design/icons-vue'
import { typeApi } from '@/apis/knowledge_api'
import HeaderComponent from '@/components/HeaderComponent.vue'
import ModelSelectorComponent from '@/components/ModelSelectorComponent.vue'
@ -206,6 +219,7 @@ import ShareConfigForm from '@/components/ShareConfigForm.vue'
import dayjs, { parseToShanghai } from '@/utils/time'
import AiTextarea from '@/components/AiTextarea.vue'
import { getKbTypeLabel, getKbTypeIcon, getKbTypeColor } from '@/utils/kb_utils'
import { CHUNK_PRESET_OPTIONS, getChunkPresetDescription } from '@/utils/chunk_presets'
const route = useRoute()
const router = useRouter()
@ -240,6 +254,8 @@ const languageOptions = [
{ label: '印地语 Hindi', value: 'Hindi' }
]
const chunkPresetOptions = CHUNK_PRESET_OPTIONS.map(({ label, value }) => ({ label, value }))
const createEmptyDatabaseForm = () => ({
name: '',
description: '',
@ -247,6 +263,7 @@ const createEmptyDatabaseForm = () => ({
kb_type: 'milvus',
is_private: false,
storage: '',
chunk_preset_id: 'general',
language: 'Chinese',
llm_info: {
provider: '',
@ -256,6 +273,10 @@ const createEmptyDatabaseForm = () => ({
const newDatabase = reactive(createEmptyDatabaseForm())
const selectedPresetDescription = computed(() =>
getChunkPresetDescription(newDatabase.chunk_preset_id)
)
const llmModelSpec = computed(() => {
const provider = newDatabase.llm_info?.provider || ''
const modelName = newDatabase.llm_info?.model_name || ''
@ -364,7 +385,8 @@ const buildRequestData = () => {
embed_model_name: newDatabase.embed_model_name || configStore.config.embed_model,
kb_type: newDatabase.kb_type,
additional_params: {
is_private: newDatabase.is_private || false
is_private: newDatabase.is_private || false,
chunk_preset_id: newDatabase.chunk_preset_id || 'general'
}
}
@ -429,6 +451,20 @@ onMounted(() => {
<style lang="less" scoped>
.new-database-modal {
.chunk-preset-title-row {
margin-top: 20px;
margin-bottom: 8px;
display: flex;
align-items: center;
gap: 6px;
}
.chunk-preset-help-icon {
color: var(--gray-500);
cursor: help;
font-size: 14px;
}
.kb-type-guide {
margin: 12px 0;
}