feat(knowledge): add ragflow-like chunk preset pipeline
This commit is contained in:
parent
4bd77fce12
commit
60bc408a3e
@ -13,6 +13,7 @@ from starlette.responses import StreamingResponse
|
||||
from src.services.task_service import TaskContext, tasker
|
||||
from server.utils.auth_middleware import get_admin_user, get_required_user
|
||||
from src import config, knowledge_base
|
||||
from src.knowledge.chunking.ragflow_like.presets import ensure_chunk_defaults_in_additional_params
|
||||
from src.knowledge.indexing import SUPPORTED_FILE_EXTENSIONS, is_supported_file_extension, process_file_to_markdown
|
||||
from src.knowledge.utils import calculate_content_hash
|
||||
from src.models.embed import test_all_embedding_models_status, test_embedding_model_status
|
||||
@ -116,6 +117,7 @@ async def create_database(
|
||||
params.pop("reranker_config", None)
|
||||
|
||||
remove_reranker_config(kb_type, additional_params)
|
||||
additional_params = ensure_chunk_defaults_in_additional_params(additional_params)
|
||||
|
||||
embed_info = config.embed_model_names[embed_model_name]
|
||||
# 将Pydantic模型转换为字典以便JSON序列化
|
||||
@ -180,7 +182,7 @@ async def update_database_info(
|
||||
name: str = Body(...),
|
||||
description: str = Body(...),
|
||||
llm_info: dict = Body(None),
|
||||
additional_params: dict = Body({}),
|
||||
additional_params: dict | None = Body(None),
|
||||
share_config: dict = Body(None),
|
||||
current_user: User = Depends(get_admin_user),
|
||||
):
|
||||
@ -190,6 +192,9 @@ async def update_database_info(
|
||||
f"additional_params={additional_params}, share_config={share_config}"
|
||||
)
|
||||
try:
|
||||
if additional_params is not None:
|
||||
additional_params = ensure_chunk_defaults_in_additional_params(additional_params)
|
||||
|
||||
database = await knowledge_base.update_database(
|
||||
db_id,
|
||||
name,
|
||||
@ -267,7 +272,13 @@ async def add_documents(
|
||||
"chunk_size": params.get("chunk_size", 1000),
|
||||
"chunk_overlap": params.get("chunk_overlap", 200),
|
||||
"qa_separator": params.get("qa_separator", ""),
|
||||
"chunk_preset_id": params.get("chunk_preset_id"),
|
||||
"chunk_parser_config": params.get("chunk_parser_config"),
|
||||
}
|
||||
if not indexing_params.get("chunk_preset_id"):
|
||||
indexing_params.pop("chunk_preset_id", None)
|
||||
if not isinstance(indexing_params.get("chunk_parser_config"), dict):
|
||||
indexing_params.pop("chunk_parser_config", None)
|
||||
|
||||
# URL 解析与入库(需白名单验证)
|
||||
if content_type == "url":
|
||||
|
||||
@ -3,6 +3,10 @@ import os
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
from src.knowledge.chunking.ragflow_like.presets import (
|
||||
ensure_chunk_defaults_in_additional_params,
|
||||
resolve_chunk_processing_params,
|
||||
)
|
||||
from src.utils import logger
|
||||
from src.utils.datetime_utils import coerce_any_to_utc_datetime, utc_isoformat
|
||||
|
||||
@ -76,6 +80,7 @@ class KnowledgeBase(ABC):
|
||||
self.databases_meta = {}
|
||||
for db_id, meta in global_databases_meta.items():
|
||||
if meta.get("kb_type") == self.kb_type:
|
||||
normalized_additional_params = ensure_chunk_defaults_in_additional_params(meta.get("additional_params"))
|
||||
self.databases_meta[db_id] = {
|
||||
"name": meta.get("name"),
|
||||
"description": meta.get("description"),
|
||||
@ -83,7 +88,7 @@ class KnowledgeBase(ABC):
|
||||
"embed_info": meta.get("embed_info"),
|
||||
"llm_info": meta.get("llm_info"),
|
||||
"query_params": meta.get("query_params"),
|
||||
"metadata": meta.get("additional_params", {}),
|
||||
"metadata": normalized_additional_params,
|
||||
"created_at": meta.get("created_at"),
|
||||
}
|
||||
|
||||
@ -91,7 +96,14 @@ class KnowledgeBase(ABC):
|
||||
self.files_meta = {}
|
||||
for file_id, meta in files_meta.items():
|
||||
if meta.get("database_id") in self.databases_meta:
|
||||
self.files_meta[file_id] = meta
|
||||
db_id = meta.get("database_id")
|
||||
kb_additional_params = self.databases_meta.get(db_id, {}).get("metadata") or {}
|
||||
normalized_meta = dict(meta)
|
||||
normalized_meta["processing_params"] = resolve_chunk_processing_params(
|
||||
kb_additional_params=kb_additional_params,
|
||||
file_processing_params=meta.get("processing_params"),
|
||||
)
|
||||
self.files_meta[file_id] = normalized_meta
|
||||
|
||||
# 过滤评估基准
|
||||
self.benchmarks_meta = {}
|
||||
@ -199,6 +211,11 @@ class KnowledgeBase(ABC):
|
||||
# Prepare metadata
|
||||
metadata = await prepare_item_metadata(item, content_type, db_id, params=params)
|
||||
file_id = metadata["file_id"]
|
||||
kb_additional_params = self.databases_meta.get(db_id, {}).get("metadata") or {}
|
||||
metadata["processing_params"] = resolve_chunk_processing_params(
|
||||
kb_additional_params=kb_additional_params,
|
||||
file_processing_params=metadata.get("processing_params"),
|
||||
)
|
||||
|
||||
# Initial status
|
||||
metadata["status"] = FileStatus.UPLOADED
|
||||
@ -323,13 +340,16 @@ class KnowledgeBase(ABC):
|
||||
if not params:
|
||||
return
|
||||
|
||||
# Merge or overwrite? Usually merge is safer, or replace.
|
||||
# User might want to change chunk size.
|
||||
current_params = self.files_meta[file_id].get("processing_params", {}) or {}
|
||||
kb_additional_params = self.databases_meta.get(db_id, {}).get("metadata") or {}
|
||||
|
||||
logger.debug(f"[update_file_params] file_id={file_id}, current_params={current_params}, new_params={params}")
|
||||
|
||||
current_params.update(params)
|
||||
current_params = resolve_chunk_processing_params(
|
||||
kb_additional_params=kb_additional_params,
|
||||
file_processing_params=current_params,
|
||||
request_params=params,
|
||||
)
|
||||
|
||||
self.files_meta[file_id]["processing_params"] = current_params
|
||||
self.files_meta[file_id]["updated_at"] = utc_isoformat()
|
||||
@ -414,6 +434,8 @@ class KnowledgeBase(ABC):
|
||||
"""
|
||||
from src.utils import hashstr
|
||||
|
||||
kwargs = ensure_chunk_defaults_in_additional_params(kwargs)
|
||||
|
||||
# 从 kwargs 中获取 is_private 配置
|
||||
is_private = kwargs.get("is_private", False)
|
||||
prefix = "kb_private_" if is_private else "kb_"
|
||||
@ -982,7 +1004,7 @@ class KnowledgeBase(ABC):
|
||||
"embed_info": kb.embed_info,
|
||||
"llm_info": kb.llm_info,
|
||||
"query_params": kb.query_params,
|
||||
"metadata": kb.additional_params or {},
|
||||
"metadata": ensure_chunk_defaults_in_additional_params(kb.additional_params),
|
||||
"created_at": utc_isoformat(kb.created_at) if kb.created_at else utc_isoformat(),
|
||||
}
|
||||
for kb in databases
|
||||
@ -990,6 +1012,7 @@ class KnowledgeBase(ABC):
|
||||
|
||||
self.files_meta = {}
|
||||
for kb in databases:
|
||||
kb_additional_params = self.databases_meta.get(kb.db_id, {}).get("metadata") or {}
|
||||
for record in await file_repo.list_by_db_id(kb.db_id):
|
||||
self.files_meta[record.file_id] = {
|
||||
"file_id": record.file_id,
|
||||
@ -1003,7 +1026,10 @@ class KnowledgeBase(ABC):
|
||||
"content_hash": record.content_hash,
|
||||
"size": record.file_size,
|
||||
"content_type": record.content_type,
|
||||
"processing_params": record.processing_params,
|
||||
"processing_params": resolve_chunk_processing_params(
|
||||
kb_additional_params=kb_additional_params,
|
||||
file_processing_params=record.processing_params,
|
||||
),
|
||||
"is_folder": record.is_folder,
|
||||
"error": record.error_message,
|
||||
"created_by": record.created_by,
|
||||
|
||||
1
src/knowledge/chunking/__init__.py
Normal file
1
src/knowledge/chunking/__init__.py
Normal file
@ -0,0 +1 @@
|
||||
__all__ = []
|
||||
3
src/knowledge/chunking/ragflow_like/__init__.py
Normal file
3
src/knowledge/chunking/ragflow_like/__init__.py
Normal file
@ -0,0 +1,3 @@
|
||||
from src.knowledge.chunking.ragflow_like.dispatcher import chunk_file, chunk_markdown
|
||||
|
||||
__all__ = ["chunk_file", "chunk_markdown"]
|
||||
58
src/knowledge/chunking/ragflow_like/dispatcher.py
Normal file
58
src/knowledge/chunking/ragflow_like/dispatcher.py
Normal file
@ -0,0 +1,58 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from src.knowledge.chunking.ragflow_like.parsers import book, general, laws, qa
|
||||
from src.knowledge.chunking.ragflow_like.presets import map_to_internal_parser_id, normalize_chunk_preset_id
|
||||
|
||||
|
||||
def _build_chunk_records(text_chunks: list[str], file_id: str, filename: str) -> list[dict[str, Any]]:
|
||||
records: list[dict[str, Any]] = []
|
||||
|
||||
for idx, chunk_content in enumerate(text_chunks):
|
||||
text = (chunk_content or "").strip()
|
||||
if not text:
|
||||
continue
|
||||
|
||||
records.append(
|
||||
{
|
||||
"id": f"{file_id}_chunk_{idx}",
|
||||
"content": text,
|
||||
"file_id": file_id,
|
||||
"filename": filename,
|
||||
"chunk_index": idx,
|
||||
"source": filename,
|
||||
"chunk_id": f"{file_id}_chunk_{idx}",
|
||||
}
|
||||
)
|
||||
|
||||
return records
|
||||
|
||||
|
||||
def _dispatch_markdown_parser(preset_id: str, filename: str, markdown_content: str, parser_config: dict[str, Any]) -> list[str]:
|
||||
parser_id = map_to_internal_parser_id(preset_id)
|
||||
|
||||
if parser_id == "naive":
|
||||
return general.chunk_markdown(markdown_content, parser_config)
|
||||
if parser_id == "qa":
|
||||
return qa.chunk_markdown(filename, markdown_content, parser_config)
|
||||
if parser_id == "book":
|
||||
return book.chunk_markdown(markdown_content, parser_config)
|
||||
if parser_id == "laws":
|
||||
return laws.chunk_markdown(filename, markdown_content, parser_config)
|
||||
|
||||
return general.chunk_markdown(markdown_content, parser_config)
|
||||
|
||||
|
||||
def chunk_markdown(markdown_content: str, file_id: str, filename: str, processing_params: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
params = dict(processing_params or {})
|
||||
preset_id = normalize_chunk_preset_id(params.get("chunk_preset_id"))
|
||||
parser_config = params.get("chunk_parser_config") if isinstance(params.get("chunk_parser_config"), dict) else {}
|
||||
|
||||
text_chunks = _dispatch_markdown_parser(preset_id, filename, markdown_content, parser_config)
|
||||
return _build_chunk_records(text_chunks, file_id, filename)
|
||||
|
||||
|
||||
def chunk_file(file_content: str, file_id: str, filename: str, processing_params: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
# 当前链路中入库前均已转换为 markdown,因此与 chunk_markdown 保持同实现。
|
||||
return chunk_markdown(file_content, file_id, filename, processing_params)
|
||||
536
src/knowledge/chunking/ragflow_like/nlp.py
Normal file
536
src/knowledge/chunking/ragflow_like/nlp.py
Normal file
@ -0,0 +1,536 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
import re
|
||||
from collections import Counter
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
|
||||
BULLET_PATTERN = [
|
||||
[
|
||||
r"第[零一二三四五六七八九十百0-9]+(分?编|部分)",
|
||||
r"第[零一二三四五六七八九十百0-9]+章",
|
||||
r"第[零一二三四五六七八九十百0-9]+节",
|
||||
r"第[零一二三四五六七八九十百0-9]+条",
|
||||
r"[\((][零一二三四五六七八九十百]+[\))]",
|
||||
],
|
||||
[
|
||||
r"第[0-9]+章",
|
||||
r"第[0-9]+节",
|
||||
r"[0-9]{,2}[\. 、]",
|
||||
r"[0-9]{,2}\.[0-9]{,2}[^a-zA-Z/%~-]",
|
||||
r"[0-9]{,2}\.[0-9]{,2}\.[0-9]{,2}",
|
||||
r"[0-9]{,2}\.[0-9]{,2}\.[0-9]{,2}\.[0-9]{,2}",
|
||||
],
|
||||
[
|
||||
r"第[零一二三四五六七八九十百0-9]+章",
|
||||
r"第[零一二三四五六七八九十百0-9]+节",
|
||||
r"[零一二三四五六七八九十百]+[ 、]",
|
||||
r"[\((][零一二三四五六七八九十百]+[\))]",
|
||||
r"[\((][0-9]{,2}[\))]",
|
||||
],
|
||||
[
|
||||
r"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)",
|
||||
r"Chapter (I+V?|VI*|XI|IX|X)",
|
||||
r"Section [0-9]+",
|
||||
r"Article [0-9]+",
|
||||
],
|
||||
[
|
||||
r"^#[^#]",
|
||||
r"^##[^#]",
|
||||
r"^###.*",
|
||||
r"^####.*",
|
||||
r"^#####.*",
|
||||
r"^######.*",
|
||||
],
|
||||
]
|
||||
|
||||
MARKDOWN_BULLET_GROUP_INDEX = 4
|
||||
|
||||
|
||||
def count_tokens(text: str) -> int:
|
||||
"""近似 token 计数,避免引入额外依赖。"""
|
||||
if not text:
|
||||
return 0
|
||||
# 英文单词 + 数字 + CJK 单字
|
||||
parts = re.findall(r"[A-Za-z0-9_]+|[\u4e00-\u9fff]", text)
|
||||
return max(1, len(parts)) if text.strip() else 0
|
||||
|
||||
|
||||
def random_choices(arr: list[str], k: int) -> list[str]:
|
||||
if not arr:
|
||||
return []
|
||||
return random.choices(arr, k=min(len(arr), k))
|
||||
|
||||
|
||||
def is_english(texts: str | list[str]) -> bool:
|
||||
if not texts:
|
||||
return False
|
||||
|
||||
patt = re.compile(r"[`a-zA-Z0-9\s.,':;/\"?<>!\(\)\-]+")
|
||||
if isinstance(texts, str):
|
||||
seq = [texts]
|
||||
else:
|
||||
seq = [t for t in texts if isinstance(t, str) and t.strip()]
|
||||
|
||||
if not seq:
|
||||
return False
|
||||
|
||||
hits = sum(1 for t in seq if patt.fullmatch(t.strip()))
|
||||
return (hits / len(seq)) > 0.8
|
||||
|
||||
|
||||
def not_bullet(line: str) -> bool:
|
||||
patt = [
|
||||
r"0",
|
||||
r"[0-9]+ +[0-9~个只-]",
|
||||
r"[0-9]+\.{2,}",
|
||||
]
|
||||
return any(re.match(p, line) for p in patt)
|
||||
|
||||
|
||||
def is_probable_heading_line(line: str) -> bool:
|
||||
text = (line or "").strip()
|
||||
if not text:
|
||||
return False
|
||||
|
||||
if re.match(r"^#{1,6}\s+\S", text):
|
||||
return True
|
||||
|
||||
# 表格/HTML 残留通常不是标题。
|
||||
if re.search(r"</?(table|tr|td|th|caption|tbody|thead)[^>]*>", text, flags=re.IGNORECASE):
|
||||
return False
|
||||
|
||||
# 超长行基本是正文或条款,不是章节标题。
|
||||
if len(text) > 96:
|
||||
return False
|
||||
if count_tokens(text) > 72:
|
||||
return False
|
||||
|
||||
# 标题前段通常不会出现明显句号/逗号;出现则大概率是正文。
|
||||
if re.search(r"[,。;!?!?::]", text[:24]):
|
||||
return False
|
||||
|
||||
if text.endswith(("。", ";", "!", "!", "?", "?")) and len(text) > 20:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def _is_mid_sentence_bullet(line: str) -> bool:
|
||||
text = (line or "").strip()
|
||||
if not text:
|
||||
return False
|
||||
|
||||
if re.match(r"^#{1,6}\s+\S", text):
|
||||
return False
|
||||
|
||||
marker = re.search(
|
||||
r"([一二三四五六七八九十百]+、|[\((][一二三四五六七八九十百]+[\))]|[0-9]{1,2}[\.、])",
|
||||
text,
|
||||
)
|
||||
if not marker:
|
||||
return False
|
||||
|
||||
if marker.start() == 0:
|
||||
return False
|
||||
|
||||
prev = text[marker.start() - 1]
|
||||
return prev not in {"#", "\n"}
|
||||
|
||||
|
||||
def bullets_category(sections: list[str]) -> int:
|
||||
hits: list[float] = [0.0] * len(BULLET_PATTERN)
|
||||
|
||||
def bullet_weight(group_idx: int, line: str) -> float:
|
||||
# 对 markdown 标题候选增加权重,避免“正文里的 一、/(一)”压过真正的 # 标题层级。
|
||||
if group_idx != MARKDOWN_BULLET_GROUP_INDEX:
|
||||
return 1.0
|
||||
|
||||
heading = line.strip()
|
||||
if not re.match(r"^#{1,6}\s+\S", heading):
|
||||
return 1.0
|
||||
|
||||
level = len(heading) - len(heading.lstrip("#"))
|
||||
if level <= 2:
|
||||
return 4.0
|
||||
if level <= 4:
|
||||
return 3.0
|
||||
return 2.0
|
||||
|
||||
for i, pro in enumerate(BULLET_PATTERN):
|
||||
for sec in sections:
|
||||
sec = sec.strip()
|
||||
for p in pro:
|
||||
if re.match(p, sec) and not not_bullet(sec):
|
||||
w = bullet_weight(i, sec)
|
||||
if _is_mid_sentence_bullet(sec):
|
||||
w *= 0.1
|
||||
if i != MARKDOWN_BULLET_GROUP_INDEX and not is_probable_heading_line(sec):
|
||||
w *= 0.2
|
||||
hits[i] += w
|
||||
break
|
||||
maximum = 0
|
||||
res = -1
|
||||
for i, hit in enumerate(hits):
|
||||
if hit <= maximum:
|
||||
continue
|
||||
res = i
|
||||
maximum = hit
|
||||
return res
|
||||
|
||||
|
||||
def _get_text(section: str | tuple[str, str]) -> str:
|
||||
if isinstance(section, str):
|
||||
return section.strip()
|
||||
return (section[0] or "").strip()
|
||||
|
||||
|
||||
def remove_contents_table(sections: list[str] | list[tuple[str, str]], eng: bool = False) -> None:
|
||||
i = 0
|
||||
while i < len(sections):
|
||||
line = re.sub(r"( | |\u3000)+", "", _get_text(sections[i]).split("@@")[0], flags=re.IGNORECASE)
|
||||
if not re.match(r"(contents|目录|目次|tableofcontents|致谢|acknowledge)$", line, flags=re.IGNORECASE):
|
||||
i += 1
|
||||
continue
|
||||
|
||||
sections.pop(i)
|
||||
if i >= len(sections):
|
||||
break
|
||||
|
||||
prefix = _get_text(sections[i])[:3] if not eng else " ".join(_get_text(sections[i]).split()[:2])
|
||||
while not prefix and i < len(sections):
|
||||
sections.pop(i)
|
||||
if i >= len(sections):
|
||||
break
|
||||
prefix = _get_text(sections[i])[:3] if not eng else " ".join(_get_text(sections[i]).split()[:2])
|
||||
|
||||
if i >= len(sections) or not prefix:
|
||||
break
|
||||
|
||||
sections.pop(i)
|
||||
if i >= len(sections):
|
||||
break
|
||||
|
||||
for j in range(i, min(i + 128, len(sections))):
|
||||
if not re.match(re.escape(prefix), _get_text(sections[j])):
|
||||
continue
|
||||
for _ in range(i, j):
|
||||
sections.pop(i)
|
||||
break
|
||||
|
||||
|
||||
def make_colon_as_title(sections: list[str] | list[tuple[str, str]]) -> list[str] | list[tuple[str, str]]:
|
||||
if not sections:
|
||||
return sections
|
||||
if isinstance(sections[0], str):
|
||||
return sections
|
||||
|
||||
i = 0
|
||||
while i < len(sections):
|
||||
text, layout = sections[i]
|
||||
i += 1
|
||||
text = text.split("@")[0].strip()
|
||||
if not text or text[-1] not in "::":
|
||||
continue
|
||||
|
||||
rev = text[::-1]
|
||||
arr = re.split(r"([。?!!?;;]| \.)", rev)
|
||||
if len(arr) < 2 or len(arr[1]) < 32:
|
||||
continue
|
||||
|
||||
sections.insert(i - 1, (arr[0][::-1], "title"))
|
||||
i += 1
|
||||
|
||||
return sections
|
||||
|
||||
|
||||
def not_title(text: str) -> bool:
|
||||
if re.match(r"第[零一二三四五六七八九十百0-9]+条", text):
|
||||
return False
|
||||
if len(text.split()) > 12 or (" " not in text and len(text) >= 32):
|
||||
return True
|
||||
return bool(re.search(r"[,;,。;!!]", text))
|
||||
|
||||
|
||||
def tree_merge(bull: int, sections: list[str] | list[tuple[str, str]], depth: int) -> list[str]:
|
||||
if not sections or bull < 0:
|
||||
return [s if isinstance(s, str) else s[0] for s in sections]
|
||||
|
||||
if isinstance(sections[0], str):
|
||||
typed_sections: list[tuple[str, str]] = [(s, "") for s in sections]
|
||||
else:
|
||||
typed_sections = sections # type: ignore[assignment]
|
||||
|
||||
typed_sections = [
|
||||
(t, o)
|
||||
for t, o in typed_sections
|
||||
if t and len(t.split("@")[0].strip()) > 1 and not re.match(r"[0-9]+$", t.split("@")[0].strip())
|
||||
]
|
||||
|
||||
def get_level(section: tuple[str, str]) -> tuple[int, str]:
|
||||
text, layout = section
|
||||
text = re.sub(r"\u3000", " ", text).strip()
|
||||
|
||||
for i, patt in enumerate(BULLET_PATTERN[bull]):
|
||||
if re.match(patt, text) and is_probable_heading_line(text):
|
||||
return i + 1, text
|
||||
|
||||
if re.search(r"(title|head)", layout) and not not_title(text):
|
||||
return len(BULLET_PATTERN[bull]) + 1, text
|
||||
|
||||
return len(BULLET_PATTERN[bull]) + 2, text
|
||||
|
||||
lines: list[tuple[int, str]] = []
|
||||
level_set: set[int] = set()
|
||||
for section in typed_sections:
|
||||
level, text = get_level(section)
|
||||
if not text.strip("\n"):
|
||||
continue
|
||||
lines.append((level, text))
|
||||
level_set.add(level)
|
||||
|
||||
if not lines:
|
||||
return []
|
||||
|
||||
sorted_levels = sorted(level_set)
|
||||
target_level = sorted_levels[depth - 1] if depth <= len(sorted_levels) else sorted_levels[-1]
|
||||
|
||||
max_body_level = len(BULLET_PATTERN[bull]) + 2
|
||||
if target_level == max_body_level:
|
||||
target_level = sorted_levels[-2] if len(sorted_levels) > 1 else sorted_levels[0]
|
||||
|
||||
root = Node(level=0, depth=target_level, texts=[])
|
||||
root.build_tree(lines)
|
||||
return [item for item in root.get_tree() if item]
|
||||
|
||||
|
||||
def hierarchical_merge(bull: int, sections: list[str] | list[tuple[str, str]], depth: int) -> list[list[str]]:
|
||||
if not sections or bull < 0:
|
||||
return []
|
||||
|
||||
if isinstance(sections[0], str):
|
||||
typed_sections: list[tuple[str, str]] = [(s, "") for s in sections]
|
||||
else:
|
||||
typed_sections = sections # type: ignore[assignment]
|
||||
|
||||
typed_sections = [
|
||||
(t, o)
|
||||
for t, o in typed_sections
|
||||
if t and len(t.split("@")[0].strip()) > 1 and not re.match(r"[0-9]+$", t.split("@")[0].strip())
|
||||
]
|
||||
|
||||
bullets_size = len(BULLET_PATTERN[bull])
|
||||
levels: list[list[int]] = [[] for _ in range(bullets_size + 2)]
|
||||
|
||||
for i, (text, layout) in enumerate(typed_sections):
|
||||
for j, patt in enumerate(BULLET_PATTERN[bull]):
|
||||
if re.match(patt, text.strip()) and is_probable_heading_line(text):
|
||||
levels[j].append(i)
|
||||
break
|
||||
else:
|
||||
if re.search(r"(title|head)", layout) and not not_title(text):
|
||||
levels[bullets_size].append(i)
|
||||
else:
|
||||
levels[bullets_size + 1].append(i)
|
||||
|
||||
pure_sections = [t for t, _ in typed_sections]
|
||||
|
||||
def binary_search(arr: list[int], target: int) -> int:
|
||||
if not arr:
|
||||
return -1
|
||||
if target > arr[-1]:
|
||||
return len(arr) - 1
|
||||
if target < arr[0]:
|
||||
return -1
|
||||
|
||||
s, e = 0, len(arr)
|
||||
while e - s > 1:
|
||||
mid = (e + s) // 2
|
||||
if target > arr[mid]:
|
||||
s = mid
|
||||
elif target < arr[mid]:
|
||||
e = mid
|
||||
else:
|
||||
return mid
|
||||
return s
|
||||
|
||||
cks: list[list[int]] = []
|
||||
readed = [False] * len(pure_sections)
|
||||
levels = list(reversed(levels))
|
||||
for i, arr in enumerate(levels[:depth]):
|
||||
for j in arr:
|
||||
if readed[j]:
|
||||
continue
|
||||
readed[j] = True
|
||||
cks.append([j])
|
||||
if i + 1 == len(levels) - 1:
|
||||
continue
|
||||
|
||||
for ii in range(i + 1, len(levels)):
|
||||
jj = binary_search(levels[ii], j)
|
||||
if jj < 0:
|
||||
continue
|
||||
if levels[ii][jj] > cks[-1][-1]:
|
||||
cks[-1].pop(-1)
|
||||
cks[-1].append(levels[ii][jj])
|
||||
|
||||
for ii in cks[-1]:
|
||||
readed[ii] = True
|
||||
|
||||
if not cks:
|
||||
return []
|
||||
|
||||
for i in range(len(cks)):
|
||||
cks[i] = [pure_sections[j] for j in reversed(cks[i])]
|
||||
|
||||
res: list[list[str]] = [[]]
|
||||
num = [0]
|
||||
for ck in cks:
|
||||
if len(ck) == 1:
|
||||
n = count_tokens(re.sub(r"@@[0-9]+.*", "", ck[0]))
|
||||
if n + num[-1] < 218:
|
||||
res[-1].append(ck[0])
|
||||
num[-1] += n
|
||||
continue
|
||||
res.append(ck)
|
||||
num.append(n)
|
||||
continue
|
||||
res.append(ck)
|
||||
num.append(218)
|
||||
|
||||
return [chunk for chunk in res if chunk]
|
||||
|
||||
|
||||
def _remove_pdf_tags(text: str) -> str:
|
||||
return re.sub(r"@@[0-9-]+\t[0-9.\t]+##", "", text or "")
|
||||
|
||||
|
||||
def _extract_custom_delimiters(delimiter: str) -> list[str]:
|
||||
return [m.group(1) for m in re.finditer(r"`([^`]+)`", delimiter or "")]
|
||||
|
||||
|
||||
def naive_merge(
|
||||
sections: str | list[str] | list[tuple[str, str]],
|
||||
chunk_token_num: int = 128,
|
||||
delimiter: str = "\n。;!?",
|
||||
overlapped_percent: int = 0,
|
||||
) -> list[str]:
|
||||
if not sections:
|
||||
return []
|
||||
|
||||
if isinstance(sections, str):
|
||||
typed_sections: list[tuple[str, str]] = [(sections, "")]
|
||||
elif isinstance(sections[0], str):
|
||||
typed_sections = [(s, "") for s in sections] # type: ignore[index]
|
||||
else:
|
||||
typed_sections = sections # type: ignore[assignment]
|
||||
|
||||
chunk_token_num = max(int(chunk_token_num or 0), 0)
|
||||
overlap = max(0, min(int(overlapped_percent or 0), 99))
|
||||
|
||||
custom_delimiters = _extract_custom_delimiters(delimiter)
|
||||
if custom_delimiters:
|
||||
pattern = "|".join(re.escape(t) for t in sorted(set(custom_delimiters), key=len, reverse=True))
|
||||
chunks: list[str] = []
|
||||
for sec, pos in typed_sections:
|
||||
split_sec = re.split(rf"({pattern})", sec, flags=re.DOTALL)
|
||||
for sub in split_sec:
|
||||
if re.fullmatch(pattern, sub or ""):
|
||||
continue
|
||||
text = "\n" + sub
|
||||
local_pos = pos if count_tokens(text) >= 8 else ""
|
||||
if local_pos and local_pos not in text:
|
||||
text += local_pos
|
||||
if text.strip():
|
||||
chunks.append(text)
|
||||
return chunks
|
||||
|
||||
if chunk_token_num <= 0:
|
||||
merged = "\n".join(sec for sec, _ in typed_sections if sec and sec.strip())
|
||||
return [merged] if merged.strip() else []
|
||||
|
||||
chunks = [""]
|
||||
token_nums = [0]
|
||||
|
||||
def add_chunk(text: str, pos: str) -> None:
|
||||
tnum = count_tokens(text)
|
||||
local_pos = pos or ""
|
||||
if tnum < 8:
|
||||
local_pos = ""
|
||||
|
||||
threshold = chunk_token_num * (100 - overlap) / 100.0
|
||||
if chunks[-1] == "" or token_nums[-1] > threshold:
|
||||
if chunks:
|
||||
prev = _remove_pdf_tags(chunks[-1])
|
||||
start = int(len(prev) * (100 - overlap) / 100.0)
|
||||
text = prev[start:] + text
|
||||
if local_pos and local_pos not in text:
|
||||
text += local_pos
|
||||
chunks.append(text)
|
||||
token_nums.append(tnum)
|
||||
else:
|
||||
if local_pos and local_pos not in chunks[-1]:
|
||||
text += local_pos
|
||||
chunks[-1] += text
|
||||
token_nums[-1] += tnum
|
||||
|
||||
for sec, pos in typed_sections:
|
||||
if not sec:
|
||||
continue
|
||||
add_chunk("\n" + sec, pos)
|
||||
|
||||
return [chunk for chunk in chunks if chunk.strip()]
|
||||
|
||||
|
||||
@dataclass
|
||||
class Node:
|
||||
level: int
|
||||
depth: int = -1
|
||||
texts: list[str] = field(default_factory=list)
|
||||
children: list["Node"] = field(default_factory=list)
|
||||
|
||||
def add_child(self, child_node: "Node") -> None:
|
||||
self.children.append(child_node)
|
||||
|
||||
def add_text(self, text: str) -> None:
|
||||
self.texts.append(text)
|
||||
|
||||
def build_tree(self, lines: list[tuple[int, str]]) -> "Node":
|
||||
stack: list[Node] = [self]
|
||||
for level, text in lines:
|
||||
if self.depth != -1 and level > self.depth:
|
||||
stack[-1].add_text(text)
|
||||
continue
|
||||
|
||||
while len(stack) > 1 and level <= stack[-1].level:
|
||||
stack.pop()
|
||||
|
||||
node = Node(level=level, texts=[text])
|
||||
stack[-1].add_child(node)
|
||||
stack.append(node)
|
||||
|
||||
return self
|
||||
|
||||
def get_tree(self) -> list[str]:
|
||||
tree_list: list[str] = []
|
||||
self._dfs(self, tree_list, [])
|
||||
return tree_list
|
||||
|
||||
def _dfs(self, node: "Node", tree_list: list[str], titles: list[str]) -> None:
|
||||
level = node.level
|
||||
texts = node.texts
|
||||
child = node.children
|
||||
|
||||
if level == 0 and texts:
|
||||
tree_list.append("\n".join(titles + texts))
|
||||
|
||||
path_titles = titles + texts if 1 <= level <= self.depth else titles
|
||||
|
||||
if level > self.depth and texts:
|
||||
tree_list.append("\n".join(path_titles + texts))
|
||||
elif not child and (1 <= level <= self.depth):
|
||||
tree_list.append("\n".join(path_titles))
|
||||
|
||||
for c in child:
|
||||
self._dfs(c, tree_list, path_titles)
|
||||
3
src/knowledge/chunking/ragflow_like/parsers/__init__.py
Normal file
3
src/knowledge/chunking/ragflow_like/parsers/__init__.py
Normal file
@ -0,0 +1,3 @@
|
||||
from src.knowledge.chunking.ragflow_like.parsers import book, general, laws, qa
|
||||
|
||||
__all__ = ["general", "qa", "book", "laws"]
|
||||
61
src/knowledge/chunking/ragflow_like/parsers/book.py
Normal file
61
src/knowledge/chunking/ragflow_like/parsers/book.py
Normal file
@ -0,0 +1,61 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from src.knowledge.chunking.ragflow_like import nlp
|
||||
|
||||
|
||||
def _unescape_delimiter(delimiter: str) -> str:
|
||||
return delimiter.replace("\\n", "\n").replace("\\r", "\r").replace("\\t", "\t").replace("\\\\", "\\")
|
||||
|
||||
|
||||
def _iter_sections(markdown_content: str) -> list[tuple[str, str]]:
|
||||
sections: list[tuple[str, str]] = []
|
||||
for line in (markdown_content or "").splitlines():
|
||||
text = line.strip()
|
||||
if not text:
|
||||
continue
|
||||
sections.append((text, ""))
|
||||
|
||||
if not sections and markdown_content and markdown_content.strip():
|
||||
sections.append((markdown_content.strip(), ""))
|
||||
|
||||
return sections
|
||||
|
||||
|
||||
def chunk_markdown(markdown_content: str, parser_config: dict[str, Any] | None = None) -> list[str]:
|
||||
parser_config = parser_config or {}
|
||||
|
||||
delimiter = _unescape_delimiter(str(parser_config.get("delimiter", "\n") or "\n"))
|
||||
chunk_token_num = int(parser_config.get("chunk_token_num", 512) or 512)
|
||||
overlapped_percent = int(parser_config.get("overlapped_percent", 0) or 0)
|
||||
|
||||
sections = _iter_sections(markdown_content)
|
||||
if not sections:
|
||||
return []
|
||||
|
||||
section_texts = [text for text, _ in sections]
|
||||
nlp.remove_contents_table(sections, eng=nlp.is_english(nlp.random_choices(section_texts, k=200)))
|
||||
nlp.make_colon_as_title(sections)
|
||||
|
||||
bull = nlp.bullets_category([t for t in nlp.random_choices([t for t, _ in sections], k=100)])
|
||||
|
||||
if bull >= 0:
|
||||
chunks = ["\n".join(ck) for ck in nlp.hierarchical_merge(bull, sections, depth=5)]
|
||||
else:
|
||||
chunks = nlp.naive_merge(
|
||||
sections,
|
||||
chunk_token_num=chunk_token_num,
|
||||
delimiter=delimiter,
|
||||
overlapped_percent=overlapped_percent,
|
||||
)
|
||||
|
||||
if chunks:
|
||||
return chunks
|
||||
|
||||
return nlp.naive_merge(
|
||||
sections,
|
||||
chunk_token_num=chunk_token_num,
|
||||
delimiter=delimiter,
|
||||
overlapped_percent=overlapped_percent,
|
||||
)
|
||||
46
src/knowledge/chunking/ragflow_like/parsers/general.py
Normal file
46
src/knowledge/chunking/ragflow_like/parsers/general.py
Normal file
@ -0,0 +1,46 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from src.knowledge.chunking.ragflow_like import nlp
|
||||
|
||||
|
||||
def _unescape_delimiter(delimiter: str) -> str:
|
||||
return delimiter.replace("\\n", "\n").replace("\\r", "\r").replace("\\t", "\t").replace("\\\\", "\\")
|
||||
|
||||
|
||||
def _iter_sections(markdown_content: str, delimiter: str) -> list[tuple[str, str]]:
|
||||
sections: list[tuple[str, str]] = []
|
||||
text = markdown_content or ""
|
||||
if delimiter and delimiter not in {"\n", "\r\n"} and "`" not in delimiter:
|
||||
for part in text.split(delimiter):
|
||||
block = part.strip()
|
||||
if block:
|
||||
sections.append((block, ""))
|
||||
else:
|
||||
for line in text.splitlines():
|
||||
block = line.strip()
|
||||
if not block:
|
||||
continue
|
||||
sections.append((block, ""))
|
||||
|
||||
if not sections and text.strip():
|
||||
sections.append((text.strip(), ""))
|
||||
|
||||
return sections
|
||||
|
||||
|
||||
def chunk_markdown(markdown_content: str, parser_config: dict[str, Any] | None = None) -> list[str]:
|
||||
parser_config = parser_config or {}
|
||||
|
||||
delimiter = _unescape_delimiter(str(parser_config.get("delimiter", "\n") or "\n"))
|
||||
chunk_token_num = int(parser_config.get("chunk_token_num", 512) or 512)
|
||||
overlapped_percent = int(parser_config.get("overlapped_percent", 0) or 0)
|
||||
|
||||
sections = _iter_sections(markdown_content, delimiter)
|
||||
return nlp.naive_merge(
|
||||
sections,
|
||||
chunk_token_num=chunk_token_num,
|
||||
delimiter=delimiter,
|
||||
overlapped_percent=overlapped_percent,
|
||||
)
|
||||
85
src/knowledge/chunking/ragflow_like/parsers/laws.py
Normal file
85
src/knowledge/chunking/ragflow_like/parsers/laws.py
Normal file
@ -0,0 +1,85 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from src.knowledge.chunking.ragflow_like import nlp
|
||||
|
||||
|
||||
def _unescape_delimiter(delimiter: str) -> str:
|
||||
return delimiter.replace("\\n", "\n").replace("\\r", "\r").replace("\\t", "\t").replace("\\\\", "\\")
|
||||
|
||||
|
||||
def _iter_lines(markdown_content: str) -> list[str]:
|
||||
return [line.strip() for line in (markdown_content or "").splitlines() if line.strip()]
|
||||
|
||||
|
||||
def _docx_heading_tree(markdown_content: str) -> list[str]:
|
||||
lines: list[tuple[int, str]] = []
|
||||
level_set: set[int] = set()
|
||||
|
||||
for raw in (markdown_content or "").splitlines():
|
||||
text = raw.strip()
|
||||
if not text:
|
||||
continue
|
||||
|
||||
heading_match = re.match(r"^(#{1,6})\s+(.*)$", text)
|
||||
if heading_match:
|
||||
level = len(heading_match.group(1))
|
||||
value = heading_match.group(2).strip()
|
||||
else:
|
||||
level = 99
|
||||
value = text
|
||||
|
||||
if not value:
|
||||
continue
|
||||
|
||||
lines.append((level, value))
|
||||
level_set.add(level)
|
||||
|
||||
if not lines:
|
||||
return []
|
||||
|
||||
sorted_levels = sorted(level_set)
|
||||
h2_level = sorted_levels[1] if len(sorted_levels) > 1 else 1
|
||||
h2_level = sorted_levels[-2] if h2_level == sorted_levels[-1] and len(sorted_levels) > 2 else h2_level
|
||||
|
||||
root = nlp.Node(level=0, depth=h2_level, texts=[])
|
||||
root.build_tree(lines)
|
||||
return [element for element in root.get_tree() if element]
|
||||
|
||||
|
||||
def chunk_markdown(filename: str, markdown_content: str, parser_config: dict[str, Any] | None = None) -> list[str]:
|
||||
parser_config = parser_config or {}
|
||||
|
||||
delimiter = _unescape_delimiter(str(parser_config.get("delimiter", "\n") or "\n"))
|
||||
chunk_token_num = int(parser_config.get("chunk_token_num", 512) or 512)
|
||||
overlapped_percent = int(parser_config.get("overlapped_percent", 0) or 0)
|
||||
|
||||
if re.search(r"\.docx$", filename or "", re.IGNORECASE):
|
||||
chunks = _docx_heading_tree(markdown_content)
|
||||
if chunks:
|
||||
return chunks
|
||||
|
||||
sections = _iter_lines(markdown_content)
|
||||
if not sections:
|
||||
return []
|
||||
|
||||
eng = nlp.is_english(sections)
|
||||
nlp.remove_contents_table(sections, eng=eng)
|
||||
|
||||
typed_sections = [(s, "") for s in sections]
|
||||
nlp.make_colon_as_title(typed_sections)
|
||||
|
||||
bull = nlp.bullets_category([s for s, _ in typed_sections])
|
||||
merged = nlp.tree_merge(bull, typed_sections, depth=2)
|
||||
|
||||
if merged:
|
||||
return merged
|
||||
|
||||
return nlp.naive_merge(
|
||||
typed_sections,
|
||||
chunk_token_num=chunk_token_num,
|
||||
delimiter=delimiter,
|
||||
overlapped_percent=overlapped_percent,
|
||||
)
|
||||
261
src/knowledge/chunking/ragflow_like/parsers/qa.py
Normal file
261
src/knowledge/chunking/ragflow_like/parsers/qa.py
Normal file
@ -0,0 +1,261 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import re
|
||||
from io import StringIO
|
||||
from typing import Any
|
||||
|
||||
|
||||
def _rm_prefix(text: str) -> str:
|
||||
return re.sub(
|
||||
r"^(问题|答案|回答|user|assistant|Q|A|Question|Answer|问|答)[\t:: ]+",
|
||||
"",
|
||||
(text or "").strip(),
|
||||
flags=re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _to_qa_chunk(question: str, answer: str, eng: bool = False) -> str:
|
||||
qprefix = "Question: " if eng else "问题:"
|
||||
aprefix = "Answer: " if eng else "回答:"
|
||||
return "\t".join([qprefix + _rm_prefix(question), aprefix + _rm_prefix(answer)])
|
||||
|
||||
|
||||
def _guess_delimiter(lines: list[str]) -> str:
|
||||
comma = 0
|
||||
tab = 0
|
||||
for line in lines:
|
||||
if len(line.split(",")) == 2:
|
||||
comma += 1
|
||||
if len(line.split("\t")) == 2:
|
||||
tab += 1
|
||||
return "\t" if tab >= comma else ","
|
||||
|
||||
|
||||
def _extract_pairs_with_delimiter(lines: list[str], delimiter: str) -> list[tuple[str, str]]:
|
||||
pairs: list[tuple[str, str]] = []
|
||||
question = ""
|
||||
answer = ""
|
||||
|
||||
for line in lines:
|
||||
arr = line.split(delimiter)
|
||||
if len(arr) != 2:
|
||||
if question:
|
||||
answer += "\n" + line
|
||||
continue
|
||||
|
||||
if question and answer:
|
||||
pairs.append((question, answer))
|
||||
question, answer = arr
|
||||
|
||||
if question:
|
||||
pairs.append((question, answer))
|
||||
|
||||
return [(q.strip(), a.strip()) for q, a in pairs if q.strip()]
|
||||
|
||||
|
||||
def _extract_pairs_from_csv(lines: list[str], delimiter: str) -> list[tuple[str, str]]:
|
||||
pairs: list[tuple[str, str]] = []
|
||||
question = ""
|
||||
answer = ""
|
||||
|
||||
reader = csv.reader(lines, delimiter=delimiter)
|
||||
for row, raw_line in zip(reader, lines, strict=False):
|
||||
if len(row) != 2:
|
||||
if question:
|
||||
answer += "\n" + raw_line
|
||||
continue
|
||||
|
||||
if question and answer:
|
||||
pairs.append((question, answer))
|
||||
question, answer = row
|
||||
|
||||
if question:
|
||||
pairs.append((question, answer))
|
||||
|
||||
return [(q.strip(), a.strip()) for q, a in pairs if q.strip()]
|
||||
|
||||
|
||||
def _parse_markdown_table_row(line: str) -> list[str] | None:
|
||||
if "|" not in line:
|
||||
return None
|
||||
|
||||
text = line.strip()
|
||||
if not text:
|
||||
return None
|
||||
|
||||
if text.startswith("|"):
|
||||
text = text[1:]
|
||||
if text.endswith("|"):
|
||||
text = text[:-1]
|
||||
|
||||
cells = [cell.strip() for cell in text.split("|")]
|
||||
if not cells:
|
||||
return None
|
||||
|
||||
if all(re.fullmatch(r":?-{3,}:?", c.replace(" ", "")) for c in cells if c):
|
||||
return None
|
||||
|
||||
return cells
|
||||
|
||||
|
||||
def _extract_pairs_from_markdown_tables(markdown_content: str) -> list[tuple[str, str]]:
|
||||
pairs: list[tuple[str, str]] = []
|
||||
|
||||
for line in (markdown_content or "").splitlines():
|
||||
cells = _parse_markdown_table_row(line)
|
||||
if not cells or len(cells) < 2:
|
||||
continue
|
||||
|
||||
question = cells[0]
|
||||
answer = cells[1]
|
||||
if question and answer:
|
||||
pairs.append((question, answer))
|
||||
|
||||
return pairs
|
||||
|
||||
|
||||
def _md_question_level(line: str) -> tuple[int, str]:
|
||||
match = re.match(r"^#*", line)
|
||||
if not match:
|
||||
return 0, line
|
||||
return len(match.group(0)), line.lstrip("#").lstrip()
|
||||
|
||||
|
||||
def _extract_pairs_from_markdown_headings(markdown_content: str) -> list[tuple[str, str]]:
|
||||
lines = (markdown_content or "").splitlines()
|
||||
if not lines:
|
||||
return []
|
||||
|
||||
pairs: list[tuple[str, str]] = []
|
||||
last_answer = ""
|
||||
question_stack: list[str] = []
|
||||
level_stack: list[int] = []
|
||||
code_block = False
|
||||
|
||||
for line in lines:
|
||||
if line.strip().startswith("```"):
|
||||
code_block = not code_block
|
||||
|
||||
question_level = 0
|
||||
question = ""
|
||||
if not code_block:
|
||||
question_level, question = _md_question_level(line)
|
||||
|
||||
if not question_level or question_level > 6:
|
||||
last_answer = f"{last_answer}\n{line}"
|
||||
continue
|
||||
|
||||
if last_answer.strip():
|
||||
sum_question = "\n".join(question_stack)
|
||||
if sum_question:
|
||||
pairs.append((sum_question, last_answer.strip()))
|
||||
last_answer = ""
|
||||
|
||||
while question_stack and question_level <= level_stack[-1]:
|
||||
question_stack.pop()
|
||||
level_stack.pop()
|
||||
|
||||
question_stack.append(question)
|
||||
level_stack.append(question_level)
|
||||
|
||||
if last_answer.strip():
|
||||
sum_question = "\n".join(question_stack)
|
||||
if sum_question:
|
||||
pairs.append((sum_question, last_answer.strip()))
|
||||
|
||||
return pairs
|
||||
|
||||
|
||||
def _extract_pairs_by_prefix(lines: list[str]) -> list[tuple[str, str]]:
|
||||
pairs: list[tuple[str, str]] = []
|
||||
question = ""
|
||||
answer_lines: list[str] = []
|
||||
|
||||
for line in lines:
|
||||
if re.match(r"^(Q|Question|问|问题)\s*[::]", line, flags=re.IGNORECASE):
|
||||
if question:
|
||||
pairs.append((question, "\n".join(answer_lines)))
|
||||
question = re.sub(r"^(Q|Question|问|问题)\s*[::]", "", line, flags=re.IGNORECASE).strip()
|
||||
answer_lines = []
|
||||
continue
|
||||
|
||||
if re.match(r"^(A|Answer|答|回答)\s*[::]", line, flags=re.IGNORECASE):
|
||||
answer_lines.append(re.sub(r"^(A|Answer|答|回答)\s*[::]", "", line, flags=re.IGNORECASE).strip())
|
||||
continue
|
||||
|
||||
if question:
|
||||
answer_lines.append(line)
|
||||
|
||||
if question:
|
||||
pairs.append((question, "\n".join(answer_lines)))
|
||||
|
||||
return [(q.strip(), a.strip()) for q, a in pairs if q.strip() and a.strip()]
|
||||
|
||||
|
||||
def _dedupe_pairs(pairs: list[tuple[str, str]]) -> list[tuple[str, str]]:
|
||||
res: list[tuple[str, str]] = []
|
||||
seen: set[tuple[str, str]] = set()
|
||||
|
||||
for question, answer in pairs:
|
||||
q = question.strip()
|
||||
a = answer.strip()
|
||||
if not q or not a:
|
||||
continue
|
||||
key = (q, a)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
res.append((q, a))
|
||||
|
||||
return res
|
||||
|
||||
|
||||
def chunk_markdown(filename: str, markdown_content: str, parser_config: dict[str, Any] | None = None) -> list[str]:
|
||||
parser_config = parser_config or {}
|
||||
eng = str(parser_config.get("language", "Chinese")).lower() == "english"
|
||||
|
||||
suffix = ""
|
||||
if filename and "." in filename:
|
||||
suffix = "." + filename.lower().split(".")[-1]
|
||||
|
||||
lines = [line for line in (markdown_content or "").splitlines() if line.strip()]
|
||||
pairs: list[tuple[str, str]] = []
|
||||
|
||||
if suffix in {".xlsx", ".xls"}:
|
||||
pairs.extend(_extract_pairs_from_markdown_tables(markdown_content))
|
||||
if not pairs:
|
||||
delimiter = _guess_delimiter(lines)
|
||||
pairs.extend(_extract_pairs_with_delimiter(lines, delimiter))
|
||||
elif suffix == ".csv":
|
||||
pairs.extend(_extract_pairs_from_markdown_tables(markdown_content))
|
||||
delimiter = "\t" if any("\t" in line for line in lines) else ","
|
||||
pairs.extend(_extract_pairs_from_csv(lines, delimiter))
|
||||
elif suffix == ".txt":
|
||||
delimiter = _guess_delimiter(lines)
|
||||
pairs.extend(_extract_pairs_with_delimiter(lines, delimiter))
|
||||
elif suffix in {".md", ".markdown", ".mdx"}:
|
||||
pairs.extend(_extract_pairs_from_markdown_headings(markdown_content))
|
||||
pairs.extend(_extract_pairs_from_markdown_tables(markdown_content))
|
||||
elif suffix == ".docx":
|
||||
pairs.extend(_extract_pairs_from_markdown_headings(markdown_content))
|
||||
pairs.extend(_extract_pairs_from_markdown_tables(markdown_content))
|
||||
else:
|
||||
pairs.extend(_extract_pairs_from_markdown_headings(markdown_content))
|
||||
pairs.extend(_extract_pairs_from_markdown_tables(markdown_content))
|
||||
pairs.extend(_extract_pairs_by_prefix(lines))
|
||||
if not pairs:
|
||||
delimiter = _guess_delimiter(lines)
|
||||
pairs.extend(_extract_pairs_with_delimiter(lines, delimiter))
|
||||
|
||||
pairs = _dedupe_pairs(pairs)
|
||||
|
||||
if not pairs and lines:
|
||||
# 最后兜底:把内容按 2 行一组构成问答
|
||||
for i in range(0, len(lines), 2):
|
||||
q = lines[i]
|
||||
a = lines[i + 1] if i + 1 < len(lines) else ""
|
||||
if q.strip() and a.strip():
|
||||
pairs.append((q, a))
|
||||
|
||||
return [_to_qa_chunk(q, a, eng=eng) for q, a in pairs]
|
||||
235
src/knowledge/chunking/ragflow_like/presets.py
Normal file
235
src/knowledge/chunking/ragflow_like/presets.py
Normal file
@ -0,0 +1,235 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from copy import deepcopy
|
||||
from typing import Any
|
||||
|
||||
from src.utils import logger
|
||||
|
||||
CHUNK_PRESET_GENERAL = "general"
|
||||
CHUNK_PRESET_QA = "qa"
|
||||
CHUNK_PRESET_BOOK = "book"
|
||||
CHUNK_PRESET_LAWS = "laws"
|
||||
|
||||
CHUNK_PRESET_IDS = {
|
||||
CHUNK_PRESET_GENERAL,
|
||||
CHUNK_PRESET_QA,
|
||||
CHUNK_PRESET_BOOK,
|
||||
CHUNK_PRESET_LAWS,
|
||||
}
|
||||
|
||||
CHUNK_PRESET_DESCRIPTIONS: dict[str, str] = {
|
||||
CHUNK_PRESET_GENERAL: "通用分块:按分隔符和长度切分,适合大多数普通文档。",
|
||||
CHUNK_PRESET_QA: "问答分块:优先抽取问题-回答结构,适合 FAQ、题库、问答手册。",
|
||||
CHUNK_PRESET_BOOK: "书籍分块:强化章节标题识别并做层级合并,适合教材、手册、长章节文档。",
|
||||
CHUNK_PRESET_LAWS: "法规分块:按法条层级组织与合并,适合法律法规、制度规范类文本。",
|
||||
}
|
||||
|
||||
CHUNK_ENGINE_VERSION = "ragflow_like_v1"
|
||||
GENERAL_INTERNAL_PARSER_ID = "naive"
|
||||
|
||||
_BASE_DEFAULTS: dict[str, Any] = {
|
||||
"table_context_size": 0,
|
||||
"image_context_size": 0,
|
||||
}
|
||||
|
||||
_PRESET_DEFAULTS: dict[str, dict[str, Any] | None] = {
|
||||
CHUNK_PRESET_GENERAL: {
|
||||
"layout_recognize": "DeepDOC",
|
||||
"chunk_token_num": 512,
|
||||
"delimiter": "\n",
|
||||
"auto_keywords": 0,
|
||||
"auto_questions": 0,
|
||||
"html4excel": False,
|
||||
"topn_tags": 3,
|
||||
"raptor": {
|
||||
"use_raptor": True,
|
||||
"prompt": "Please summarize the following paragraphs. Be careful with the numbers, do not make things up. Paragraphs as following:\n {cluster_content}\nThe above is the content you need to summarize.",
|
||||
"max_token": 256,
|
||||
"threshold": 0.1,
|
||||
"max_cluster": 64,
|
||||
"random_seed": 0,
|
||||
},
|
||||
"graphrag": {
|
||||
"use_graphrag": True,
|
||||
"entity_types": ["organization", "person", "geo", "event", "category"],
|
||||
"method": "light",
|
||||
},
|
||||
},
|
||||
CHUNK_PRESET_QA: {"raptor": {"use_raptor": False}, "graphrag": {"use_graphrag": False}},
|
||||
CHUNK_PRESET_BOOK: {"raptor": {"use_raptor": False}, "graphrag": {"use_graphrag": False}},
|
||||
CHUNK_PRESET_LAWS: {"raptor": {"use_raptor": False}, "graphrag": {"use_graphrag": False}},
|
||||
}
|
||||
|
||||
|
||||
def deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]:
|
||||
result = deepcopy(base)
|
||||
for key, value in (override or {}).items():
|
||||
if isinstance(value, dict) and isinstance(result.get(key), dict):
|
||||
result[key] = deep_merge(result[key], value)
|
||||
else:
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
|
||||
def normalize_chunk_preset_id(value: str | None) -> str:
|
||||
if not value:
|
||||
return CHUNK_PRESET_GENERAL
|
||||
|
||||
normalized = str(value).strip().lower()
|
||||
if normalized == GENERAL_INTERNAL_PARSER_ID:
|
||||
return CHUNK_PRESET_GENERAL
|
||||
|
||||
if normalized in CHUNK_PRESET_IDS:
|
||||
return normalized
|
||||
|
||||
logger.warning(f"Unknown chunk preset id '{value}', fallback to general")
|
||||
return CHUNK_PRESET_GENERAL
|
||||
|
||||
|
||||
def map_to_internal_parser_id(preset_id: str) -> str:
|
||||
normalized = normalize_chunk_preset_id(preset_id)
|
||||
if normalized == CHUNK_PRESET_GENERAL:
|
||||
return GENERAL_INTERNAL_PARSER_ID
|
||||
return normalized
|
||||
|
||||
|
||||
def get_default_chunk_parser_config(preset_id: str) -> dict[str, Any]:
|
||||
normalized = normalize_chunk_preset_id(preset_id)
|
||||
default_config = deepcopy(_PRESET_DEFAULTS.get(normalized) or {})
|
||||
return deep_merge(_BASE_DEFAULTS, default_config)
|
||||
|
||||
|
||||
def _safe_int(value: Any) -> int | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _legacy_params_to_parser_config(params: dict[str, Any] | None) -> dict[str, Any]:
|
||||
if not isinstance(params, dict):
|
||||
return {}
|
||||
|
||||
parser_config: dict[str, Any] = {}
|
||||
|
||||
chunk_size = _safe_int(params.get("chunk_size"))
|
||||
chunk_overlap = _safe_int(params.get("chunk_overlap"))
|
||||
|
||||
if chunk_size and chunk_size > 0:
|
||||
parser_config["chunk_token_num"] = chunk_size
|
||||
if chunk_size and chunk_size > 0 and chunk_overlap is not None:
|
||||
overlap_percent = round(max(0, min(chunk_overlap, chunk_size - 1)) * 100 / chunk_size)
|
||||
parser_config["overlapped_percent"] = max(0, min(overlap_percent, 99))
|
||||
|
||||
if isinstance(params.get("qa_separator"), str) and params.get("qa_separator"):
|
||||
parser_config["delimiter"] = params["qa_separator"]
|
||||
|
||||
if isinstance(params.get("delimiter"), str) and params.get("delimiter"):
|
||||
parser_config["delimiter"] = params["delimiter"]
|
||||
|
||||
if "chunk_token_num" in params:
|
||||
normalized_chunk_token_num = _safe_int(params.get("chunk_token_num"))
|
||||
if normalized_chunk_token_num is not None:
|
||||
parser_config["chunk_token_num"] = normalized_chunk_token_num
|
||||
|
||||
if "overlapped_percent" in params:
|
||||
normalized_overlapped_percent = _safe_int(params.get("overlapped_percent"))
|
||||
if normalized_overlapped_percent is not None:
|
||||
parser_config["overlapped_percent"] = max(0, min(normalized_overlapped_percent, 99))
|
||||
|
||||
return parser_config
|
||||
|
||||
|
||||
def ensure_chunk_defaults_in_additional_params(additional_params: dict[str, Any] | None) -> dict[str, Any]:
|
||||
params = dict(additional_params or {})
|
||||
params["chunk_preset_id"] = normalize_chunk_preset_id(params.get("chunk_preset_id"))
|
||||
|
||||
if "chunk_parser_config" in params and not isinstance(params.get("chunk_parser_config"), dict):
|
||||
logger.warning("Invalid chunk_parser_config in additional_params, fallback to empty dict")
|
||||
params["chunk_parser_config"] = {}
|
||||
|
||||
return params
|
||||
|
||||
|
||||
def resolve_chunk_processing_params(
|
||||
kb_additional_params: dict[str, Any] | None,
|
||||
file_processing_params: dict[str, Any] | None,
|
||||
request_params: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
kb_additional = ensure_chunk_defaults_in_additional_params(kb_additional_params)
|
||||
file_params = dict(file_processing_params or {})
|
||||
request = dict(request_params or {})
|
||||
|
||||
preset_id = normalize_chunk_preset_id(
|
||||
request.get("chunk_preset_id")
|
||||
or file_params.get("chunk_preset_id")
|
||||
or kb_additional.get("chunk_preset_id")
|
||||
)
|
||||
|
||||
parser_config = get_default_chunk_parser_config(preset_id)
|
||||
|
||||
kb_parser_config = kb_additional.get("chunk_parser_config")
|
||||
if isinstance(kb_parser_config, dict):
|
||||
parser_config = deep_merge(parser_config, kb_parser_config)
|
||||
|
||||
file_parser_config = file_params.get("chunk_parser_config")
|
||||
if isinstance(file_parser_config, dict):
|
||||
parser_config = deep_merge(parser_config, file_parser_config)
|
||||
|
||||
req_parser_config = request.get("chunk_parser_config")
|
||||
if isinstance(req_parser_config, dict):
|
||||
parser_config = deep_merge(parser_config, req_parser_config)
|
||||
|
||||
merged_legacy = {}
|
||||
merged_legacy.update(file_params)
|
||||
merged_legacy.update(request)
|
||||
parser_config = deep_merge(parser_config, _legacy_params_to_parser_config(merged_legacy))
|
||||
|
||||
# Build processing params snapshot (keep existing + request overrides for non-chunk fields)
|
||||
snapshot: dict[str, Any] = {}
|
||||
snapshot.update(file_params)
|
||||
snapshot.update(request)
|
||||
snapshot["chunk_preset_id"] = preset_id
|
||||
snapshot["chunk_parser_config"] = parser_config
|
||||
snapshot["chunk_engine_version"] = CHUNK_ENGINE_VERSION
|
||||
|
||||
# Keep backward-compatible fields for current UI
|
||||
if "chunk_size" not in snapshot and isinstance(parser_config.get("chunk_token_num"), int):
|
||||
snapshot["chunk_size"] = parser_config["chunk_token_num"]
|
||||
|
||||
if "chunk_overlap" not in snapshot and isinstance(parser_config.get("overlapped_percent"), int):
|
||||
token_num = parser_config.get("chunk_token_num")
|
||||
if isinstance(token_num, int) and token_num > 0:
|
||||
snapshot["chunk_overlap"] = int(token_num * parser_config["overlapped_percent"] / 100)
|
||||
|
||||
if "qa_separator" not in snapshot and isinstance(parser_config.get("delimiter"), str):
|
||||
snapshot["qa_separator"] = parser_config["delimiter"]
|
||||
|
||||
return snapshot
|
||||
|
||||
|
||||
def get_chunk_preset_options() -> list[dict[str, str]]:
|
||||
return [
|
||||
{
|
||||
"value": CHUNK_PRESET_GENERAL,
|
||||
"label": "General",
|
||||
"description": CHUNK_PRESET_DESCRIPTIONS[CHUNK_PRESET_GENERAL],
|
||||
},
|
||||
{
|
||||
"value": CHUNK_PRESET_QA,
|
||||
"label": "QA",
|
||||
"description": CHUNK_PRESET_DESCRIPTIONS[CHUNK_PRESET_QA],
|
||||
},
|
||||
{
|
||||
"value": CHUNK_PRESET_BOOK,
|
||||
"label": "Book",
|
||||
"description": CHUNK_PRESET_DESCRIPTIONS[CHUNK_PRESET_BOOK],
|
||||
},
|
||||
{
|
||||
"value": CHUNK_PRESET_LAWS,
|
||||
"label": "Laws",
|
||||
"description": CHUNK_PRESET_DESCRIPTIONS[CHUNK_PRESET_LAWS],
|
||||
},
|
||||
]
|
||||
@ -11,6 +11,8 @@ from pymilvus import connections, utility
|
||||
|
||||
from src import config
|
||||
from src.knowledge.base import FileStatus, KnowledgeBase
|
||||
from src.knowledge.chunking.ragflow_like.dispatcher import chunk_markdown
|
||||
from src.knowledge.chunking.ragflow_like.presets import resolve_chunk_processing_params
|
||||
from src.knowledge.indexing import process_file_to_markdown
|
||||
from src.knowledge.utils.kb_utils import get_embedding_config
|
||||
from src.utils import hashstr, logger
|
||||
@ -40,6 +42,18 @@ class LightRagKB(KnowledgeBase):
|
||||
"""知识库类型标识"""
|
||||
return "lightrag"
|
||||
|
||||
@staticmethod
|
||||
def _prepare_lightrag_insert_payload(chunks: list[dict]) -> tuple[str, str | None, bool]:
|
||||
if not chunks:
|
||||
return "", None, False
|
||||
|
||||
if len(chunks) == 1:
|
||||
return chunks[0]["content"], None, False
|
||||
|
||||
delimiter = "\n<|YUXI_CHUNK_DELIM|>\n"
|
||||
payload = delimiter.join(chunk["content"] for chunk in chunks if chunk.get("content"))
|
||||
return payload, delimiter, True
|
||||
|
||||
def delete_database(self, db_id: str) -> dict:
|
||||
"""删除数据库,同时清除Milvus和Neo4j中的数据"""
|
||||
# Drop Milvus collection
|
||||
@ -137,6 +151,21 @@ class LightRagKB(KnowledgeBase):
|
||||
await instance.initialize_storages()
|
||||
await initialize_pipeline_status()
|
||||
|
||||
@staticmethod
|
||||
async def _ensure_doc_processed(rag: LightRAG, file_id: str) -> None:
|
||||
"""确保 LightRAG 文档处理成功,否则抛出异常。"""
|
||||
status_doc = await rag.doc_status.get_by_id(file_id)
|
||||
if not status_doc:
|
||||
raise ValueError(f"LightRAG 文档状态缺失: {file_id}")
|
||||
|
||||
status = status_doc.get("status")
|
||||
status_value = status.value if hasattr(status, "value") else status
|
||||
if status_value not in {"processed", "preprocessed"}:
|
||||
error_msg = status_doc.get("error_msg") or "unknown error"
|
||||
raise ValueError(
|
||||
f"LightRAG 实体关系抽取失败: file_id={file_id}, status={status_value}, error={error_msg}"
|
||||
)
|
||||
|
||||
async def _get_lightrag_instance(self, db_id: str) -> LightRAG | None:
|
||||
"""获取或创建 LightRAG 实例"""
|
||||
if db_id in self.instances:
|
||||
@ -293,14 +322,36 @@ class LightRagKB(KnowledgeBase):
|
||||
# Read markdown
|
||||
markdown_content = await self._read_markdown_from_minio(file_meta["markdown_file"])
|
||||
file_path = file_meta.get("path")
|
||||
filename = file_meta.get("filename") or file_id
|
||||
processing_params = resolve_chunk_processing_params(
|
||||
kb_additional_params=self.databases_meta.get(db_id, {}).get("metadata"),
|
||||
file_processing_params=file_meta.get("processing_params"),
|
||||
)
|
||||
self.files_meta[file_id]["processing_params"] = processing_params
|
||||
await self._save_metadata()
|
||||
|
||||
chunks = chunk_markdown(markdown_content, file_id, filename, processing_params)
|
||||
chunk_input, split_by_character, split_by_character_only = self._prepare_lightrag_insert_payload(chunks)
|
||||
if not chunk_input:
|
||||
chunk_input = markdown_content
|
||||
|
||||
# Clean up existing chunks if any (for re-indexing)
|
||||
await self.delete_file_chunks_only(db_id, file_id)
|
||||
|
||||
# Insert
|
||||
await rag.ainsert(input=markdown_content, ids=file_id, file_paths=file_path)
|
||||
await rag.ainsert(
|
||||
input=chunk_input,
|
||||
ids=file_id,
|
||||
file_paths=file_path,
|
||||
split_by_character=split_by_character,
|
||||
split_by_character_only=split_by_character_only,
|
||||
)
|
||||
await self._ensure_doc_processed(rag, file_id)
|
||||
|
||||
logger.info(f"Indexed file {file_id} into LightRAG")
|
||||
logger.info(
|
||||
f"Indexed file {file_id} into LightRAG with {len(chunks)} chunks, "
|
||||
f"chunk_preset_id={processing_params.get('chunk_preset_id')}"
|
||||
)
|
||||
|
||||
# Update status
|
||||
self.files_meta[file_id]["status"] = FileStatus.INDEXED
|
||||
@ -358,7 +409,12 @@ class LightRagKB(KnowledgeBase):
|
||||
|
||||
try:
|
||||
# 更新状态为处理中
|
||||
self.files_meta[file_id]["processing_params"] = params.copy()
|
||||
resolved_params = resolve_chunk_processing_params(
|
||||
kb_additional_params=self.databases_meta.get(db_id, {}).get("metadata"),
|
||||
file_processing_params=self.files_meta[file_id].get("processing_params"),
|
||||
request_params=params,
|
||||
)
|
||||
self.files_meta[file_id]["processing_params"] = resolved_params
|
||||
self.files_meta[file_id]["status"] = "processing"
|
||||
await self._save_metadata()
|
||||
|
||||
@ -368,12 +424,24 @@ class LightRagKB(KnowledgeBase):
|
||||
markdown_content = await process_file_to_markdown(file_path, params=params)
|
||||
markdown_content_lines = markdown_content[:100].replace("\n", " ")
|
||||
logger.info(f"Markdown content: {markdown_content_lines}...")
|
||||
filename = file_meta.get("filename") or file_id
|
||||
chunks = chunk_markdown(markdown_content, file_id, filename, resolved_params)
|
||||
chunk_input, split_by_character, split_by_character_only = self._prepare_lightrag_insert_payload(chunks)
|
||||
if not chunk_input:
|
||||
chunk_input = markdown_content
|
||||
|
||||
# 先删除现有的 LightRAG 数据(仅删除chunks,保留元数据)
|
||||
await self.delete_file_chunks_only(db_id, file_id)
|
||||
|
||||
# 使用 LightRAG 重新插入内容
|
||||
await rag.ainsert(input=markdown_content, ids=file_id, file_paths=file_path)
|
||||
await rag.ainsert(
|
||||
input=chunk_input,
|
||||
ids=file_id,
|
||||
file_paths=file_path,
|
||||
split_by_character=split_by_character,
|
||||
split_by_character_only=split_by_character_only,
|
||||
)
|
||||
await self._ensure_doc_processed(rag, file_id)
|
||||
|
||||
logger.info(f"Updated {content_type} {file_path} in LightRAG. Done.")
|
||||
|
||||
|
||||
@ -10,11 +10,10 @@ from pymilvus import Collection, CollectionSchema, DataType, FieldSchema, connec
|
||||
|
||||
from src import config
|
||||
from src.knowledge.base import FileStatus, KnowledgeBase
|
||||
from src.knowledge.chunking.ragflow_like.dispatcher import chunk_markdown
|
||||
from src.knowledge.chunking.ragflow_like.presets import resolve_chunk_processing_params
|
||||
from src.knowledge.indexing import process_file_to_markdown
|
||||
from src.knowledge.utils.kb_utils import (
|
||||
get_embedding_config,
|
||||
split_text_into_chunks,
|
||||
)
|
||||
from src.knowledge.utils.kb_utils import get_embedding_config
|
||||
from src.models.embed import OtherEmbedding
|
||||
from src.utils import hashstr, logger
|
||||
from src.utils.datetime_utils import utc_isoformat
|
||||
@ -222,7 +221,7 @@ class MilvusKB(KnowledgeBase):
|
||||
|
||||
def _split_text_into_chunks(self, text: str, file_id: str, filename: str, params: dict) -> list[dict]:
|
||||
"""将文本分割成块"""
|
||||
return split_text_into_chunks(text, file_id, filename, params)
|
||||
return chunk_markdown(text, file_id, filename, params)
|
||||
|
||||
async def index_file(self, db_id: str, file_id: str, operator_id: str | None = None) -> dict:
|
||||
"""
|
||||
@ -281,10 +280,14 @@ class MilvusKB(KnowledgeBase):
|
||||
self.files_meta[file_id]["updated_at"] = utc_isoformat()
|
||||
if operator_id:
|
||||
self.files_meta[file_id]["updated_by"] = operator_id
|
||||
await self._save_metadata()
|
||||
|
||||
# Read processing params inside lock to ensure we get the latest values
|
||||
params = file_meta.get("processing_params", {}) or {}
|
||||
params = resolve_chunk_processing_params(
|
||||
kb_additional_params=self.databases_meta.get(db_id, {}).get("metadata"),
|
||||
file_processing_params=file_meta.get("processing_params"),
|
||||
)
|
||||
self.files_meta[file_id]["processing_params"] = params
|
||||
await self._save_metadata()
|
||||
logger.debug(f"[index_file] file_id={file_id}, processing_params={params}")
|
||||
|
||||
# Add to processing queue
|
||||
@ -299,6 +302,7 @@ class MilvusKB(KnowledgeBase):
|
||||
chunks = self._split_text_into_chunks(markdown_content, file_id, filename, params)
|
||||
logger.info(
|
||||
f"Split {filename} into {len(chunks)} chunks with params: "
|
||||
f"chunk_preset_id={params.get('chunk_preset_id')}, "
|
||||
f"chunk_size={params.get('chunk_size')}, "
|
||||
f"chunk_overlap={params.get('chunk_overlap')}, "
|
||||
f"qa_separator={params.get('qa_separator')}"
|
||||
@ -391,7 +395,12 @@ class MilvusKB(KnowledgeBase):
|
||||
try:
|
||||
# 更新状态为处理中
|
||||
async with self._metadata_lock:
|
||||
self.files_meta[file_id]["processing_params"] = params.copy()
|
||||
resolved_params = resolve_chunk_processing_params(
|
||||
kb_additional_params=self.databases_meta.get(db_id, {}).get("metadata"),
|
||||
file_processing_params=self.files_meta[file_id].get("processing_params"),
|
||||
request_params=params,
|
||||
)
|
||||
self.files_meta[file_id]["processing_params"] = resolved_params
|
||||
self.files_meta[file_id]["status"] = "processing"
|
||||
await self._save_metadata()
|
||||
|
||||
@ -404,7 +413,7 @@ class MilvusKB(KnowledgeBase):
|
||||
await self.delete_file_chunks_only(db_id, file_id)
|
||||
|
||||
# 重新生成 chunks
|
||||
chunks = self._split_text_into_chunks(markdown_content, file_id, filename, params)
|
||||
chunks = self._split_text_into_chunks(markdown_content, file_id, filename, resolved_params)
|
||||
logger.info(f"Split {filename} into {len(chunks)} chunks")
|
||||
|
||||
if chunks:
|
||||
|
||||
@ -2,6 +2,10 @@ import asyncio
|
||||
import os
|
||||
|
||||
from src.knowledge.base import KBNotFoundError, KnowledgeBase
|
||||
from src.knowledge.chunking.ragflow_like.presets import (
|
||||
deep_merge,
|
||||
ensure_chunk_defaults_in_additional_params,
|
||||
)
|
||||
from src.knowledge.factory import KnowledgeBaseFactory
|
||||
from src.utils import logger
|
||||
from src.utils.datetime_utils import utc_isoformat
|
||||
@ -179,7 +183,7 @@ class KnowledgeBaseManager:
|
||||
if db_info:
|
||||
# 补充 share_config 和 additional_params
|
||||
db_info["share_config"] = row.share_config or {"is_shared": True, "accessible_departments": []}
|
||||
db_info["additional_params"] = row.additional_params or {}
|
||||
db_info["additional_params"] = ensure_chunk_defaults_in_additional_params(row.additional_params)
|
||||
all_databases.append(db_info)
|
||||
return {"databases": all_databases}
|
||||
|
||||
@ -310,6 +314,8 @@ class KnowledgeBaseManager:
|
||||
if share_config is None:
|
||||
share_config = {"is_shared": True, "accessible_departments": []}
|
||||
|
||||
kwargs = ensure_chunk_defaults_in_additional_params(kwargs)
|
||||
|
||||
kb_instance = self._get_or_create_kb_instance(kb_type)
|
||||
db_info = await kb_instance.create_database(database_name, description, embed_info, **kwargs)
|
||||
db_id = db_info["db_id"]
|
||||
@ -414,7 +420,7 @@ class KnowledgeBaseManager:
|
||||
}
|
||||
|
||||
# 添加数据库中的附加字段
|
||||
db_info["additional_params"] = kb.additional_params or {}
|
||||
db_info["additional_params"] = ensure_chunk_defaults_in_additional_params(kb.additional_params)
|
||||
db_info["share_config"] = kb.share_config or {"is_shared": True, "accessible_departments": []}
|
||||
db_info["mindmap"] = kb.mindmap
|
||||
db_info["sample_questions"] = kb.sample_questions or []
|
||||
@ -566,6 +572,11 @@ class KnowledgeBaseManager:
|
||||
"""更新数据库"""
|
||||
from src.repositories.knowledge_base_repository import KnowledgeBaseRepository
|
||||
|
||||
kb_repo = KnowledgeBaseRepository()
|
||||
kb = await kb_repo.get_by_id(db_id)
|
||||
if kb is None:
|
||||
raise ValueError(f"数据库 {db_id} 不存在")
|
||||
|
||||
kb_instance = await self._get_kb_for_database(db_id)
|
||||
kb_instance.update_database(db_id, name, description, llm_info)
|
||||
|
||||
@ -576,13 +587,19 @@ class KnowledgeBaseManager:
|
||||
}
|
||||
if llm_info is not None:
|
||||
update_data["llm_info"] = llm_info
|
||||
|
||||
if additional_params is not None:
|
||||
update_data["additional_params"] = additional_params
|
||||
merged_additional_params = ensure_chunk_defaults_in_additional_params(
|
||||
deep_merge(kb.additional_params or {}, additional_params)
|
||||
)
|
||||
update_data["additional_params"] = merged_additional_params
|
||||
if db_id in kb_instance.databases_meta:
|
||||
kb_instance.databases_meta[db_id]["metadata"] = merged_additional_params
|
||||
|
||||
if share_config is not None:
|
||||
update_data["share_config"] = share_config
|
||||
|
||||
# 保存到数据库
|
||||
kb_repo = KnowledgeBaseRepository()
|
||||
await kb_repo.update(db_id, update_data)
|
||||
|
||||
return await self.get_database_info(db_id)
|
||||
|
||||
@ -40,6 +40,59 @@ async def test_admin_can_manage_knowledge_databases(test_client, admin_headers,
|
||||
assert update_response.json()["database"]["description"] == "Updated by pytest"
|
||||
|
||||
|
||||
async def test_create_database_with_chunk_preset(test_client, admin_headers):
|
||||
db_name = f"pytest_chunk_preset_{uuid.uuid4().hex[:6]}"
|
||||
payload = {
|
||||
"database_name": db_name,
|
||||
"description": "Chunk preset create test",
|
||||
"embed_model_name": "siliconflow/BAAI/bge-m3",
|
||||
"kb_type": "milvus",
|
||||
"additional_params": {"chunk_preset_id": "book"},
|
||||
}
|
||||
|
||||
create_response = await test_client.post("/api/knowledge/databases", json=payload, headers=admin_headers)
|
||||
assert create_response.status_code == 200, create_response.text
|
||||
db_id = create_response.json()["db_id"]
|
||||
|
||||
info_response = await test_client.get(f"/api/knowledge/databases/{db_id}", headers=admin_headers)
|
||||
assert info_response.status_code == 200, info_response.text
|
||||
assert info_response.json()["additional_params"]["chunk_preset_id"] == "book"
|
||||
|
||||
delete_response = await test_client.delete(f"/api/knowledge/databases/{db_id}", headers=admin_headers)
|
||||
assert delete_response.status_code == 200, delete_response.text
|
||||
|
||||
|
||||
async def test_update_database_additional_params_merge_keeps_chunk_preset(
|
||||
test_client, admin_headers, knowledge_database
|
||||
):
|
||||
db_id = knowledge_database["db_id"]
|
||||
|
||||
first_update = await test_client.put(
|
||||
f"/api/knowledge/databases/{db_id}",
|
||||
json={
|
||||
"name": knowledge_database["name"],
|
||||
"description": "update with chunk preset",
|
||||
"additional_params": {"chunk_preset_id": "qa"},
|
||||
},
|
||||
headers=admin_headers,
|
||||
)
|
||||
assert first_update.status_code == 200, first_update.text
|
||||
|
||||
second_update = await test_client.put(
|
||||
f"/api/knowledge/databases/{db_id}",
|
||||
json={
|
||||
"name": knowledge_database["name"],
|
||||
"description": "update without additional params",
|
||||
},
|
||||
headers=admin_headers,
|
||||
)
|
||||
assert second_update.status_code == 200, second_update.text
|
||||
|
||||
info_response = await test_client.get(f"/api/knowledge/databases/{db_id}", headers=admin_headers)
|
||||
assert info_response.status_code == 200, info_response.text
|
||||
assert info_response.json()["additional_params"]["chunk_preset_id"] == "qa"
|
||||
|
||||
|
||||
async def test_knowledge_routes_enforce_permissions(test_client, standard_user, knowledge_database):
|
||||
db_id = knowledge_database["db_id"]
|
||||
|
||||
|
||||
111
test/test_ragflow_like_chunking.py
Normal file
111
test/test_ragflow_like_chunking.py
Normal file
@ -0,0 +1,111 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.append(os.getcwd())
|
||||
|
||||
from src.knowledge.chunking.ragflow_like.dispatcher import chunk_markdown
|
||||
from src.knowledge.chunking.ragflow_like.nlp import bullets_category
|
||||
from src.knowledge.chunking.ragflow_like.presets import (
|
||||
CHUNK_ENGINE_VERSION,
|
||||
get_chunk_preset_options,
|
||||
map_to_internal_parser_id,
|
||||
resolve_chunk_processing_params,
|
||||
)
|
||||
|
||||
|
||||
def test_general_maps_to_naive() -> None:
|
||||
assert map_to_internal_parser_id("general") == "naive"
|
||||
|
||||
|
||||
def test_resolve_chunk_processing_params_priority() -> None:
|
||||
resolved = resolve_chunk_processing_params(
|
||||
kb_additional_params={
|
||||
"chunk_preset_id": "book",
|
||||
"chunk_parser_config": {"chunk_token_num": 300, "delimiter": "\\n"},
|
||||
},
|
||||
file_processing_params={
|
||||
"chunk_preset_id": "qa",
|
||||
"chunk_parser_config": {"delimiter": "###"},
|
||||
},
|
||||
request_params={
|
||||
"chunk_preset_id": "laws",
|
||||
"chunk_parser_config": {"chunk_token_num": 666},
|
||||
"chunk_size": 777,
|
||||
},
|
||||
)
|
||||
|
||||
assert resolved["chunk_preset_id"] == "laws"
|
||||
assert resolved["chunk_engine_version"] == CHUNK_ENGINE_VERSION
|
||||
# legacy chunk_size 在当前实现里会映射为 chunk_token_num
|
||||
assert resolved["chunk_parser_config"]["chunk_token_num"] == 777
|
||||
assert resolved["chunk_parser_config"]["delimiter"] == "###"
|
||||
|
||||
|
||||
def test_qa_chunking_from_markdown_headings() -> None:
|
||||
content = """
|
||||
# 问题一
|
||||
这是答案一。
|
||||
|
||||
## 子问题
|
||||
这是答案二。
|
||||
""".strip()
|
||||
|
||||
chunks = chunk_markdown(
|
||||
markdown_content=content,
|
||||
file_id="file_1",
|
||||
filename="faq.md",
|
||||
processing_params={"chunk_preset_id": "qa", "chunk_parser_config": {}},
|
||||
)
|
||||
|
||||
assert len(chunks) >= 1
|
||||
assert "问题:" in chunks[0]["content"]
|
||||
assert "回答:" in chunks[0]["content"]
|
||||
|
||||
|
||||
def test_book_chunking_hierarchical_merge() -> None:
|
||||
content = """
|
||||
第一章 总则
|
||||
第一节 适用范围
|
||||
本规范适用于测试场景。
|
||||
第二节 基本原则
|
||||
应当遵循最小改动原则。
|
||||
""".strip()
|
||||
|
||||
chunks = chunk_markdown(
|
||||
markdown_content=content,
|
||||
file_id="file_2",
|
||||
filename="book.txt",
|
||||
processing_params={"chunk_preset_id": "book", "chunk_parser_config": {"chunk_token_num": 256}},
|
||||
)
|
||||
|
||||
assert len(chunks) >= 1
|
||||
assert any("第一章" in ck["content"] for ck in chunks)
|
||||
|
||||
|
||||
def test_markdown_heading_has_higher_weight_in_bullet_category() -> None:
|
||||
sections = [
|
||||
"# 3.2 个人所得项目及计税、申报方式概括",
|
||||
"一、关于季节工、临时工等费用税前扣除问题,以下规定继续执行。",
|
||||
"二、根据现行规定,补贴收入应并入工资薪金所得。",
|
||||
"(一)从超出国家规定比例支付的补贴,不属于免税福利费。",
|
||||
]
|
||||
|
||||
# 命中 markdown 标题模式(BULLET_PATTERN 下标 4)时,应该优先选中该组。
|
||||
assert bullets_category(sections) == 4
|
||||
|
||||
|
||||
def test_mid_sentence_bullet_marker_should_not_be_treated_as_heading() -> None:
|
||||
sections = [
|
||||
"根据前述规则:一、这里是句中枚举,不是章节标题,不能被当成层级。",
|
||||
"延续上文:(二)这里同样是正文中的枚举表达,不是独立标题。",
|
||||
"## 3.4 交通补贴的个税处理",
|
||||
]
|
||||
assert bullets_category(sections) == 4
|
||||
|
||||
|
||||
def test_chunk_preset_options_include_description() -> None:
|
||||
options = get_chunk_preset_options()
|
||||
assert len(options) == 4
|
||||
assert all(isinstance(option.get("description"), str) and option["description"] for option in options)
|
||||
@ -4,7 +4,27 @@
|
||||
<p>调整分块参数可以控制文本的切分方式,影响检索质量和文档加载效率。</p>
|
||||
</div>
|
||||
<a-form :model="tempChunkParams" name="chunkConfig" autocomplete="off" layout="vertical">
|
||||
<div class="chunk-row">
|
||||
<a-form-item v-if="showPreset" name="chunk_preset_id">
|
||||
<template #label>
|
||||
<span class="chunk-preset-label">
|
||||
分块策略
|
||||
<a-tooltip :title="presetDescription">
|
||||
<QuestionCircleOutlined class="chunk-preset-help-icon" />
|
||||
</a-tooltip>
|
||||
</span>
|
||||
</template>
|
||||
<a-select
|
||||
v-model:value="tempChunkParams.chunk_preset_id"
|
||||
:options="presetOptions"
|
||||
style="width: 100%"
|
||||
/>
|
||||
<p class="param-description">
|
||||
预设策略对齐 RAGFlow:General、QA、Book、Laws。
|
||||
<span v-if="allowPresetFollowDefault">留空时沿用知识库默认策略。</span>
|
||||
</p>
|
||||
</a-form-item>
|
||||
|
||||
<div class="chunk-row" v-if="showChunkSizeOverlap">
|
||||
<a-form-item label="Chunk Size" name="chunk_size">
|
||||
<a-input-number
|
||||
v-model:value="tempChunkParams.chunk_size"
|
||||
@ -42,7 +62,15 @@
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
defineProps({
|
||||
import { computed } from 'vue'
|
||||
import { QuestionCircleOutlined } from '@ant-design/icons-vue'
|
||||
import {
|
||||
CHUNK_PRESET_OPTIONS,
|
||||
CHUNK_PRESET_LABEL_MAP,
|
||||
getChunkPresetDescription
|
||||
} from '@/utils/chunk_presets'
|
||||
|
||||
const props = defineProps({
|
||||
tempChunkParams: {
|
||||
type: Object,
|
||||
required: true
|
||||
@ -50,8 +78,45 @@ defineProps({
|
||||
showQaSplit: {
|
||||
type: Boolean,
|
||||
default: true
|
||||
},
|
||||
showChunkSizeOverlap: {
|
||||
type: Boolean,
|
||||
default: true
|
||||
},
|
||||
showPreset: {
|
||||
type: Boolean,
|
||||
default: true
|
||||
},
|
||||
allowPresetFollowDefault: {
|
||||
type: Boolean,
|
||||
default: false
|
||||
},
|
||||
databasePresetId: {
|
||||
type: String,
|
||||
default: 'general'
|
||||
}
|
||||
})
|
||||
|
||||
const presetOptions = computed(() => {
|
||||
const options = []
|
||||
const defaultPresetLabel = CHUNK_PRESET_LABEL_MAP[props.databasePresetId] || 'General'
|
||||
|
||||
if (props.allowPresetFollowDefault) {
|
||||
options.push({
|
||||
value: '',
|
||||
label: `沿用知识库默认(${defaultPresetLabel})`
|
||||
})
|
||||
}
|
||||
|
||||
options.push(...CHUNK_PRESET_OPTIONS.map(({ value, label }) => ({ value, label })))
|
||||
|
||||
return options
|
||||
})
|
||||
|
||||
const effectivePresetId = computed(
|
||||
() => props.tempChunkParams.chunk_preset_id || props.databasePresetId || 'general'
|
||||
)
|
||||
const presetDescription = computed(() => getChunkPresetDescription(effectivePresetId.value))
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
@ -92,4 +157,16 @@ defineProps({
|
||||
margin: 4px 0 0 0;
|
||||
line-height: 1.4;
|
||||
}
|
||||
|
||||
.chunk-preset-label {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.chunk-preset-help-icon {
|
||||
color: var(--gray-500);
|
||||
cursor: help;
|
||||
font-size: 14px;
|
||||
}
|
||||
</style>
|
||||
|
||||
@ -189,7 +189,14 @@
|
||||
<a-button key="submit" type="primary" @click="handleIndexConfigConfirm">确定</a-button>
|
||||
</template>
|
||||
<div class="index-params">
|
||||
<ChunkParamsConfig :temp-chunk-params="indexParams" :show-qa-split="true" />
|
||||
<ChunkParamsConfig
|
||||
:temp-chunk-params="indexParams"
|
||||
:show-qa-split="true"
|
||||
:show-chunk-size-overlap="!isLightRAG"
|
||||
:show-preset="true"
|
||||
:allow-preset-follow-default="true"
|
||||
:database-preset-id="store.database?.additional_params?.chunk_preset_id || 'general'"
|
||||
/>
|
||||
</div>
|
||||
</a-modal>
|
||||
|
||||
@ -704,8 +711,25 @@ const indexConfigModalTitle = ref('入库参数配置')
|
||||
const indexParams = ref({
|
||||
chunk_size: 1000,
|
||||
chunk_overlap: 200,
|
||||
qa_separator: ''
|
||||
qa_separator: '',
|
||||
chunk_preset_id: ''
|
||||
})
|
||||
const buildIndexParamsPayload = () => {
|
||||
const payload = {}
|
||||
if (indexParams.value.chunk_preset_id) {
|
||||
payload.chunk_preset_id = indexParams.value.chunk_preset_id
|
||||
}
|
||||
|
||||
if (isLightRAG.value) {
|
||||
payload.qa_separator = indexParams.value.qa_separator || ''
|
||||
return payload
|
||||
}
|
||||
|
||||
return {
|
||||
...indexParams.value,
|
||||
...payload
|
||||
}
|
||||
}
|
||||
const currentIndexFileIds = ref([])
|
||||
const isBatchIndexOperation = ref(false)
|
||||
|
||||
@ -1088,12 +1112,6 @@ const handleBatchIndex = async () => {
|
||||
return
|
||||
}
|
||||
|
||||
if (isLightRAG.value) {
|
||||
await store.indexFiles(validKeys)
|
||||
selectedRowKeys.value = []
|
||||
return
|
||||
}
|
||||
|
||||
currentIndexFileIds.value = [...validKeys]
|
||||
isBatchIndexOperation.value = true
|
||||
indexConfigModalTitle.value = '批量入库参数配置'
|
||||
@ -1172,11 +1190,6 @@ const handleParseFile = async (record) => {
|
||||
|
||||
const handleIndexFile = async (record) => {
|
||||
closePopover(record.file_id)
|
||||
if (isLightRAG.value) {
|
||||
await store.indexFiles([record.file_id])
|
||||
return
|
||||
}
|
||||
|
||||
// 打开参数配置弹窗
|
||||
currentIndexFileIds.value = [record.file_id]
|
||||
isBatchIndexOperation.value = false
|
||||
@ -1189,7 +1202,8 @@ const handleIndexFile = async (record) => {
|
||||
Object.assign(indexParams.value, {
|
||||
chunk_size: 1000,
|
||||
chunk_overlap: 200,
|
||||
qa_separator: ''
|
||||
qa_separator: '',
|
||||
chunk_preset_id: ''
|
||||
})
|
||||
}
|
||||
|
||||
@ -1214,7 +1228,7 @@ const handleReindexFile = async (record) => {
|
||||
const handleIndexConfigConfirm = async () => {
|
||||
try {
|
||||
// 调用 indexFiles 接口 (支持 params)
|
||||
const result = await store.indexFiles(currentIndexFileIds.value, indexParams.value)
|
||||
const result = await store.indexFiles(currentIndexFileIds.value, buildIndexParamsPayload())
|
||||
if (result) {
|
||||
currentIndexFileIds.value = []
|
||||
// 清空选择
|
||||
@ -1228,7 +1242,8 @@ const handleIndexConfigConfirm = async () => {
|
||||
Object.assign(indexParams.value, {
|
||||
chunk_size: 1000,
|
||||
chunk_overlap: 200,
|
||||
qa_separator: ''
|
||||
qa_separator: '',
|
||||
chunk_preset_id: ''
|
||||
})
|
||||
} else {
|
||||
// message.error(`入库失败: ${result.message}`); // store already shows message
|
||||
@ -1249,7 +1264,8 @@ const handleIndexConfigCancel = () => {
|
||||
Object.assign(indexParams.value, {
|
||||
chunk_size: 1000,
|
||||
chunk_overlap: 200,
|
||||
qa_separator: ''
|
||||
qa_separator: '',
|
||||
chunk_preset_id: ''
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@ -108,15 +108,17 @@
|
||||
<div class="col-item">
|
||||
<div class="setting-label">入库参数配置</div>
|
||||
<div class="setting-content">
|
||||
<template v-if="!isGraphBased">
|
||||
<ChunkParamsConfig :temp-chunk-params="indexParams" :show-qa-split="true" />
|
||||
</template>
|
||||
<template v-else>
|
||||
<div class="lightrag-tip">
|
||||
<Info :size="14" style="margin-right: 6px" />
|
||||
<span>LightRAG 将使用默认参数自动入库</span>
|
||||
</div>
|
||||
</template>
|
||||
<ChunkParamsConfig
|
||||
:temp-chunk-params="indexParams"
|
||||
:show-qa-split="true"
|
||||
:show-chunk-size-overlap="!isGraphBased"
|
||||
:show-preset="true"
|
||||
:allow-preset-follow-default="true"
|
||||
:database-preset-id="store.database?.additional_params?.chunk_preset_id || 'general'"
|
||||
/>
|
||||
<p v-if="isGraphBased" class="param-description">
|
||||
LightRAG 按分隔符预切分,超长片段仍会按 token 大小继续切分。
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@ -564,9 +566,26 @@ const autoIndex = ref(false)
|
||||
const indexParams = ref({
|
||||
chunk_size: 1000,
|
||||
chunk_overlap: 200,
|
||||
qa_separator: ''
|
||||
qa_separator: '',
|
||||
chunk_preset_id: ''
|
||||
})
|
||||
|
||||
const buildAutoIndexParams = () => {
|
||||
const payload = {}
|
||||
if (indexParams.value.chunk_preset_id) {
|
||||
payload.chunk_preset_id = indexParams.value.chunk_preset_id
|
||||
}
|
||||
|
||||
if (isGraphBased.value) {
|
||||
payload.qa_separator = indexParams.value.qa_separator || ''
|
||||
return payload
|
||||
}
|
||||
return {
|
||||
...indexParams.value,
|
||||
...payload
|
||||
}
|
||||
}
|
||||
|
||||
// 计算属性:是否支持QA分割
|
||||
const isQaSplitSupported = computed(() => {
|
||||
const type = kbType.value?.toLowerCase()
|
||||
@ -990,7 +1009,7 @@ const chunkData = async () => {
|
||||
const params = { ...chunkParams.value }
|
||||
if (autoIndex.value) {
|
||||
params.auto_index = true
|
||||
Object.assign(params, indexParams.value)
|
||||
Object.assign(params, buildAutoIndexParams())
|
||||
}
|
||||
|
||||
// 构造 _preprocessed_map 和 items (minio urls)
|
||||
@ -1068,7 +1087,7 @@ const chunkData = async () => {
|
||||
const params = { ...chunkParams.value, content_hashes }
|
||||
if (autoIndex.value) {
|
||||
params.auto_index = true
|
||||
Object.assign(params, indexParams.value)
|
||||
Object.assign(params, buildAutoIndexParams())
|
||||
}
|
||||
|
||||
await store.addFiles({
|
||||
@ -1604,17 +1623,6 @@ const chunkData = async () => {
|
||||
border-radius: 6px;
|
||||
}
|
||||
|
||||
.lightrag-tip {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
margin-top: 8px;
|
||||
padding: 8px 12px;
|
||||
background: var(--main-50);
|
||||
border-radius: 6px;
|
||||
font-size: 13px;
|
||||
color: var(--gray-600);
|
||||
}
|
||||
|
||||
.setting-label .ant-checkbox {
|
||||
margin-right: 8px;
|
||||
}
|
||||
|
||||
@ -40,6 +40,9 @@
|
||||
{{ getKbTypeLabel(database.kb_type || 'lightrag') }}
|
||||
</a-tag>
|
||||
<a-tag color="blue" size="small">{{ database.embed_info?.name || 'N/A' }}</a-tag>
|
||||
<a-tag color="cyan" size="small">{{
|
||||
chunkPresetLabelMap[database.additional_params?.chunk_preset_id || 'general'] || 'General'
|
||||
}}</a-tag>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@ -81,6 +84,18 @@
|
||||
>
|
||||
</a-form-item>
|
||||
|
||||
<a-form-item name="chunk_preset_id">
|
||||
<template #label>
|
||||
<span class="chunk-preset-label">
|
||||
分块策略
|
||||
<a-tooltip :title="editPresetDescription">
|
||||
<QuestionCircleOutlined class="chunk-preset-help-icon" />
|
||||
</a-tooltip>
|
||||
</span>
|
||||
</template>
|
||||
<a-select v-model:value="editForm.chunk_preset_id" :options="chunkPresetOptions" />
|
||||
</a-form-item>
|
||||
|
||||
<!-- 仅对 LightRAG 类型显示 LLM 配置 -->
|
||||
<a-form-item v-if="database.kb_type === 'lightrag'" label="语言模型 (LLM)" name="llm_info">
|
||||
<ModelSelectorComponent
|
||||
@ -122,8 +137,13 @@ import { useRouter } from 'vue-router'
|
||||
import { useDatabaseStore } from '@/stores/database'
|
||||
import { useUserStore } from '@/stores/user'
|
||||
import { getKbTypeLabel, getKbTypeColor } from '@/utils/kb_utils'
|
||||
import {
|
||||
CHUNK_PRESET_OPTIONS,
|
||||
CHUNK_PRESET_LABEL_MAP,
|
||||
getChunkPresetDescription
|
||||
} from '@/utils/chunk_presets'
|
||||
import { message } from 'ant-design-vue'
|
||||
import { LeftOutlined } from '@ant-design/icons-vue'
|
||||
import { LeftOutlined, QuestionCircleOutlined } from '@ant-design/icons-vue'
|
||||
import { Pencil, Trash2, Copy } from 'lucide-vue-next'
|
||||
import { departmentApi } from '@/apis/department_api'
|
||||
import ModelSelectorComponent from '@/components/ModelSelectorComponent.vue'
|
||||
@ -220,12 +240,17 @@ const editForm = reactive({
|
||||
name: '',
|
||||
description: '',
|
||||
auto_generate_questions: false,
|
||||
chunk_preset_id: 'general',
|
||||
llm_info: {
|
||||
provider: '',
|
||||
model_name: ''
|
||||
}
|
||||
})
|
||||
|
||||
const chunkPresetOptions = CHUNK_PRESET_OPTIONS.map(({ label, value }) => ({ label, value }))
|
||||
const chunkPresetLabelMap = CHUNK_PRESET_LABEL_MAP
|
||||
const editPresetDescription = computed(() => getChunkPresetDescription(editForm.chunk_preset_id))
|
||||
|
||||
const rules = {
|
||||
name: [{ required: true, message: '请输入知识库名称' }]
|
||||
}
|
||||
@ -238,6 +263,7 @@ const showEditModal = () => {
|
||||
editForm.description = database.value.description || ''
|
||||
editForm.auto_generate_questions =
|
||||
database.value.additional_params?.auto_generate_questions || false
|
||||
editForm.chunk_preset_id = database.value.additional_params?.chunk_preset_id || 'general'
|
||||
|
||||
// 如果是 LightRAG 类型,加载当前的 LLM 配置
|
||||
if (database.value.kb_type === 'lightrag') {
|
||||
@ -283,7 +309,8 @@ const handleEditSubmit = () => {
|
||||
name: editForm.name,
|
||||
description: editForm.description,
|
||||
additional_params: {
|
||||
auto_generate_questions: editForm.auto_generate_questions
|
||||
auto_generate_questions: editForm.auto_generate_questions,
|
||||
chunk_preset_id: editForm.chunk_preset_id || 'general'
|
||||
},
|
||||
share_config: {
|
||||
is_shared: finalIsShared,
|
||||
@ -428,4 +455,16 @@ const deleteDatabase = () => {
|
||||
align-items: center;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
.chunk-preset-label {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.chunk-preset-help-icon {
|
||||
color: var(--gray-500);
|
||||
cursor: help;
|
||||
font-size: 14px;
|
||||
}
|
||||
</style>
|
||||
|
||||
33
web/src/utils/chunk_presets.js
Normal file
33
web/src/utils/chunk_presets.js
Normal file
@ -0,0 +1,33 @@
|
||||
export const CHUNK_PRESET_OPTIONS = [
|
||||
{
|
||||
value: 'general',
|
||||
label: 'General',
|
||||
description: '通用分块:按分隔符和长度切分,适合大多数普通文档。'
|
||||
},
|
||||
{
|
||||
value: 'qa',
|
||||
label: 'QA',
|
||||
description: '问答分块:优先抽取问题-回答结构,适合 FAQ、题库、问答手册。'
|
||||
},
|
||||
{
|
||||
value: 'book',
|
||||
label: 'Book',
|
||||
description: '书籍分块:强化章节标题识别并做层级合并,适合教材、手册、长章节文档。'
|
||||
},
|
||||
{
|
||||
value: 'laws',
|
||||
label: 'Laws',
|
||||
description: '法规分块:按法条层级组织与合并,适合法律法规、制度规范类文本。'
|
||||
}
|
||||
]
|
||||
|
||||
export const CHUNK_PRESET_LABEL_MAP = Object.fromEntries(
|
||||
CHUNK_PRESET_OPTIONS.map((item) => [item.value, item.label])
|
||||
)
|
||||
|
||||
export const CHUNK_PRESET_DESCRIPTION_MAP = Object.fromEntries(
|
||||
CHUNK_PRESET_OPTIONS.map((item) => [item.value, item.description])
|
||||
)
|
||||
|
||||
export const getChunkPresetDescription = (presetId) =>
|
||||
CHUNK_PRESET_DESCRIPTION_MAP[presetId] || CHUNK_PRESET_DESCRIPTION_MAP.general
|
||||
@ -56,6 +56,19 @@
|
||||
placeholder="请选择嵌入模型"
|
||||
/>
|
||||
|
||||
<div class="chunk-preset-title-row">
|
||||
<h3 style="margin: 0">分块策略</h3>
|
||||
<a-tooltip :title="selectedPresetDescription">
|
||||
<QuestionCircleOutlined class="chunk-preset-help-icon" />
|
||||
</a-tooltip>
|
||||
</div>
|
||||
<a-select
|
||||
v-model:value="newDatabase.chunk_preset_id"
|
||||
:options="chunkPresetOptions"
|
||||
style="width: 100%"
|
||||
size="large"
|
||||
/>
|
||||
|
||||
<!-- 仅对 LightRAG 提供语言选择和LLM选择 -->
|
||||
<div v-if="newDatabase.kb_type === 'lightrag'">
|
||||
<h3 style="margin-top: 20px">语言</h3>
|
||||
@ -197,7 +210,7 @@ import { useRouter, useRoute } from 'vue-router'
|
||||
import { storeToRefs } from 'pinia'
|
||||
import { useConfigStore } from '@/stores/config'
|
||||
import { useDatabaseStore } from '@/stores/database'
|
||||
import { LockOutlined, InfoCircleOutlined, PlusOutlined } from '@ant-design/icons-vue'
|
||||
import { LockOutlined, PlusOutlined, QuestionCircleOutlined } from '@ant-design/icons-vue'
|
||||
import { typeApi } from '@/apis/knowledge_api'
|
||||
import HeaderComponent from '@/components/HeaderComponent.vue'
|
||||
import ModelSelectorComponent from '@/components/ModelSelectorComponent.vue'
|
||||
@ -206,6 +219,7 @@ import ShareConfigForm from '@/components/ShareConfigForm.vue'
|
||||
import dayjs, { parseToShanghai } from '@/utils/time'
|
||||
import AiTextarea from '@/components/AiTextarea.vue'
|
||||
import { getKbTypeLabel, getKbTypeIcon, getKbTypeColor } from '@/utils/kb_utils'
|
||||
import { CHUNK_PRESET_OPTIONS, getChunkPresetDescription } from '@/utils/chunk_presets'
|
||||
|
||||
const route = useRoute()
|
||||
const router = useRouter()
|
||||
@ -240,6 +254,8 @@ const languageOptions = [
|
||||
{ label: '印地语 Hindi', value: 'Hindi' }
|
||||
]
|
||||
|
||||
const chunkPresetOptions = CHUNK_PRESET_OPTIONS.map(({ label, value }) => ({ label, value }))
|
||||
|
||||
const createEmptyDatabaseForm = () => ({
|
||||
name: '',
|
||||
description: '',
|
||||
@ -247,6 +263,7 @@ const createEmptyDatabaseForm = () => ({
|
||||
kb_type: 'milvus',
|
||||
is_private: false,
|
||||
storage: '',
|
||||
chunk_preset_id: 'general',
|
||||
language: 'Chinese',
|
||||
llm_info: {
|
||||
provider: '',
|
||||
@ -256,6 +273,10 @@ const createEmptyDatabaseForm = () => ({
|
||||
|
||||
const newDatabase = reactive(createEmptyDatabaseForm())
|
||||
|
||||
const selectedPresetDescription = computed(() =>
|
||||
getChunkPresetDescription(newDatabase.chunk_preset_id)
|
||||
)
|
||||
|
||||
const llmModelSpec = computed(() => {
|
||||
const provider = newDatabase.llm_info?.provider || ''
|
||||
const modelName = newDatabase.llm_info?.model_name || ''
|
||||
@ -364,7 +385,8 @@ const buildRequestData = () => {
|
||||
embed_model_name: newDatabase.embed_model_name || configStore.config.embed_model,
|
||||
kb_type: newDatabase.kb_type,
|
||||
additional_params: {
|
||||
is_private: newDatabase.is_private || false
|
||||
is_private: newDatabase.is_private || false,
|
||||
chunk_preset_id: newDatabase.chunk_preset_id || 'general'
|
||||
}
|
||||
}
|
||||
|
||||
@ -429,6 +451,20 @@ onMounted(() => {
|
||||
|
||||
<style lang="less" scoped>
|
||||
.new-database-modal {
|
||||
.chunk-preset-title-row {
|
||||
margin-top: 20px;
|
||||
margin-bottom: 8px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.chunk-preset-help-icon {
|
||||
color: var(--gray-500);
|
||||
cursor: help;
|
||||
font-size: 14px;
|
||||
}
|
||||
|
||||
.kb-type-guide {
|
||||
margin: 12px 0;
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user