fix: 移除 chat_stream_service.py 中的知识库权限过滤逻辑，已在工具层添加
This commit is contained in:
Wenjie Zhang
parent
dc88874118
commit
aeed32264f
@ -1,100 +0,0 @@
|
|||||||
"""Knowledge text chunking helpers.
|
|
||||||
|
|
||||||
Parser and markdown conversion logic has been moved to ``yuxi.plugins.parser``.
|
|
||||||
This module only keeps chunking-related utilities.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
from langchain_community.document_loaders import (
|
|
||||||
CSVLoader,
|
|
||||||
JSONLoader,
|
|
||||||
TextLoader,
|
|
||||||
UnstructuredHTMLLoader,
|
|
||||||
UnstructuredMarkdownLoader,
|
|
||||||
UnstructuredWordDocumentLoader,
|
|
||||||
)
|
|
||||||
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
|
||||||
|
|
||||||
|
|
||||||
def chunk_with_parser(file_path, params=None):
|
|
||||||
"""
|
|
||||||
使用文件解析器将文件切分成固定大小的块
|
|
||||||
|
|
||||||
Args:
|
|
||||||
file_path: 文件路径
|
|
||||||
params: 参数
|
|
||||||
"""
|
|
||||||
params = params or {}
|
|
||||||
chunk_size = int(params.get("chunk_size", 500))
|
|
||||||
chunk_overlap = int(params.get("chunk_overlap", 100))
|
|
||||||
|
|
||||||
file_type = Path(file_path).suffix.lower()
|
|
||||||
|
|
||||||
# 选择合适的加载器
|
|
||||||
if file_type in [".txt"]:
|
|
||||||
loader = TextLoader(file_path)
|
|
||||||
|
|
||||||
elif file_type in [".md"]:
|
|
||||||
loader = UnstructuredMarkdownLoader(file_path)
|
|
||||||
|
|
||||||
elif file_type in [".docx", ".doc"]:
|
|
||||||
loader = UnstructuredWordDocumentLoader(file_path)
|
|
||||||
|
|
||||||
elif file_type in [".html", ".htm"]:
|
|
||||||
loader = UnstructuredHTMLLoader(file_path)
|
|
||||||
|
|
||||||
elif file_type in [".json"]:
|
|
||||||
loader = JSONLoader(file_path, jq_schema=".")
|
|
||||||
|
|
||||||
elif file_type in [".csv"]:
|
|
||||||
loader = CSVLoader(file_path)
|
|
||||||
|
|
||||||
else:
|
|
||||||
raise ValueError(f"不支持的文件类型: {file_type}")
|
|
||||||
|
|
||||||
# 加载文档
|
|
||||||
docs = loader.load()
|
|
||||||
|
|
||||||
# 创建文本分割器
|
|
||||||
text_splitter = RecursiveCharacterTextSplitter(
|
|
||||||
chunk_size=chunk_size,
|
|
||||||
chunk_overlap=chunk_overlap,
|
|
||||||
separators=["\n\n", "\n", ".", " ", ""],
|
|
||||||
)
|
|
||||||
|
|
||||||
# 分割文档
|
|
||||||
nodes = text_splitter.split_documents(docs)
|
|
||||||
|
|
||||||
# 添加序号信息到metadata
|
|
||||||
for i, node in enumerate(nodes):
|
|
||||||
if node.metadata is None:
|
|
||||||
node.metadata = {}
|
|
||||||
node.metadata["chunk_idx"] = i
|
|
||||||
|
|
||||||
return nodes
|
|
||||||
|
|
||||||
|
|
||||||
def chunk_text(text, params=None):
|
|
||||||
"""
|
|
||||||
将文本切分成固定大小的块
|
|
||||||
"""
|
|
||||||
params = params or {}
|
|
||||||
chunk_size = int(params.get("chunk_size", 500))
|
|
||||||
chunk_overlap = int(params.get("chunk_overlap", 100))
|
|
||||||
|
|
||||||
# 创建文本分割器
|
|
||||||
text_splitter = RecursiveCharacterTextSplitter(
|
|
||||||
chunk_size=chunk_size, chunk_overlap=chunk_overlap, separators=["\n\n", "\n", ".", " ", ""]
|
|
||||||
)
|
|
||||||
|
|
||||||
# 分割文档
|
|
||||||
nodes = text_splitter.split_text(text)
|
|
||||||
|
|
||||||
# 添加序号信息到metadata
|
|
||||||
nodes = [{"text": node, "metadata": {"chunk_idx": i}} for i, node in enumerate(nodes)]
|
|
||||||
return nodes
|
|
||||||
|
|
||||||
|
|
||||||
def chunk(text_or_path, params=None):
|
|
||||||
raise NotImplementedError("chunk is deprecated, use chunk_with_parser or chunk_text instead")
|
|
||||||
@ -16,7 +16,6 @@ from yuxi.repositories.conversation_repository import ConversationRepository
|
|||||||
from yuxi.storage.postgres.manager import pg_manager
|
from yuxi.storage.postgres.manager import pg_manager
|
||||||
from yuxi.storage.postgres.models_business import User
|
from yuxi.storage.postgres.models_business import User
|
||||||
from yuxi.utils.logging_config import logger
|
from yuxi.utils.logging_config import logger
|
||||||
from yuxi import knowledge_base
|
|
||||||
from yuxi.utils.question_utils import (
|
from yuxi.utils.question_utils import (
|
||||||
normalize_options as _normalize_interrupt_options,
|
normalize_options as _normalize_interrupt_options,
|
||||||
)
|
)
|
||||||
@ -441,24 +440,6 @@ async def stream_agent_chat(
|
|||||||
# LangGraph 会自动从 checkpointer 恢复 state（包括 uploads）
|
# LangGraph 会自动从 checkpointer 恢复 state（包括 uploads）
|
||||||
# 无需手动加载或传递
|
# 无需手动加载或传递
|
||||||
|
|
||||||
# 根据用户权限过滤知识库
|
|
||||||
requested_knowledge_names = input_context.get("knowledges")
|
|
||||||
logger.info(f"Requesting knowledges: {requested_knowledge_names}")
|
|
||||||
if requested_knowledge_names and isinstance(requested_knowledge_names, list) and requested_knowledge_names:
|
|
||||||
user_info = {"role": "user", "department_id": department_id}
|
|
||||||
accessible_databases = await knowledge_base.get_databases_by_user(user_info)
|
|
||||||
accessible_kb_names = {
|
|
||||||
db.get("name")
|
|
||||||
for db in accessible_databases.get("databases", [])
|
|
||||||
if isinstance(db, dict) and db.get("name")
|
|
||||||
}
|
|
||||||
logger.info(f"Accessible knowledges: {accessible_kb_names}")
|
|
||||||
|
|
||||||
filtered_knowledge_names = [kb for kb in requested_knowledge_names if kb in accessible_kb_names]
|
|
||||||
blocked_knowledge_names = [kb for kb in requested_knowledge_names if kb not in accessible_kb_names]
|
|
||||||
if blocked_knowledge_names:
|
|
||||||
logger.warning(f"用户 {user_id} 无权访问知识库: {blocked_knowledge_names}, 已自动过滤")
|
|
||||||
input_context["knowledges"] = filtered_knowledge_names
|
|
||||||
full_msg = None
|
full_msg = None
|
||||||
accumulated_content = []
|
accumulated_content = []
|
||||||
async for msg, metadata in agent.stream_messages(messages, input_context=input_context):
|
async for msg, metadata in agent.stream_messages(messages, input_context=input_context):
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user