fix: 移除 chat_stream_service.py 中的知识库权限过滤逻辑,已在工具层添加
This commit is contained in:
parent
dc88874118
commit
aeed32264f
@ -1,100 +0,0 @@
|
||||
"""Knowledge text chunking helpers.
|
||||
|
||||
Parser and markdown conversion logic has been moved to ``yuxi.plugins.parser``.
|
||||
This module only keeps chunking-related utilities.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from langchain_community.document_loaders import (
|
||||
CSVLoader,
|
||||
JSONLoader,
|
||||
TextLoader,
|
||||
UnstructuredHTMLLoader,
|
||||
UnstructuredMarkdownLoader,
|
||||
UnstructuredWordDocumentLoader,
|
||||
)
|
||||
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
||||
|
||||
|
||||
def chunk_with_parser(file_path, params=None):
|
||||
"""
|
||||
使用文件解析器将文件切分成固定大小的块
|
||||
|
||||
Args:
|
||||
file_path: 文件路径
|
||||
params: 参数
|
||||
"""
|
||||
params = params or {}
|
||||
chunk_size = int(params.get("chunk_size", 500))
|
||||
chunk_overlap = int(params.get("chunk_overlap", 100))
|
||||
|
||||
file_type = Path(file_path).suffix.lower()
|
||||
|
||||
# 选择合适的加载器
|
||||
if file_type in [".txt"]:
|
||||
loader = TextLoader(file_path)
|
||||
|
||||
elif file_type in [".md"]:
|
||||
loader = UnstructuredMarkdownLoader(file_path)
|
||||
|
||||
elif file_type in [".docx", ".doc"]:
|
||||
loader = UnstructuredWordDocumentLoader(file_path)
|
||||
|
||||
elif file_type in [".html", ".htm"]:
|
||||
loader = UnstructuredHTMLLoader(file_path)
|
||||
|
||||
elif file_type in [".json"]:
|
||||
loader = JSONLoader(file_path, jq_schema=".")
|
||||
|
||||
elif file_type in [".csv"]:
|
||||
loader = CSVLoader(file_path)
|
||||
|
||||
else:
|
||||
raise ValueError(f"不支持的文件类型: {file_type}")
|
||||
|
||||
# 加载文档
|
||||
docs = loader.load()
|
||||
|
||||
# 创建文本分割器
|
||||
text_splitter = RecursiveCharacterTextSplitter(
|
||||
chunk_size=chunk_size,
|
||||
chunk_overlap=chunk_overlap,
|
||||
separators=["\n\n", "\n", ".", " ", ""],
|
||||
)
|
||||
|
||||
# 分割文档
|
||||
nodes = text_splitter.split_documents(docs)
|
||||
|
||||
# 添加序号信息到metadata
|
||||
for i, node in enumerate(nodes):
|
||||
if node.metadata is None:
|
||||
node.metadata = {}
|
||||
node.metadata["chunk_idx"] = i
|
||||
|
||||
return nodes
|
||||
|
||||
|
||||
def chunk_text(text, params=None):
|
||||
"""
|
||||
将文本切分成固定大小的块
|
||||
"""
|
||||
params = params or {}
|
||||
chunk_size = int(params.get("chunk_size", 500))
|
||||
chunk_overlap = int(params.get("chunk_overlap", 100))
|
||||
|
||||
# 创建文本分割器
|
||||
text_splitter = RecursiveCharacterTextSplitter(
|
||||
chunk_size=chunk_size, chunk_overlap=chunk_overlap, separators=["\n\n", "\n", ".", " ", ""]
|
||||
)
|
||||
|
||||
# 分割文档
|
||||
nodes = text_splitter.split_text(text)
|
||||
|
||||
# 添加序号信息到metadata
|
||||
nodes = [{"text": node, "metadata": {"chunk_idx": i}} for i, node in enumerate(nodes)]
|
||||
return nodes
|
||||
|
||||
|
||||
def chunk(text_or_path, params=None):
|
||||
raise NotImplementedError("chunk is deprecated, use chunk_with_parser or chunk_text instead")
|
||||
@ -16,7 +16,6 @@ from yuxi.repositories.conversation_repository import ConversationRepository
|
||||
from yuxi.storage.postgres.manager import pg_manager
|
||||
from yuxi.storage.postgres.models_business import User
|
||||
from yuxi.utils.logging_config import logger
|
||||
from yuxi import knowledge_base
|
||||
from yuxi.utils.question_utils import (
|
||||
normalize_options as _normalize_interrupt_options,
|
||||
)
|
||||
@ -441,24 +440,6 @@ async def stream_agent_chat(
|
||||
# LangGraph 会自动从 checkpointer 恢复 state(包括 uploads)
|
||||
# 无需手动加载或传递
|
||||
|
||||
# 根据用户权限过滤知识库
|
||||
requested_knowledge_names = input_context.get("knowledges")
|
||||
logger.info(f"Requesting knowledges: {requested_knowledge_names}")
|
||||
if requested_knowledge_names and isinstance(requested_knowledge_names, list) and requested_knowledge_names:
|
||||
user_info = {"role": "user", "department_id": department_id}
|
||||
accessible_databases = await knowledge_base.get_databases_by_user(user_info)
|
||||
accessible_kb_names = {
|
||||
db.get("name")
|
||||
for db in accessible_databases.get("databases", [])
|
||||
if isinstance(db, dict) and db.get("name")
|
||||
}
|
||||
logger.info(f"Accessible knowledges: {accessible_kb_names}")
|
||||
|
||||
filtered_knowledge_names = [kb for kb in requested_knowledge_names if kb in accessible_kb_names]
|
||||
blocked_knowledge_names = [kb for kb in requested_knowledge_names if kb not in accessible_kb_names]
|
||||
if blocked_knowledge_names:
|
||||
logger.warning(f"用户 {user_id} 无权访问知识库: {blocked_knowledge_names}, 已自动过滤")
|
||||
input_context["knowledges"] = filtered_knowledge_names
|
||||
full_msg = None
|
||||
accumulated_content = []
|
||||
async for msg, metadata in agent.stream_messages(messages, input_context=input_context):
|
||||
|
||||
Loading…
Reference in New Issue
Block a user