diff --git a/backend/package/yuxi/knowledge/indexing.py b/backend/package/yuxi/knowledge/indexing.py deleted file mode 100644 index 89702f7a..00000000 --- a/backend/package/yuxi/knowledge/indexing.py +++ /dev/null @@ -1,100 +0,0 @@ -"""Knowledge text chunking helpers. - -Parser and markdown conversion logic has been moved to ``yuxi.plugins.parser``. -This module only keeps chunking-related utilities. -""" - -from pathlib import Path - -from langchain_community.document_loaders import ( - CSVLoader, - JSONLoader, - TextLoader, - UnstructuredHTMLLoader, - UnstructuredMarkdownLoader, - UnstructuredWordDocumentLoader, -) -from langchain_text_splitters import RecursiveCharacterTextSplitter - - -def chunk_with_parser(file_path, params=None): - """ - 使用文件解析器将文件切分成固定大小的块 - - Args: - file_path: 文件路径 - params: 参数 - """ - params = params or {} - chunk_size = int(params.get("chunk_size", 500)) - chunk_overlap = int(params.get("chunk_overlap", 100)) - - file_type = Path(file_path).suffix.lower() - - # 选择合适的加载器 - if file_type in [".txt"]: - loader = TextLoader(file_path) - - elif file_type in [".md"]: - loader = UnstructuredMarkdownLoader(file_path) - - elif file_type in [".docx", ".doc"]: - loader = UnstructuredWordDocumentLoader(file_path) - - elif file_type in [".html", ".htm"]: - loader = UnstructuredHTMLLoader(file_path) - - elif file_type in [".json"]: - loader = JSONLoader(file_path, jq_schema=".") - - elif file_type in [".csv"]: - loader = CSVLoader(file_path) - - else: - raise ValueError(f"不支持的文件类型: {file_type}") - - # 加载文档 - docs = loader.load() - - # 创建文本分割器 - text_splitter = RecursiveCharacterTextSplitter( - chunk_size=chunk_size, - chunk_overlap=chunk_overlap, - separators=["\n\n", "\n", ".", " ", ""], - ) - - # 分割文档 - nodes = text_splitter.split_documents(docs) - - # 添加序号信息到metadata - for i, node in enumerate(nodes): - if node.metadata is None: - node.metadata = {} - node.metadata["chunk_idx"] = i - - return nodes - - -def chunk_text(text, params=None): - """ - 将文本切分成固定大小的块 - """ - params = params or {} - chunk_size = int(params.get("chunk_size", 500)) - chunk_overlap = int(params.get("chunk_overlap", 100)) - - # 创建文本分割器 - text_splitter = RecursiveCharacterTextSplitter( - chunk_size=chunk_size, chunk_overlap=chunk_overlap, separators=["\n\n", "\n", ".", " ", ""] - ) - - # 分割文档 - nodes = text_splitter.split_text(text) - - # 添加序号信息到metadata - nodes = [{"text": node, "metadata": {"chunk_idx": i}} for i, node in enumerate(nodes)] - return nodes - - -def chunk(text_or_path, params=None): - raise NotImplementedError("chunk is deprecated, use chunk_with_parser or chunk_text instead") diff --git a/backend/package/yuxi/services/chat_stream_service.py b/backend/package/yuxi/services/chat_stream_service.py index 5465b3ae..870ea486 100644 --- a/backend/package/yuxi/services/chat_stream_service.py +++ b/backend/package/yuxi/services/chat_stream_service.py @@ -16,7 +16,6 @@ from yuxi.repositories.conversation_repository import ConversationRepository from yuxi.storage.postgres.manager import pg_manager from yuxi.storage.postgres.models_business import User from yuxi.utils.logging_config import logger -from yuxi import knowledge_base from yuxi.utils.question_utils import ( normalize_options as _normalize_interrupt_options, ) @@ -441,24 +440,6 @@ async def stream_agent_chat( # LangGraph 会自动从 checkpointer 恢复 state(包括 uploads) # 无需手动加载或传递 - # 根据用户权限过滤知识库 - requested_knowledge_names = input_context.get("knowledges") - logger.info(f"Requesting knowledges: {requested_knowledge_names}") - if requested_knowledge_names and isinstance(requested_knowledge_names, list) and requested_knowledge_names: - user_info = {"role": "user", "department_id": department_id} - accessible_databases = await knowledge_base.get_databases_by_user(user_info) - accessible_kb_names = { - db.get("name") - for db in accessible_databases.get("databases", []) - if isinstance(db, dict) and db.get("name") - } - logger.info(f"Accessible knowledges: {accessible_kb_names}") - - filtered_knowledge_names = [kb for kb in requested_knowledge_names if kb in accessible_kb_names] - blocked_knowledge_names = [kb for kb in requested_knowledge_names if kb not in accessible_kb_names] - if blocked_knowledge_names: - logger.warning(f"用户 {user_id} 无权访问知识库: {blocked_knowledge_names}, 已自动过滤") - input_context["knowledges"] = filtered_knowledge_names full_msg = None accumulated_content = [] async for msg, metadata in agent.stream_messages(messages, input_context=input_context):