From 2112d4833d7447e651ece5358e36a5c3ed224ad2 Mon Sep 17 00:00:00 2001 From: Wenjie Zhang Date: Tue, 23 Sep 2025 13:07:50 +0800 Subject: [PATCH] =?UTF-8?q?refactor(knowledge):=20=E9=87=8D=E6=9E=84?= =?UTF-8?q?=E7=9F=A5=E8=AF=86=E5=BA=93=E6=A8=A1=E5=9D=97=EF=BC=8C=E4=BC=98?= =?UTF-8?q?=E5=8C=96=E6=9E=B6=E6=9E=84=E5=92=8C=E5=AE=9E=E7=8E=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 优化知识库文件夹结构，增强可维护性 - 新增知识库管理器和工厂类，统一管理不同类型的知识库实例 - 实现知识库的基本操作接口，包括创建、更新和删除数据库 - 引入新的知识库实现类（ChromaKB、MilvusKB、LightRagKB），支持多种存储后端 - 增强文件路径验证，确保安全性 - 更新文档，提供新的知识库接口说明 --- docs/changelog/update.md | 1 + src/knowledge/__init__.py | 12 +++++----- src/knowledge/{knowledge_base.py => base.py} | 0 src/knowledge/{kb_factory.py => factory.py} | 2 +- src/knowledge/{graphbase.py => graph.py} | 0 src/knowledge/implementations/__init__.py | 13 +++++++++++ .../chroma.py} | 4 ++-- .../lightrag.py} | 4 ++-- .../milvus.py} | 4 ++-- src/knowledge/{kb_manager.py => manager.py} | 4 ++-- src/knowledge/utils/__init__.py | 22 +++++++++++++++++++ src/knowledge/{ => utils}/kb_utils.py | 0 12 files changed, 51 insertions(+), 15 deletions(-) rename src/knowledge/{knowledge_base.py => base.py} (100%) rename src/knowledge/{kb_factory.py => factory.py} (97%) rename src/knowledge/{graphbase.py => graph.py} (100%) create mode 100644 src/knowledge/implementations/__init__.py rename src/knowledge/{chroma_kb.py => implementations/chroma.py} (99%) rename src/knowledge/{lightrag_kb.py => implementations/lightrag.py} (99%) rename src/knowledge/{milvus_kb.py => implementations/milvus.py} (99%) rename src/knowledge/{kb_manager.py => manager.py} (99%) create mode 100644 src/knowledge/utils/__init__.py rename src/knowledge/{ => utils}/kb_utils.py (100%) diff --git a/docs/changelog/update.md b/docs/changelog/update.md index e59edecf..9ea46feb 100644 --- a/docs/changelog/update.md +++ b/docs/changelog/update.md @@ -27,6 +27,7 @@ - [x] 添加内容审查功能 - [x] 补充 embedding 模型和 reranker 模型的配置说明 - [ ] 支持 MinerU 的解析方法 +- [x] 优化 knowledge 文件夹下面的架构 📝 **Base** diff --git a/src/knowledge/__init__.py b/src/knowledge/__init__.py index fc0b48b6..f4a342f9 100644 --- a/src/knowledge/__init__.py +++ b/src/knowledge/__init__.py @@ -1,12 +1,12 @@ import os from ..config import config -from .chroma_kb import ChromaKB -from .graphbase import GraphDatabase -from .kb_factory import KnowledgeBaseFactory -from .kb_manager import KnowledgeBaseManager -from .lightrag_kb import LightRagKB -from .milvus_kb import MilvusKB +from .implementations.chroma import ChromaKB +from .graph import GraphDatabase +from .factory import KnowledgeBaseFactory +from .manager import KnowledgeBaseManager +from .implementations.lightrag import LightRagKB +from .implementations.milvus import MilvusKB # 注册知识库类型 KnowledgeBaseFactory.register("chroma", ChromaKB, {"description": "基于 ChromaDB 的轻量级向量知识库，适合开发和小规模"}) diff --git a/src/knowledge/knowledge_base.py b/src/knowledge/base.py similarity index 100% rename from src/knowledge/knowledge_base.py rename to src/knowledge/base.py diff --git a/src/knowledge/kb_factory.py b/src/knowledge/factory.py similarity index 97% rename from src/knowledge/kb_factory.py rename to src/knowledge/factory.py index dfd1ba71..5778f63d 100644 --- a/src/knowledge/kb_factory.py +++ b/src/knowledge/factory.py @@ -1,4 +1,4 @@ -from src.knowledge.knowledge_base import KBNotFoundError, KnowledgeBase +from src.knowledge.base import KBNotFoundError, KnowledgeBase from src.utils import logger diff --git a/src/knowledge/graphbase.py b/src/knowledge/graph.py similarity index 100% rename from src/knowledge/graphbase.py rename to src/knowledge/graph.py diff --git a/src/knowledge/implementations/__init__.py b/src/knowledge/implementations/__init__.py new file mode 100644 index 00000000..abc0ecd2 --- /dev/null +++ b/src/knowledge/implementations/__init__.py @@ -0,0 +1,13 @@ +"""知识库具体实现模块 + +包含各种知识库的具体实现： +- ChromaKB: 基于 ChromaDB 的向量知识库 +- MilvusKB: 基于 Milvus 的向量知识库 +- LightRagKB: 基于 LightRAG 的图检索知识库 +""" + +from .chroma import ChromaKB +from .milvus import MilvusKB +from .lightrag import LightRagKB + +__all__ = ["ChromaKB", "MilvusKB", "LightRagKB"] diff --git a/src/knowledge/chroma_kb.py b/src/knowledge/implementations/chroma.py similarity index 99% rename from src/knowledge/chroma_kb.py rename to src/knowledge/implementations/chroma.py index 71eb4cf9..1dfe70b3 100644 --- a/src/knowledge/chroma_kb.py +++ b/src/knowledge/implementations/chroma.py @@ -8,13 +8,13 @@ from chromadb.config import Settings from chromadb.utils.embedding_functions import OpenAIEmbeddingFunction from src.knowledge.indexing import process_file_to_markdown, process_url_to_markdown -from src.knowledge.kb_utils import ( +from src.knowledge.utils.kb_utils import ( get_embedding_config, prepare_item_metadata, split_text_into_chunks, split_text_into_qa_chunks, ) -from src.knowledge.knowledge_base import KnowledgeBase +from src.knowledge.base import KnowledgeBase from src.utils import logger diff --git a/src/knowledge/lightrag_kb.py b/src/knowledge/implementations/lightrag.py similarity index 99% rename from src/knowledge/lightrag_kb.py rename to src/knowledge/implementations/lightrag.py index d1ce5393..da4a5af6 100644 --- a/src/knowledge/lightrag_kb.py +++ b/src/knowledge/implementations/lightrag.py @@ -10,8 +10,8 @@ from neo4j import GraphDatabase from pymilvus import connections, utility from src.knowledge.indexing import process_file_to_markdown, process_url_to_markdown -from src.knowledge.kb_utils import get_embedding_config, prepare_item_metadata -from src.knowledge.knowledge_base import KnowledgeBase +from src.knowledge.utils.kb_utils import get_embedding_config, prepare_item_metadata +from src.knowledge.base import KnowledgeBase from src.utils import hashstr, logger LIGHTRAG_LLM_PROVIDER = os.getenv("LIGHTRAG_LLM_PROVIDER", "siliconflow") diff --git a/src/knowledge/milvus_kb.py b/src/knowledge/implementations/milvus.py similarity index 99% rename from src/knowledge/milvus_kb.py rename to src/knowledge/implementations/milvus.py index 0843cf86..aa6554b3 100644 --- a/src/knowledge/milvus_kb.py +++ b/src/knowledge/implementations/milvus.py @@ -7,13 +7,13 @@ from typing import Any from pymilvus import Collection, CollectionSchema, DataType, FieldSchema, connections, db, utility from src.knowledge.indexing import process_file_to_markdown, process_url_to_markdown -from src.knowledge.kb_utils import ( +from src.knowledge.utils.kb_utils import ( get_embedding_config, prepare_item_metadata, split_text_into_chunks, split_text_into_qa_chunks, ) -from src.knowledge.knowledge_base import KnowledgeBase +from src.knowledge.base import KnowledgeBase from src.models.embed import OtherEmbedding from src.utils import hashstr, logger diff --git a/src/knowledge/kb_manager.py b/src/knowledge/manager.py similarity index 99% rename from src/knowledge/kb_manager.py rename to src/knowledge/manager.py index eb3cd06b..08a2bf49 100644 --- a/src/knowledge/kb_manager.py +++ b/src/knowledge/manager.py @@ -3,8 +3,8 @@ import json import os from datetime import datetime -from src.knowledge.kb_factory import KnowledgeBaseFactory -from src.knowledge.knowledge_base import KBNotFoundError, KnowledgeBase +from src.knowledge.factory import KnowledgeBaseFactory +from src.knowledge.base import KBNotFoundError, KnowledgeBase from src.utils import logger diff --git a/src/knowledge/utils/__init__.py b/src/knowledge/utils/__init__.py new file mode 100644 index 00000000..69fac12a --- /dev/null +++ b/src/knowledge/utils/__init__.py @@ -0,0 +1,22 @@ +"""知识库工具模块 + +包含知识库相关的工具函数： +- kb_utils: 知识库通用工具函数 +- indexing: 文件处理和索引相关功能 +""" + +from .kb_utils import ( + get_embedding_config, + prepare_item_metadata, + split_text_into_chunks, + split_text_into_qa_chunks, + validate_file_path, +) + +__all__ = [ + "get_embedding_config", + "prepare_item_metadata", + "split_text_into_chunks", + "split_text_into_qa_chunks", + "validate_file_path", +] diff --git a/src/knowledge/kb_utils.py b/src/knowledge/utils/kb_utils.py similarity index 100% rename from src/knowledge/kb_utils.py rename to src/knowledge/utils/kb_utils.py