最小化启动依赖

This commit is contained in:
Wenjie Zhang 2024-08-07 17:24:46 +08:00
parent 786a02eb6a
commit adf130740b
6 changed files with 7 additions and 10 deletions

View File

@ -49,7 +49,7 @@ class Config(SimpleConfig):
# 模型配置
## 注意这里是模型名,而不是具体的模型路径,默认使用 HuggingFace 的路径
## 如果需要自定义路径,则在 config/base.yaml 中配置 model_local_paths
self.add_item("model_provider", default="qianfan", des="模型提供商", choices=["qianfan", "vllm", "zhipu", "deepseek", "dashscope"])
self.add_item("model_provider", default="zhipu", des="模型提供商", choices=["qianfan", "vllm", "zhipu", "deepseek", "dashscope"])
self.add_item("model_name", default=None, des="模型名称")
self.add_item("embed_model", default="bge-large-zh-v1.5", des="Embedding 模型", choices=["bge-large-zh-v1.5", "zhipu"])
self.add_item("reranker", default="bge-reranker-v2-m3", des="Re-Ranker 模型", choices=["bge-reranker-v2-m3"])

View File

@ -2,10 +2,7 @@ import os
import json
import time
from src.utils import hashstr, setup_logger, is_text_pdf
from src.plugins import pdf2txt
from src.core.knowledgebase import KnowledgeBase
from src.core.filereader import pdfreader, plainreader
from src.core.graphbase import GraphDatabase
from src.models.embedding import get_embedding_model
logger = setup_logger("DataBaseManager")
@ -57,6 +54,8 @@ class DataBaseManager:
self.embed_model = get_embedding_model(config)
if self.config.enable_knowledge_base:
from src.core.knowledgebase import KnowledgeBase
from src.core.graphbase import GraphDatabase
self.knowledge_base = KnowledgeBase(config, self.embed_model)
self.graph_base = GraphDatabase(self.config, self.embed_model)
self.graph_base.start()
@ -180,6 +179,7 @@ class DataBaseManager:
if is_text_pdf(file):
return pdfreader(file)
else:
from src.plugins import pdf2txt
return pdf2txt(file, return_text=True)
elif file.endswith(".txt") or file.endswith(".md"):

View File

@ -1,14 +1,13 @@
import os
from pathlib import Path
from llama_index.readers.file import PDFReader
def pdfreader(file_path):
"""读取PDF文件并返回text文本"""
assert os.path.exists(file_path), "File not found"
assert file_path.endswith(".pdf"), "File format not supported"
from llama_index.readers.file import PDFReader
doc = PDFReader().load_data(file=Path(file_path))
# 简单的拼接起来之后返回纯文本

View File

@ -14,6 +14,7 @@ class KnowledgeBase:
assert embed_model, "embed_model=None"
self.embed_model = embed_model
self.client = MilvusClient(self.milvus_path)
def _init_config(self, config):

View File

@ -64,7 +64,6 @@ class VLLM(OpenAIBase):
super().__init__(api_key=api_key, base_url=base_url, model_name=model_name)
import qianfan
class GeneralResponse:
@ -76,6 +75,7 @@ class GeneralResponse:
class Qianfan:
def __init__(self, model_name="ernie_speed") -> None:
import qianfan
self.model_name = model_name
access_key = os.getenv("QIANFAN_ACCESS_KEY")
secret_key = os.getenv("QIANFAN_SECRET_KEY")

View File

@ -1,9 +1,6 @@
FlagEmbedding==1.2.10
Flask==3.0.3
Flask_Cors==4.0.1
llama_index==0.10.53
openai==1.35.10
pymilvus==2.4.4
python-dotenv==1.0.1
PyYAML==6.0.1
qianfan==0.4.0.1