最小化启动依赖
This commit is contained in:
parent
786a02eb6a
commit
adf130740b
@ -49,7 +49,7 @@ class Config(SimpleConfig):
|
|||||||
# 模型配置
|
# 模型配置
|
||||||
## 注意这里是模型名,而不是具体的模型路径,默认使用 HuggingFace 的路径
|
## 注意这里是模型名,而不是具体的模型路径,默认使用 HuggingFace 的路径
|
||||||
## 如果需要自定义路径,则在 config/base.yaml 中配置 model_local_paths
|
## 如果需要自定义路径,则在 config/base.yaml 中配置 model_local_paths
|
||||||
self.add_item("model_provider", default="qianfan", des="模型提供商", choices=["qianfan", "vllm", "zhipu", "deepseek", "dashscope"])
|
self.add_item("model_provider", default="zhipu", des="模型提供商", choices=["qianfan", "vllm", "zhipu", "deepseek", "dashscope"])
|
||||||
self.add_item("model_name", default=None, des="模型名称")
|
self.add_item("model_name", default=None, des="模型名称")
|
||||||
self.add_item("embed_model", default="bge-large-zh-v1.5", des="Embedding 模型", choices=["bge-large-zh-v1.5", "zhipu"])
|
self.add_item("embed_model", default="bge-large-zh-v1.5", des="Embedding 模型", choices=["bge-large-zh-v1.5", "zhipu"])
|
||||||
self.add_item("reranker", default="bge-reranker-v2-m3", des="Re-Ranker 模型", choices=["bge-reranker-v2-m3"])
|
self.add_item("reranker", default="bge-reranker-v2-m3", des="Re-Ranker 模型", choices=["bge-reranker-v2-m3"])
|
||||||
|
|||||||
@ -2,10 +2,7 @@ import os
|
|||||||
import json
|
import json
|
||||||
import time
|
import time
|
||||||
from src.utils import hashstr, setup_logger, is_text_pdf
|
from src.utils import hashstr, setup_logger, is_text_pdf
|
||||||
from src.plugins import pdf2txt
|
|
||||||
from src.core.knowledgebase import KnowledgeBase
|
|
||||||
from src.core.filereader import pdfreader, plainreader
|
from src.core.filereader import pdfreader, plainreader
|
||||||
from src.core.graphbase import GraphDatabase
|
|
||||||
from src.models.embedding import get_embedding_model
|
from src.models.embedding import get_embedding_model
|
||||||
|
|
||||||
logger = setup_logger("DataBaseManager")
|
logger = setup_logger("DataBaseManager")
|
||||||
@ -57,6 +54,8 @@ class DataBaseManager:
|
|||||||
self.embed_model = get_embedding_model(config)
|
self.embed_model = get_embedding_model(config)
|
||||||
|
|
||||||
if self.config.enable_knowledge_base:
|
if self.config.enable_knowledge_base:
|
||||||
|
from src.core.knowledgebase import KnowledgeBase
|
||||||
|
from src.core.graphbase import GraphDatabase
|
||||||
self.knowledge_base = KnowledgeBase(config, self.embed_model)
|
self.knowledge_base = KnowledgeBase(config, self.embed_model)
|
||||||
self.graph_base = GraphDatabase(self.config, self.embed_model)
|
self.graph_base = GraphDatabase(self.config, self.embed_model)
|
||||||
self.graph_base.start()
|
self.graph_base.start()
|
||||||
@ -180,6 +179,7 @@ class DataBaseManager:
|
|||||||
if is_text_pdf(file):
|
if is_text_pdf(file):
|
||||||
return pdfreader(file)
|
return pdfreader(file)
|
||||||
else:
|
else:
|
||||||
|
from src.plugins import pdf2txt
|
||||||
return pdf2txt(file, return_text=True)
|
return pdf2txt(file, return_text=True)
|
||||||
|
|
||||||
elif file.endswith(".txt") or file.endswith(".md"):
|
elif file.endswith(".txt") or file.endswith(".md"):
|
||||||
|
|||||||
@ -1,14 +1,13 @@
|
|||||||
import os
|
import os
|
||||||
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from llama_index.readers.file import PDFReader
|
|
||||||
|
|
||||||
|
|
||||||
def pdfreader(file_path):
|
def pdfreader(file_path):
|
||||||
"""读取PDF文件并返回text文本"""
|
"""读取PDF文件并返回text文本"""
|
||||||
assert os.path.exists(file_path), "File not found"
|
assert os.path.exists(file_path), "File not found"
|
||||||
assert file_path.endswith(".pdf"), "File format not supported"
|
assert file_path.endswith(".pdf"), "File format not supported"
|
||||||
|
|
||||||
|
from llama_index.readers.file import PDFReader
|
||||||
doc = PDFReader().load_data(file=Path(file_path))
|
doc = PDFReader().load_data(file=Path(file_path))
|
||||||
|
|
||||||
# 简单的拼接起来之后返回纯文本
|
# 简单的拼接起来之后返回纯文本
|
||||||
|
|||||||
@ -14,6 +14,7 @@ class KnowledgeBase:
|
|||||||
|
|
||||||
assert embed_model, "embed_model=None"
|
assert embed_model, "embed_model=None"
|
||||||
self.embed_model = embed_model
|
self.embed_model = embed_model
|
||||||
|
|
||||||
self.client = MilvusClient(self.milvus_path)
|
self.client = MilvusClient(self.milvus_path)
|
||||||
|
|
||||||
def _init_config(self, config):
|
def _init_config(self, config):
|
||||||
|
|||||||
@ -64,7 +64,6 @@ class VLLM(OpenAIBase):
|
|||||||
super().__init__(api_key=api_key, base_url=base_url, model_name=model_name)
|
super().__init__(api_key=api_key, base_url=base_url, model_name=model_name)
|
||||||
|
|
||||||
|
|
||||||
import qianfan
|
|
||||||
|
|
||||||
|
|
||||||
class GeneralResponse:
|
class GeneralResponse:
|
||||||
@ -76,6 +75,7 @@ class GeneralResponse:
|
|||||||
class Qianfan:
|
class Qianfan:
|
||||||
|
|
||||||
def __init__(self, model_name="ernie_speed") -> None:
|
def __init__(self, model_name="ernie_speed") -> None:
|
||||||
|
import qianfan
|
||||||
self.model_name = model_name
|
self.model_name = model_name
|
||||||
access_key = os.getenv("QIANFAN_ACCESS_KEY")
|
access_key = os.getenv("QIANFAN_ACCESS_KEY")
|
||||||
secret_key = os.getenv("QIANFAN_SECRET_KEY")
|
secret_key = os.getenv("QIANFAN_SECRET_KEY")
|
||||||
|
|||||||
@ -1,9 +1,6 @@
|
|||||||
FlagEmbedding==1.2.10
|
FlagEmbedding==1.2.10
|
||||||
Flask==3.0.3
|
Flask==3.0.3
|
||||||
Flask_Cors==4.0.1
|
Flask_Cors==4.0.1
|
||||||
llama_index==0.10.53
|
|
||||||
openai==1.35.10
|
openai==1.35.10
|
||||||
pymilvus==2.4.4
|
|
||||||
python-dotenv==1.0.1
|
python-dotenv==1.0.1
|
||||||
PyYAML==6.0.1
|
PyYAML==6.0.1
|
||||||
qianfan==0.4.0.1
|
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user