chore: 移除不再使用的脚本和优化日志信息
- 删除了 download_models.sh 和 migrate_kb_to_sqlite.py 脚本,清理不必要的代码。 - 在 base_router.py 中更新日志信息,调整为英文以提高可读性。 - 移除 retriever.restart() 调用,简化重启逻辑。 - 优化 OCRPlugin 中的注释,去除冗余代码。
This commit is contained in:
parent
57299fda8d
commit
db38035577
@ -1,5 +0,0 @@
|
|||||||
#! /bin/sh
|
|
||||||
source src/.env
|
|
||||||
|
|
||||||
# OCR 模型
|
|
||||||
huggingface-cli download SWHL/RapidOCR --local-dir ${MODEL_DIR}/SWHL/RapidOCR
|
|
||||||
@ -4,7 +4,7 @@ from pathlib import Path
|
|||||||
from fastapi import Request, Body, Depends, HTTPException
|
from fastapi import Request, Body, Depends, HTTPException
|
||||||
from fastapi import APIRouter
|
from fastapi import APIRouter
|
||||||
|
|
||||||
from src import config, retriever, knowledge_base, graph_base
|
from src import config, knowledge_base, graph_base
|
||||||
from server.utils.auth_middleware import get_admin_user, get_superadmin_user
|
from server.utils.auth_middleware import get_admin_user, get_superadmin_user
|
||||||
from server.models.user_model import User
|
from server.models.user_model import User
|
||||||
from src.utils.logging_config import logger
|
from src.utils.logging_config import logger
|
||||||
@ -20,18 +20,17 @@ def load_info_config():
|
|||||||
|
|
||||||
# 检查文件是否存在
|
# 检查文件是否存在
|
||||||
if not config_path.exists():
|
if not config_path.exists():
|
||||||
logger.warning(f"配置文件 {config_path} 不存在,使用默认配置")
|
logger.debug(f"The config file {config_path} does not exist, using default config")
|
||||||
return get_default_info_config()
|
return get_default_info_config()
|
||||||
|
|
||||||
# 读取配置文件
|
# 读取配置文件
|
||||||
with open(config_path, encoding='utf-8') as file:
|
with open(config_path, encoding='utf-8') as file:
|
||||||
config = yaml.safe_load(file)
|
config = yaml.safe_load(file)
|
||||||
|
|
||||||
# logger.info(f"成功加载信息配置文件: {config_path}")
|
|
||||||
return config
|
return config
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"加载信息配置文件失败: {e}")
|
logger.error(f"Failed to load info config: {e}")
|
||||||
return get_default_info_config()
|
return get_default_info_config()
|
||||||
|
|
||||||
def get_default_info_config():
|
def get_default_info_config():
|
||||||
|
|||||||
@ -1,84 +0,0 @@
|
|||||||
import os
|
|
||||||
import json
|
|
||||||
import time
|
|
||||||
from src import config
|
|
||||||
from src.utils import logger, hashstr
|
|
||||||
from server.db_manager import db_manager
|
|
||||||
|
|
||||||
def migrate_json_to_sqlite():
|
|
||||||
"""
|
|
||||||
将旧的JSON格式知识库数据迁移到SQLite数据库
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
bool: 是否成功迁移,如果没有JSON文件或已经迁移过则返回False
|
|
||||||
"""
|
|
||||||
json_path = os.path.join(config.save_dir, "data", "database.json")
|
|
||||||
|
|
||||||
# 检查JSON文件是否存在
|
|
||||||
if not os.path.exists(json_path):
|
|
||||||
logger.info("没有找到旧的JSON数据文件,无需迁移")
|
|
||||||
return False
|
|
||||||
|
|
||||||
try:
|
|
||||||
# 读取旧的JSON文件
|
|
||||||
with open(json_path, encoding='utf-8') as f:
|
|
||||||
old_data = json.load(f)
|
|
||||||
|
|
||||||
if not old_data or not isinstance(old_data, dict) or not old_data.get("databases"):
|
|
||||||
logger.warning("JSON文件格式不正确或为空")
|
|
||||||
return False
|
|
||||||
|
|
||||||
# 获取知识库列表
|
|
||||||
databases = old_data.get("databases", {})
|
|
||||||
|
|
||||||
# 迁移每个知识库
|
|
||||||
for db_id, db_info in databases.items():
|
|
||||||
logger.info(f"正在迁移知识库: {db_info.get('name')} (ID: {db_id})")
|
|
||||||
|
|
||||||
# 创建知识库
|
|
||||||
db_manager.create_database(
|
|
||||||
db_id=db_id,
|
|
||||||
name=db_info.get('name', '未命名知识库'),
|
|
||||||
description=db_info.get('description', ''),
|
|
||||||
embed_model=db_info.get('embed_model'),
|
|
||||||
dimension=db_info.get('dimension')
|
|
||||||
)
|
|
||||||
|
|
||||||
# 迁移文件
|
|
||||||
files = db_info.get('files', {})
|
|
||||||
for file_id, file_info in files.items():
|
|
||||||
logger.info(f" 正在迁移文件: {file_info.get('filename')} (ID: {file_id})")
|
|
||||||
|
|
||||||
# 添加文件
|
|
||||||
db_manager.add_file(
|
|
||||||
db_id=db_id,
|
|
||||||
file_id=file_id,
|
|
||||||
filename=file_info.get('filename', '未命名文件'),
|
|
||||||
path=file_info.get('path', ''),
|
|
||||||
file_type=file_info.get('type', 'unknown'),
|
|
||||||
status=file_info.get('status', 'done')
|
|
||||||
)
|
|
||||||
|
|
||||||
# 迁移节点
|
|
||||||
nodes = file_info.get('nodes', [])
|
|
||||||
for node in nodes:
|
|
||||||
db_manager.add_node(
|
|
||||||
file_id=file_id,
|
|
||||||
text=node.get('text', ''),
|
|
||||||
hash_value=node.get('hash'),
|
|
||||||
start_char_idx=node.get('start_char_idx'),
|
|
||||||
end_char_idx=node.get('end_char_idx'),
|
|
||||||
metadata=node.get('metadata', {})
|
|
||||||
)
|
|
||||||
|
|
||||||
# 备份旧的JSON文件
|
|
||||||
backup_path = f"{json_path}.bak.{int(time.time())}"
|
|
||||||
os.rename(json_path, backup_path)
|
|
||||||
logger.info(f"迁移完成,旧的JSON文件已备份到: {backup_path}")
|
|
||||||
logger.warning(f"请手动删除旧的JSON文件: {json_path}")
|
|
||||||
|
|
||||||
return True
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"迁移过程中出错: {e}")
|
|
||||||
return False
|
|
||||||
@ -1,48 +0,0 @@
|
|||||||
"""
|
|
||||||
这里面存放是 RAG 相关的一些组件
|
|
||||||
"""
|
|
||||||
|
|
||||||
from src.utils import prompts
|
|
||||||
|
|
||||||
class BaseOperator:
|
|
||||||
"""
|
|
||||||
基类
|
|
||||||
"""
|
|
||||||
template = None
|
|
||||||
|
|
||||||
def __init__(self):
|
|
||||||
pass
|
|
||||||
|
|
||||||
def call(self, **kwargs):
|
|
||||||
pass
|
|
||||||
|
|
||||||
def __call__(self, **kwargs):
|
|
||||||
"""
|
|
||||||
所有 RAG 相关组件的调用接口
|
|
||||||
"""
|
|
||||||
return self.call(**kwargs)
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
class HyDEOperator(BaseOperator):
|
|
||||||
"""
|
|
||||||
HyDE 重写查询
|
|
||||||
"""
|
|
||||||
template = prompts.HYDE_PROMPT_TEMPLATE
|
|
||||||
|
|
||||||
def __init__(self):
|
|
||||||
super().__init__()
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def call(cls, model_callable, query, context_str, **kwargs):
|
|
||||||
"""
|
|
||||||
重写查询
|
|
||||||
|
|
||||||
Args:
|
|
||||||
model_callable: 模型调用函数
|
|
||||||
query: 查询
|
|
||||||
context_str: 上下文
|
|
||||||
"""
|
|
||||||
prompt = cls.template.format(query=query, context_str=context_str)
|
|
||||||
response = model_callable(prompt)
|
|
||||||
return response
|
|
||||||
@ -123,12 +123,6 @@ class OCRPlugin:
|
|||||||
raise FileNotFoundError(f"PDF file not found: {pdf_path}")
|
raise FileNotFoundError(f"PDF file not found: {pdf_path}")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# 检查是否为文本PDF,可能会出现错误,比如每一页都有可读取的水印文字,但是内容本身是扫描件,需要使用OCR处理
|
|
||||||
# if is_text_pdf(pdf_path):
|
|
||||||
# from src.core.indexing import pdfreader
|
|
||||||
# logger.info("PDF file is text, use llama_index.readers.file to read")
|
|
||||||
# return pdfreader(pdf_path)
|
|
||||||
|
|
||||||
images = []
|
images = []
|
||||||
|
|
||||||
pdfDoc = fitz.open(pdf_path)
|
pdfDoc = fitz.open(pdf_path)
|
||||||
|
|||||||
@ -137,7 +137,7 @@ onMounted(() => {
|
|||||||
width: 100%;
|
width: 100%;
|
||||||
height: auto;
|
height: auto;
|
||||||
margin: 0 auto;
|
margin: 0 auto;
|
||||||
padding: 0.4rem 0.75rem;
|
padding: 0.6rem 0.75rem 0.4rem 0.75rem;
|
||||||
border: 2px solid var(--gray-200);
|
border: 2px solid var(--gray-200);
|
||||||
border-radius: 0.8rem;
|
border-radius: 0.8rem;
|
||||||
box-shadow: 0 2px 8px rgba(0, 0, 0, 0.05);
|
box-shadow: 0 2px 8px rgba(0, 0, 0, 0.05);
|
||||||
|
|||||||
@ -58,7 +58,7 @@
|
|||||||
<!-- 左侧智能体列表侧边栏 -->
|
<!-- 左侧智能体列表侧边栏 -->
|
||||||
<div class="sidebar" :class="{ 'is-open': state.agentSiderbarConfigOpen }">
|
<div class="sidebar" :class="{ 'is-open': state.agentSiderbarConfigOpen }">
|
||||||
<div class="agent-info">
|
<div class="agent-info">
|
||||||
<h3 @click="toggleDebugMode">描述</h3>
|
<h3>详细配置信<span @click="toggleDebugMode">息</span></h3>
|
||||||
<p>{{ selectedAgent.description }}</p>
|
<p>{{ selectedAgent.description }}</p>
|
||||||
<pre v-if="state.debug_mode">{{ selectedAgent }}</pre>
|
<pre v-if="state.debug_mode">{{ selectedAgent }}</pre>
|
||||||
|
|
||||||
@ -115,7 +115,7 @@
|
|||||||
>
|
>
|
||||||
<a-select-option v-for="option in value.options" :key="option" :value="option"></a-select-option>
|
<a-select-option v-for="option in value.options" :key="option" :value="option"></a-select-option>
|
||||||
</a-select>
|
</a-select>
|
||||||
<!-- 多选标签卡片 -->
|
<!-- 多选标签卡片 -->
|
||||||
<div v-else-if="value?.options && value?.type === 'list'" class="multi-select-cards">
|
<div v-else-if="value?.options && value?.type === 'list'" class="multi-select-cards">
|
||||||
<div class="multi-select-label">
|
<div class="multi-select-label">
|
||||||
<span>已选择 {{ getSelectedCount(key) }} 项</span>
|
<span>已选择 {{ getSelectedCount(key) }} 项</span>
|
||||||
@ -210,7 +210,7 @@ const agents = ref({});
|
|||||||
const selectedAgentId = ref(null);
|
const selectedAgentId = ref(null);
|
||||||
const defaultAgentId = ref(null); // 存储默认智能体ID
|
const defaultAgentId = ref(null); // 存储默认智能体ID
|
||||||
const state = reactive({
|
const state = reactive({
|
||||||
agentSiderbarConfigOpen: true,
|
agentSiderbarConfigOpen: false,
|
||||||
debug_mode: false,
|
debug_mode: false,
|
||||||
isEmptyConfig: computed(() =>
|
isEmptyConfig: computed(() =>
|
||||||
!selectedAgentId.value ||
|
!selectedAgentId.value ||
|
||||||
|
|||||||
@ -120,7 +120,7 @@ const state = reactive({
|
|||||||
showModal: false,
|
showModal: false,
|
||||||
precessing: false,
|
precessing: false,
|
||||||
indexing: false,
|
indexing: false,
|
||||||
showPage: computed(() => configStore.config.enable_knowledge_base && configStore.config.enable_knowledge_graph),
|
showPage: true,
|
||||||
})
|
})
|
||||||
|
|
||||||
// 计算未索引节点数量
|
// 计算未索引节点数量
|
||||||
@ -215,9 +215,6 @@ const loadSampleNodes = () => {
|
|||||||
.catch((error) => {
|
.catch((error) => {
|
||||||
console.error(error)
|
console.error(error)
|
||||||
message.error(error.message || '加载节点失败');
|
message.error(error.message || '加载节点失败');
|
||||||
if (configStore?.config && !configStore?.config.enable_knowledge_graph) {
|
|
||||||
message.error('请前往设置页面配置启用知识图谱')
|
|
||||||
}
|
|
||||||
})
|
})
|
||||||
.finally(() => state.fetching = false)
|
.finally(() => state.fetching = false)
|
||||||
}
|
}
|
||||||
|
|||||||
@ -95,14 +95,6 @@
|
|||||||
<p>请在 <code>src/.env</code> 文件中配置对应的 APIKEY,并重新启动服务</p>
|
<p>请在 <code>src/.env</code> 文件中配置对应的 APIKEY,并重新启动服务</p>
|
||||||
<ModelProvidersComponent />
|
<ModelProvidersComponent />
|
||||||
</div>
|
</div>
|
||||||
<div class="setting" v-if="(state.windowWidth <= 520 || state.section ==='path') && userStore.isSuperAdmin">
|
|
||||||
<h3>本地模型配置(将在 v0.2 版本移除对本地模型的支持)</h3>
|
|
||||||
<p>如果是 Docker 启动,务必确保在 docker-compose.dev.yaml 中添加了 volumes 映射。</p>
|
|
||||||
<TableConfigComponent
|
|
||||||
:config="configStore.config?.model_local_paths"
|
|
||||||
@update:config="handleModelLocalPathsUpdate"
|
|
||||||
/>
|
|
||||||
</div>
|
|
||||||
<!-- TODO 用户管理优化,添加姓名(默认使用用户名配置项) -->
|
<!-- TODO 用户管理优化,添加姓名(默认使用用户名配置项) -->
|
||||||
<div class="setting" v-if="state.section === 'user' && userStore.isAdmin">
|
<div class="setting" v-if="state.section === 'user' && userStore.isAdmin">
|
||||||
<UserManagementComponent />
|
<UserManagementComponent />
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user