fix: 更新数据库路径和日志配置,确保兼容性和可维护性
This commit is contained in:
parent
a91d57f274
commit
7b846ff76b
@ -15,7 +15,7 @@ class DBManager:
|
|||||||
"""数据库管理器 - 只提供基础的数据库连接和会话管理"""
|
"""数据库管理器 - 只提供基础的数据库连接和会话管理"""
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.db_path = os.path.join(config.save_dir, "data", "server.db")
|
self.db_path = os.path.join(config.save_dir, "database", "server.db")
|
||||||
self.ensure_db_dir()
|
self.ensure_db_dir()
|
||||||
|
|
||||||
# 创建SQLAlchemy引擎
|
# 创建SQLAlchemy引擎
|
||||||
|
|||||||
@ -132,7 +132,7 @@ async def upload_file(
|
|||||||
if db_id:
|
if db_id:
|
||||||
upload_dir = knowledge_base.get_db_upload_path(db_id)
|
upload_dir = knowledge_base.get_db_upload_path(db_id)
|
||||||
else:
|
else:
|
||||||
upload_dir = os.path.join(config.save_dir, "data", "uploads")
|
upload_dir = os.path.join(config.save_dir, "database", "uploads")
|
||||||
|
|
||||||
basename, ext = os.path.splitext(file.filename)
|
basename, ext = os.path.splitext(file.filename)
|
||||||
filename = f"{basename}_{hashstr(basename, 4, with_salt=True)}{ext}".lower()
|
filename = f"{basename}_{hashstr(basename, 4, with_salt=True)}{ext}".lower()
|
||||||
|
|||||||
@ -39,7 +39,7 @@ class Config(SimpleConfig):
|
|||||||
def __init__(self):
|
def __init__(self):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self._config_items = {}
|
self._config_items = {}
|
||||||
self.save_dir = "saves"
|
self.save_dir = os.getenv('SAVE_DIR', 'saves')
|
||||||
self.filename = str(Path(f"{self.save_dir}/config/base.yaml"))
|
self.filename = str(Path(f"{self.save_dir}/config/base.yaml"))
|
||||||
os.makedirs(os.path.dirname(self.filename), exist_ok=True)
|
os.makedirs(os.path.dirname(self.filename), exist_ok=True)
|
||||||
|
|
||||||
|
|||||||
@ -10,13 +10,16 @@ from datetime import datetime
|
|||||||
|
|
||||||
from lightrag import LightRAG, QueryParam
|
from lightrag import LightRAG, QueryParam
|
||||||
from lightrag.llm.openai import openai_complete_if_cache, openai_embed
|
from lightrag.llm.openai import openai_complete_if_cache, openai_embed
|
||||||
from lightrag.utils import EmbeddingFunc
|
from lightrag.utils import EmbeddingFunc, setup_logger
|
||||||
from lightrag.kg.shared_storage import initialize_pipeline_status
|
from lightrag.kg.shared_storage import initialize_pipeline_status
|
||||||
|
|
||||||
from src import config
|
from src import config
|
||||||
from src.utils import logger, hashstr, get_docker_safe_url
|
from src.utils import logger, hashstr, get_docker_safe_url
|
||||||
from src.plugins import ocr
|
from src.plugins import ocr
|
||||||
|
|
||||||
|
work_dir = os.path.join(config.save_dir, "lightrag_data")
|
||||||
|
log_dir = os.path.join(work_dir, "logs", "lightrag")
|
||||||
|
setup_logger("lightrag", log_file_path=os.path.join(log_dir, f"lightrag_{datetime.now().strftime('%Y-%m-%d')}.log"))
|
||||||
|
|
||||||
class LightRagBasedKB:
|
class LightRagBasedKB:
|
||||||
"""基于 LightRAG 的知识库管理类"""
|
"""基于 LightRAG 的知识库管理类"""
|
||||||
@ -167,43 +170,35 @@ class LightRagBasedKB:
|
|||||||
# 直接读取文本文件
|
# 直接读取文本文件
|
||||||
with open(file_path, 'r', encoding='utf-8') as f:
|
with open(file_path, 'r', encoding='utf-8') as f:
|
||||||
content = f.read()
|
content = f.read()
|
||||||
return f"Using f.read() to process {file_path.name}\n\n{content}"
|
return f"# {file_path.name}\n\n{content}"
|
||||||
|
|
||||||
elif file_ext in ['.doc', '.docx']:
|
elif file_ext in ['.doc', '.docx']:
|
||||||
# 处理 Word 文档
|
# 处理 Word 文档
|
||||||
try:
|
from docx import Document
|
||||||
from docx import Document
|
doc = Document(file_path)
|
||||||
doc = Document(file_path)
|
text = '\n'.join([para.text for para in doc.paragraphs])
|
||||||
text = '\n'.join([para.text for para in doc.paragraphs])
|
return f"# {file_path.name}\n\n{text}"
|
||||||
return f"Using python-docx to process {file_path.name}\n\n{text}"
|
|
||||||
except ImportError:
|
|
||||||
logger.warning("python-docx not installed, cannot process .docx files")
|
|
||||||
return f"# {file_path.name}\n\n[Cannot process .docx file - python-docx not installed]"
|
|
||||||
|
|
||||||
elif file_ext in ['.jpg', '.jpeg', '.png', '.bmp']:
|
elif file_ext in ['.jpg', '.jpeg', '.png', '.bmp']:
|
||||||
# 使用 OCR 处理图片
|
# 使用 OCR 处理图片
|
||||||
text = ocr.process_image(str(file_path))
|
text = ocr.process_image(str(file_path))
|
||||||
return f"Using OCR to process {file_path.name}\n\n{text}"
|
return f"# {file_path.name}\n\n{text}"
|
||||||
|
|
||||||
else:
|
else:
|
||||||
# 尝试作为文本文件读取
|
# 尝试作为文本文件读取
|
||||||
import textract
|
import textract
|
||||||
text = textract.process(file_path)
|
text = textract.process(file_path)
|
||||||
return f"Using textract to process {file_path.name}\n\n{text}"
|
return f"# {file_path.name}\n\n{text}"
|
||||||
|
|
||||||
async def _process_url_to_markdown(self, url: str, params=None) -> str:
|
async def _process_url_to_markdown(self, url: str, params=None) -> str:
|
||||||
"""将 URL 转换为 markdown 格式"""
|
"""将 URL 转换为 markdown 格式"""
|
||||||
try:
|
import requests
|
||||||
import requests
|
from bs4 import BeautifulSoup
|
||||||
from bs4 import BeautifulSoup
|
|
||||||
|
|
||||||
response = requests.get(url, timeout=30)
|
response = requests.get(url, timeout=30)
|
||||||
soup = BeautifulSoup(response.content, 'html.parser')
|
soup = BeautifulSoup(response.content, 'html.parser')
|
||||||
text_content = soup.get_text()
|
text_content = soup.get_text()
|
||||||
return f"# {url}\n\n{text_content}"
|
return f"# {url}\n\n{text_content}"
|
||||||
except Exception as e:
|
|
||||||
logger.warning(f"Failed to scrape URL {url}: {e}")
|
|
||||||
return f"# {url}\n\n[Failed to scrape URL: {str(e)}]"
|
|
||||||
|
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
# data_router.py 中使用的核心方法
|
# data_router.py 中使用的核心方法
|
||||||
|
|||||||
@ -1,102 +0,0 @@
|
|||||||
"""
|
|
||||||
数据库迁移工具:用于将独立的knowledge.db数据迁移到整合的server.db中
|
|
||||||
"""
|
|
||||||
|
|
||||||
import os
|
|
||||||
import sqlite3
|
|
||||||
import shutil
|
|
||||||
from datetime import datetime
|
|
||||||
from src import config
|
|
||||||
from src.utils import logger
|
|
||||||
|
|
||||||
def migrate_knowledge_db():
|
|
||||||
"""
|
|
||||||
将独立的knowledge.db数据迁移到整合的server.db中
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
bool: 迁移是否成功
|
|
||||||
"""
|
|
||||||
# 确定数据库路径
|
|
||||||
knowledge_db_path = os.path.join(config.save_dir, "data", "knowledge.db")
|
|
||||||
server_db_path = os.path.join(config.save_dir, "data", "server.db")
|
|
||||||
|
|
||||||
# 检查源数据库是否存在
|
|
||||||
if not os.path.exists(knowledge_db_path):
|
|
||||||
logger.info("未找到knowledge.db数据库,无需迁移")
|
|
||||||
return False
|
|
||||||
|
|
||||||
# 确保目标数据库目录存在
|
|
||||||
os.makedirs(os.path.dirname(server_db_path), exist_ok=True)
|
|
||||||
|
|
||||||
try:
|
|
||||||
# 备份两个数据库
|
|
||||||
timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
|
|
||||||
if os.path.exists(server_db_path):
|
|
||||||
shutil.copy2(server_db_path, f"{server_db_path}.bak.{timestamp}")
|
|
||||||
logger.info(f"已备份server.db到 {server_db_path}.bak.{timestamp}")
|
|
||||||
|
|
||||||
shutil.copy2(knowledge_db_path, f"{knowledge_db_path}.bak.{timestamp}")
|
|
||||||
logger.info(f"已备份knowledge.db到 {knowledge_db_path}.bak.{timestamp}")
|
|
||||||
|
|
||||||
# 连接到两个数据库
|
|
||||||
knowledge_conn = sqlite3.connect(knowledge_db_path)
|
|
||||||
server_conn = sqlite3.connect(server_db_path)
|
|
||||||
|
|
||||||
knowledge_cursor = knowledge_conn.cursor()
|
|
||||||
server_cursor = server_conn.cursor()
|
|
||||||
|
|
||||||
# 获取knowledge.db中的表
|
|
||||||
knowledge_cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
|
|
||||||
tables = knowledge_cursor.fetchall()
|
|
||||||
|
|
||||||
# 迁移每个表
|
|
||||||
for table in tables:
|
|
||||||
table_name = table[0]
|
|
||||||
if table_name.startswith('sqlite_'):
|
|
||||||
continue # 跳过SQLite内部表
|
|
||||||
|
|
||||||
logger.info(f"正在迁移表 {table_name}...")
|
|
||||||
|
|
||||||
# 获取表结构
|
|
||||||
knowledge_cursor.execute(f"PRAGMA table_info({table_name});")
|
|
||||||
columns = knowledge_cursor.fetchall()
|
|
||||||
|
|
||||||
# 在server.db中创建表(如果不存在)
|
|
||||||
column_defs = ", ".join([f'"{col[1]}" {col[2]}' for col in columns])
|
|
||||||
create_table_sql = f'CREATE TABLE IF NOT EXISTS "{table_name}" ({column_defs});'
|
|
||||||
server_cursor.execute(create_table_sql)
|
|
||||||
|
|
||||||
# 获取数据
|
|
||||||
knowledge_cursor.execute(f'SELECT * FROM "{table_name}";')
|
|
||||||
rows = knowledge_cursor.fetchall()
|
|
||||||
|
|
||||||
if rows:
|
|
||||||
# 准备插入语句
|
|
||||||
placeholders = ", ".join(["?" for _ in range(len(columns))])
|
|
||||||
insert_sql = f'INSERT OR REPLACE INTO "{table_name}" VALUES ({placeholders});'
|
|
||||||
|
|
||||||
# 批量插入数据
|
|
||||||
server_cursor.executemany(insert_sql, rows)
|
|
||||||
logger.info(f"已迁移 {len(rows)} 条记录到表 {table_name}")
|
|
||||||
|
|
||||||
# 提交更改
|
|
||||||
server_conn.commit()
|
|
||||||
|
|
||||||
# 关闭连接
|
|
||||||
knowledge_cursor.close()
|
|
||||||
knowledge_conn.close()
|
|
||||||
server_cursor.close()
|
|
||||||
server_conn.close()
|
|
||||||
|
|
||||||
logger.info("数据库迁移完成")
|
|
||||||
logger.warning(f"请手动删除旧的knowledge.db文件: {knowledge_db_path}")
|
|
||||||
return True
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"数据库迁移失败: {str(e)}")
|
|
||||||
return False
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
# 当作为脚本运行时执行迁移
|
|
||||||
result = migrate_knowledge_db()
|
|
||||||
print(f"迁移{'成功' if result else '失败'}")
|
|
||||||
@ -5,12 +5,13 @@ import pytz
|
|||||||
|
|
||||||
from loguru import logger as loguru_logger
|
from loguru import logger as loguru_logger
|
||||||
|
|
||||||
DATETIME = datetime.now(pytz.timezone('Asia/Shanghai')).strftime('%Y-%m-%d-%H%M%S')
|
SAVE_DIR = os.getenv('SAVE_DIR', 'saves')
|
||||||
# DATETIME = "debug" # 为了方便,调试的时候输出到 debug.log 文件
|
DATETIME = datetime.now(pytz.timezone('Asia/Shanghai')).strftime('%Y-%m-%d')
|
||||||
LOG_FILE = f'tmp/logs/project-{DATETIME}.log'
|
LOG_FILE = f'{SAVE_DIR}/logs/yuxi-{DATETIME}.log'
|
||||||
|
|
||||||
def setup_logger(name, level="DEBUG", console=True):
|
def setup_logger(name, level="DEBUG", console=True):
|
||||||
"""使用 loguru 设置日志记录器"""
|
"""使用 loguru 设置日志记录器"""
|
||||||
os.makedirs("tmp/logs", exist_ok=True)
|
os.makedirs(f"{SAVE_DIR}/logs", exist_ok=True) # 创建日志目录
|
||||||
|
|
||||||
# 移除默认的 handler
|
# 移除默认的 handler
|
||||||
loguru_logger.remove()
|
loguru_logger.remove()
|
||||||
@ -31,7 +32,7 @@ def setup_logger(name, level="DEBUG", console=True):
|
|||||||
loguru_logger.add(
|
loguru_logger.add(
|
||||||
lambda msg: print(msg, end=""),
|
lambda msg: print(msg, end=""),
|
||||||
level=level,
|
level=level,
|
||||||
format="<green>{time:YYYY-MM-DD HH:mm:ss}</green> - <level>{level}</level> - <cyan>{name}</cyan> - <level>{message}</level>",
|
format="<green>{time:MM-DD HH:mm:ss}</green> <level>{level}</level> <cyan>{name}</cyan>: <level>{message}</level>",
|
||||||
colorize=True
|
colorize=True
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@ -1181,6 +1181,9 @@ const toggleAutoRefresh = (checked) => {
|
|||||||
display: flex;
|
display: flex;
|
||||||
flex-direction: column;
|
flex-direction: column;
|
||||||
border-radius: 12px;
|
border-radius: 12px;
|
||||||
|
word-break: break-all;
|
||||||
|
word-break: break-word; /* 非标准,但某些浏览器支持 */
|
||||||
|
overflow-wrap: break-word; /* 标准写法 */
|
||||||
gap: 20px;
|
gap: 20px;
|
||||||
|
|
||||||
.results-overview {
|
.results-overview {
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user