fix: 更新数据库路径和日志配置，确保兼容性和可维护性
This commit is contained in:
Wenjie Zhang
parent
a91d57f274
commit
7b846ff76b
@ -15,7 +15,7 @@ class DBManager:
|
||||
"""数据库管理器 - 只提供基础的数据库连接和会话管理"""
|
||||
|
||||
def __init__(self):
|
||||
self.db_path = os.path.join(config.save_dir, "data", "server.db")
|
||||
self.db_path = os.path.join(config.save_dir, "database", "server.db")
|
||||
self.ensure_db_dir()
|
||||
|
||||
# 创建SQLAlchemy引擎
|
||||
|
||||
@ -132,7 +132,7 @@ async def upload_file(
|
||||
if db_id:
|
||||
upload_dir = knowledge_base.get_db_upload_path(db_id)
|
||||
else:
|
||||
upload_dir = os.path.join(config.save_dir, "data", "uploads")
|
||||
upload_dir = os.path.join(config.save_dir, "database", "uploads")
|
||||
|
||||
basename, ext = os.path.splitext(file.filename)
|
||||
filename = f"{basename}_{hashstr(basename, 4, with_salt=True)}{ext}".lower()
|
||||
|
||||
@ -39,7 +39,7 @@ class Config(SimpleConfig):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self._config_items = {}
|
||||
self.save_dir = "saves"
|
||||
self.save_dir = os.getenv('SAVE_DIR', 'saves')
|
||||
self.filename = str(Path(f"{self.save_dir}/config/base.yaml"))
|
||||
os.makedirs(os.path.dirname(self.filename), exist_ok=True)
|
||||
|
||||
|
||||
@ -10,13 +10,16 @@ from datetime import datetime
|
||||
|
||||
from lightrag import LightRAG, QueryParam
|
||||
from lightrag.llm.openai import openai_complete_if_cache, openai_embed
|
||||
from lightrag.utils import EmbeddingFunc
|
||||
from lightrag.utils import EmbeddingFunc, setup_logger
|
||||
from lightrag.kg.shared_storage import initialize_pipeline_status
|
||||
|
||||
from src import config
|
||||
from src.utils import logger, hashstr, get_docker_safe_url
|
||||
from src.plugins import ocr
|
||||
|
||||
work_dir = os.path.join(config.save_dir, "lightrag_data")
|
||||
log_dir = os.path.join(work_dir, "logs", "lightrag")
|
||||
setup_logger("lightrag", log_file_path=os.path.join(log_dir, f"lightrag_{datetime.now().strftime('%Y-%m-%d')}.log"))
|
||||
|
||||
class LightRagBasedKB:
|
||||
"""基于 LightRAG 的知识库管理类"""
|
||||
@ -167,43 +170,35 @@ class LightRagBasedKB:
|
||||
# 直接读取文本文件
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
return f"Using f.read() to process {file_path.name}\n\n{content}"
|
||||
return f"# {file_path.name}\n\n{content}"
|
||||
|
||||
elif file_ext in ['.doc', '.docx']:
|
||||
# 处理 Word 文档
|
||||
try:
|
||||
from docx import Document
|
||||
doc = Document(file_path)
|
||||
text = '\n'.join([para.text for para in doc.paragraphs])
|
||||
return f"Using python-docx to process {file_path.name}\n\n{text}"
|
||||
except ImportError:
|
||||
logger.warning("python-docx not installed, cannot process .docx files")
|
||||
return f"# {file_path.name}\n\n[Cannot process .docx file - python-docx not installed]"
|
||||
from docx import Document
|
||||
doc = Document(file_path)
|
||||
text = '\n'.join([para.text for para in doc.paragraphs])
|
||||
return f"# {file_path.name}\n\n{text}"
|
||||
|
||||
elif file_ext in ['.jpg', '.jpeg', '.png', '.bmp']:
|
||||
# 使用 OCR 处理图片
|
||||
text = ocr.process_image(str(file_path))
|
||||
return f"Using OCR to process {file_path.name}\n\n{text}"
|
||||
return f"# {file_path.name}\n\n{text}"
|
||||
|
||||
else:
|
||||
# 尝试作为文本文件读取
|
||||
import textract
|
||||
text = textract.process(file_path)
|
||||
return f"Using textract to process {file_path.name}\n\n{text}"
|
||||
return f"# {file_path.name}\n\n{text}"
|
||||
|
||||
async def _process_url_to_markdown(self, url: str, params=None) -> str:
|
||||
"""将 URL 转换为 markdown 格式"""
|
||||
try:
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
response = requests.get(url, timeout=30)
|
||||
soup = BeautifulSoup(response.content, 'html.parser')
|
||||
text_content = soup.get_text()
|
||||
return f"# {url}\n\n{text_content}"
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to scrape URL {url}: {e}")
|
||||
return f"# {url}\n\n[Failed to scrape URL: {str(e)}]"
|
||||
response = requests.get(url, timeout=30)
|
||||
soup = BeautifulSoup(response.content, 'html.parser')
|
||||
text_content = soup.get_text()
|
||||
return f"# {url}\n\n{text_content}"
|
||||
|
||||
# =============================================================================
|
||||
# data_router.py 中使用的核心方法
|
||||
|
||||
@ -1,102 +0,0 @@
|
||||
"""
|
||||
数据库迁移工具：用于将独立的knowledge.db数据迁移到整合的server.db中
|
||||
"""
|
||||
|
||||
import os
|
||||
import sqlite3
|
||||
import shutil
|
||||
from datetime import datetime
|
||||
from src import config
|
||||
from src.utils import logger
|
||||
|
||||
def migrate_knowledge_db():
|
||||
"""
|
||||
将独立的knowledge.db数据迁移到整合的server.db中
|
||||
|
||||
Returns:
|
||||
bool: 迁移是否成功
|
||||
"""
|
||||
# 确定数据库路径
|
||||
knowledge_db_path = os.path.join(config.save_dir, "data", "knowledge.db")
|
||||
server_db_path = os.path.join(config.save_dir, "data", "server.db")
|
||||
|
||||
# 检查源数据库是否存在
|
||||
if not os.path.exists(knowledge_db_path):
|
||||
logger.info("未找到knowledge.db数据库，无需迁移")
|
||||
return False
|
||||
|
||||
# 确保目标数据库目录存在
|
||||
os.makedirs(os.path.dirname(server_db_path), exist_ok=True)
|
||||
|
||||
try:
|
||||
# 备份两个数据库
|
||||
timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
|
||||
if os.path.exists(server_db_path):
|
||||
shutil.copy2(server_db_path, f"{server_db_path}.bak.{timestamp}")
|
||||
logger.info(f"已备份server.db到 {server_db_path}.bak.{timestamp}")
|
||||
|
||||
shutil.copy2(knowledge_db_path, f"{knowledge_db_path}.bak.{timestamp}")
|
||||
logger.info(f"已备份knowledge.db到 {knowledge_db_path}.bak.{timestamp}")
|
||||
|
||||
# 连接到两个数据库
|
||||
knowledge_conn = sqlite3.connect(knowledge_db_path)
|
||||
server_conn = sqlite3.connect(server_db_path)
|
||||
|
||||
knowledge_cursor = knowledge_conn.cursor()
|
||||
server_cursor = server_conn.cursor()
|
||||
|
||||
# 获取knowledge.db中的表
|
||||
knowledge_cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
|
||||
tables = knowledge_cursor.fetchall()
|
||||
|
||||
# 迁移每个表
|
||||
for table in tables:
|
||||
table_name = table[0]
|
||||
if table_name.startswith('sqlite_'):
|
||||
continue # 跳过SQLite内部表
|
||||
|
||||
logger.info(f"正在迁移表 {table_name}...")
|
||||
|
||||
# 获取表结构
|
||||
knowledge_cursor.execute(f"PRAGMA table_info({table_name});")
|
||||
columns = knowledge_cursor.fetchall()
|
||||
|
||||
# 在server.db中创建表（如果不存在）
|
||||
column_defs = ", ".join([f'"{col[1]}" {col[2]}' for col in columns])
|
||||
create_table_sql = f'CREATE TABLE IF NOT EXISTS "{table_name}" ({column_defs});'
|
||||
server_cursor.execute(create_table_sql)
|
||||
|
||||
# 获取数据
|
||||
knowledge_cursor.execute(f'SELECT * FROM "{table_name}";')
|
||||
rows = knowledge_cursor.fetchall()
|
||||
|
||||
if rows:
|
||||
# 准备插入语句
|
||||
placeholders = ", ".join(["?" for _ in range(len(columns))])
|
||||
insert_sql = f'INSERT OR REPLACE INTO "{table_name}" VALUES ({placeholders});'
|
||||
|
||||
# 批量插入数据
|
||||
server_cursor.executemany(insert_sql, rows)
|
||||
logger.info(f"已迁移 {len(rows)} 条记录到表 {table_name}")
|
||||
|
||||
# 提交更改
|
||||
server_conn.commit()
|
||||
|
||||
# 关闭连接
|
||||
knowledge_cursor.close()
|
||||
knowledge_conn.close()
|
||||
server_cursor.close()
|
||||
server_conn.close()
|
||||
|
||||
logger.info("数据库迁移完成")
|
||||
logger.warning(f"请手动删除旧的knowledge.db文件: {knowledge_db_path}")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"数据库迁移失败: {str(e)}")
|
||||
return False
|
||||
|
||||
if __name__ == "__main__":
|
||||
# 当作为脚本运行时执行迁移
|
||||
result = migrate_knowledge_db()
|
||||
print(f"迁移{'成功' if result else '失败'}")
|
||||
@ -5,12 +5,13 @@ import pytz
|
||||
|
||||
from loguru import logger as loguru_logger
|
||||
|
||||
DATETIME = datetime.now(pytz.timezone('Asia/Shanghai')).strftime('%Y-%m-%d-%H%M%S')
|
||||
# DATETIME = "debug" # 为了方便，调试的时候输出到 debug.log 文件
|
||||
LOG_FILE = f'tmp/logs/project-{DATETIME}.log'
|
||||
SAVE_DIR = os.getenv('SAVE_DIR', 'saves')
|
||||
DATETIME = datetime.now(pytz.timezone('Asia/Shanghai')).strftime('%Y-%m-%d')
|
||||
LOG_FILE = f'{SAVE_DIR}/logs/yuxi-{DATETIME}.log'
|
||||
|
||||
def setup_logger(name, level="DEBUG", console=True):
|
||||
"""使用 loguru 设置日志记录器"""
|
||||
os.makedirs("tmp/logs", exist_ok=True)
|
||||
os.makedirs(f"{SAVE_DIR}/logs", exist_ok=True) # 创建日志目录
|
||||
|
||||
# 移除默认的 handler
|
||||
loguru_logger.remove()
|
||||
@ -31,7 +32,7 @@ def setup_logger(name, level="DEBUG", console=True):
|
||||
loguru_logger.add(
|
||||
lambda msg: print(msg, end=""),
|
||||
level=level,
|
||||
format="<green>{time:YYYY-MM-DD HH:mm:ss}</green> - <level>{level}</level> - <cyan>{name}</cyan> - <level>{message}</level>",
|
||||
format="<green>{time:MM-DD HH:mm:ss}</green> <level>{level}</level> <cyan>{name}</cyan>: <level>{message}</level>",
|
||||
colorize=True
|
||||
)
|
||||
|
||||
|
||||
@ -1181,6 +1181,9 @@ const toggleAutoRefresh = (checked) => {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
border-radius: 12px;
|
||||
word-break: break-all;
|
||||
word-break: break-word; /* 非标准，但某些浏览器支持 */
|
||||
overflow-wrap: break-word; /* 标准写法 */
|
||||
gap: 20px;
|
||||
|
||||
.results-overview {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user