From 7b846ff76bf6cf34538fd9e204468a5a004731da Mon Sep 17 00:00:00 2001 From: Wenjie Zhang Date: Sat, 28 Jun 2025 20:19:26 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E6=9B=B4=E6=96=B0=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=BA=93=E8=B7=AF=E5=BE=84=E5=92=8C=E6=97=A5=E5=BF=97=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=EF=BC=8C=E7=A1=AE=E4=BF=9D=E5=85=BC=E5=AE=B9=E6=80=A7?= =?UTF-8?q?=E5=92=8C=E5=8F=AF=E7=BB=B4=E6=8A=A4=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/db_manager.py | 2 +- server/routers/data_router.py | 2 +- src/config/__init__.py | 2 +- src/core/lightrag_based_kb.py | 39 +++++------ src/utils/db_migration.py | 102 ----------------------------- src/utils/logging_config.py | 11 ++-- web/src/views/DataBaseInfoView.vue | 3 + 7 files changed, 29 insertions(+), 132 deletions(-) delete mode 100644 src/utils/db_migration.py diff --git a/server/db_manager.py b/server/db_manager.py index 2e4752fd..24308c71 100644 --- a/server/db_manager.py +++ b/server/db_manager.py @@ -15,7 +15,7 @@ class DBManager: """数据库管理器 - 只提供基础的数据库连接和会话管理""" def __init__(self): - self.db_path = os.path.join(config.save_dir, "data", "server.db") + self.db_path = os.path.join(config.save_dir, "database", "server.db") self.ensure_db_dir() # 创建SQLAlchemy引擎 diff --git a/server/routers/data_router.py b/server/routers/data_router.py index 2c397eb9..c36cae06 100644 --- a/server/routers/data_router.py +++ b/server/routers/data_router.py @@ -132,7 +132,7 @@ async def upload_file( if db_id: upload_dir = knowledge_base.get_db_upload_path(db_id) else: - upload_dir = os.path.join(config.save_dir, "data", "uploads") + upload_dir = os.path.join(config.save_dir, "database", "uploads") basename, ext = os.path.splitext(file.filename) filename = f"{basename}_{hashstr(basename, 4, with_salt=True)}{ext}".lower() diff --git a/src/config/__init__.py b/src/config/__init__.py index c9efd1f6..b21f0732 100644 --- a/src/config/__init__.py +++ b/src/config/__init__.py @@ -39,7 +39,7 @@ class Config(SimpleConfig): def __init__(self): super().__init__() self._config_items = {} - self.save_dir = "saves" + self.save_dir = os.getenv('SAVE_DIR', 'saves') self.filename = str(Path(f"{self.save_dir}/config/base.yaml")) os.makedirs(os.path.dirname(self.filename), exist_ok=True) diff --git a/src/core/lightrag_based_kb.py b/src/core/lightrag_based_kb.py index ec274351..bf6750da 100644 --- a/src/core/lightrag_based_kb.py +++ b/src/core/lightrag_based_kb.py @@ -10,13 +10,16 @@ from datetime import datetime from lightrag import LightRAG, QueryParam from lightrag.llm.openai import openai_complete_if_cache, openai_embed -from lightrag.utils import EmbeddingFunc +from lightrag.utils import EmbeddingFunc, setup_logger from lightrag.kg.shared_storage import initialize_pipeline_status from src import config from src.utils import logger, hashstr, get_docker_safe_url from src.plugins import ocr +work_dir = os.path.join(config.save_dir, "lightrag_data") +log_dir = os.path.join(work_dir, "logs", "lightrag") +setup_logger("lightrag", log_file_path=os.path.join(log_dir, f"lightrag_{datetime.now().strftime('%Y-%m-%d')}.log")) class LightRagBasedKB: """基于 LightRAG 的知识库管理类""" @@ -167,43 +170,35 @@ class LightRagBasedKB: # 直接读取文本文件 with open(file_path, 'r', encoding='utf-8') as f: content = f.read() - return f"Using f.read() to process {file_path.name}\n\n{content}" + return f"# {file_path.name}\n\n{content}" elif file_ext in ['.doc', '.docx']: # 处理 Word 文档 - try: - from docx import Document - doc = Document(file_path) - text = '\n'.join([para.text for para in doc.paragraphs]) - return f"Using python-docx to process {file_path.name}\n\n{text}" - except ImportError: - logger.warning("python-docx not installed, cannot process .docx files") - return f"# {file_path.name}\n\n[Cannot process .docx file - python-docx not installed]" + from docx import Document + doc = Document(file_path) + text = '\n'.join([para.text for para in doc.paragraphs]) + return f"# {file_path.name}\n\n{text}" elif file_ext in ['.jpg', '.jpeg', '.png', '.bmp']: # 使用 OCR 处理图片 text = ocr.process_image(str(file_path)) - return f"Using OCR to process {file_path.name}\n\n{text}" + return f"# {file_path.name}\n\n{text}" else: # 尝试作为文本文件读取 import textract text = textract.process(file_path) - return f"Using textract to process {file_path.name}\n\n{text}" + return f"# {file_path.name}\n\n{text}" async def _process_url_to_markdown(self, url: str, params=None) -> str: """将 URL 转换为 markdown 格式""" - try: - import requests - from bs4 import BeautifulSoup + import requests + from bs4 import BeautifulSoup - response = requests.get(url, timeout=30) - soup = BeautifulSoup(response.content, 'html.parser') - text_content = soup.get_text() - return f"# {url}\n\n{text_content}" - except Exception as e: - logger.warning(f"Failed to scrape URL {url}: {e}") - return f"# {url}\n\n[Failed to scrape URL: {str(e)}]" + response = requests.get(url, timeout=30) + soup = BeautifulSoup(response.content, 'html.parser') + text_content = soup.get_text() + return f"# {url}\n\n{text_content}" # ============================================================================= # data_router.py 中使用的核心方法 diff --git a/src/utils/db_migration.py b/src/utils/db_migration.py deleted file mode 100644 index 40c966ff..00000000 --- a/src/utils/db_migration.py +++ /dev/null @@ -1,102 +0,0 @@ -""" -数据库迁移工具：用于将独立的knowledge.db数据迁移到整合的server.db中 -""" - -import os -import sqlite3 -import shutil -from datetime import datetime -from src import config -from src.utils import logger - -def migrate_knowledge_db(): - """ - 将独立的knowledge.db数据迁移到整合的server.db中 - - Returns: - bool: 迁移是否成功 - """ - # 确定数据库路径 - knowledge_db_path = os.path.join(config.save_dir, "data", "knowledge.db") - server_db_path = os.path.join(config.save_dir, "data", "server.db") - - # 检查源数据库是否存在 - if not os.path.exists(knowledge_db_path): - logger.info("未找到knowledge.db数据库，无需迁移") - return False - - # 确保目标数据库目录存在 - os.makedirs(os.path.dirname(server_db_path), exist_ok=True) - - try: - # 备份两个数据库 - timestamp = datetime.now().strftime("%Y%m%d%H%M%S") - if os.path.exists(server_db_path): - shutil.copy2(server_db_path, f"{server_db_path}.bak.{timestamp}") - logger.info(f"已备份server.db到 {server_db_path}.bak.{timestamp}") - - shutil.copy2(knowledge_db_path, f"{knowledge_db_path}.bak.{timestamp}") - logger.info(f"已备份knowledge.db到 {knowledge_db_path}.bak.{timestamp}") - - # 连接到两个数据库 - knowledge_conn = sqlite3.connect(knowledge_db_path) - server_conn = sqlite3.connect(server_db_path) - - knowledge_cursor = knowledge_conn.cursor() - server_cursor = server_conn.cursor() - - # 获取knowledge.db中的表 - knowledge_cursor.execute("SELECT name FROM sqlite_master WHERE type='table';") - tables = knowledge_cursor.fetchall() - - # 迁移每个表 - for table in tables: - table_name = table[0] - if table_name.startswith('sqlite_'): - continue # 跳过SQLite内部表 - - logger.info(f"正在迁移表 {table_name}...") - - # 获取表结构 - knowledge_cursor.execute(f"PRAGMA table_info({table_name});") - columns = knowledge_cursor.fetchall() - - # 在server.db中创建表（如果不存在） - column_defs = ", ".join([f'"{col[1]}" {col[2]}' for col in columns]) - create_table_sql = f'CREATE TABLE IF NOT EXISTS "{table_name}" ({column_defs});' - server_cursor.execute(create_table_sql) - - # 获取数据 - knowledge_cursor.execute(f'SELECT * FROM "{table_name}";') - rows = knowledge_cursor.fetchall() - - if rows: - # 准备插入语句 - placeholders = ", ".join(["?" for _ in range(len(columns))]) - insert_sql = f'INSERT OR REPLACE INTO "{table_name}" VALUES ({placeholders});' - - # 批量插入数据 - server_cursor.executemany(insert_sql, rows) - logger.info(f"已迁移 {len(rows)} 条记录到表 {table_name}") - - # 提交更改 - server_conn.commit() - - # 关闭连接 - knowledge_cursor.close() - knowledge_conn.close() - server_cursor.close() - server_conn.close() - - logger.info("数据库迁移完成") - logger.warning(f"请手动删除旧的knowledge.db文件: {knowledge_db_path}") - return True - - except Exception as e: - logger.error(f"数据库迁移失败: {str(e)}") - return False - -if __name__ == "__main__": - # 当作为脚本运行时执行迁移 - result = migrate_knowledge_db() - print(f"迁移{'成功' if result else '失败'}") diff --git a/src/utils/logging_config.py b/src/utils/logging_config.py index c3692937..90d33cdf 100644 --- a/src/utils/logging_config.py +++ b/src/utils/logging_config.py @@ -5,12 +5,13 @@ import pytz from loguru import logger as loguru_logger -DATETIME = datetime.now(pytz.timezone('Asia/Shanghai')).strftime('%Y-%m-%d-%H%M%S') -# DATETIME = "debug" # 为了方便，调试的时候输出到 debug.log 文件 -LOG_FILE = f'tmp/logs/project-{DATETIME}.log' +SAVE_DIR = os.getenv('SAVE_DIR', 'saves') +DATETIME = datetime.now(pytz.timezone('Asia/Shanghai')).strftime('%Y-%m-%d') +LOG_FILE = f'{SAVE_DIR}/logs/yuxi-{DATETIME}.log' + def setup_logger(name, level="DEBUG", console=True): """使用 loguru 设置日志记录器""" - os.makedirs("tmp/logs", exist_ok=True) + os.makedirs(f"{SAVE_DIR}/logs", exist_ok=True) # 创建日志目录 # 移除默认的 handler loguru_logger.remove() @@ -31,7 +32,7 @@ def setup_logger(name, level="DEBUG", console=True): loguru_logger.add( lambda msg: print(msg, end=""), level=level, - format="{time:YYYY-MM-DD HH:mm:ss} - {level} - {name} - {message}", + format="{time:MM-DD HH:mm:ss} {level} {name}: {message}", colorize=True ) diff --git a/web/src/views/DataBaseInfoView.vue b/web/src/views/DataBaseInfoView.vue index df6b6251..64d647c0 100644 --- a/web/src/views/DataBaseInfoView.vue +++ b/web/src/views/DataBaseInfoView.vue @@ -1181,6 +1181,9 @@ const toggleAutoRefresh = (checked) => { display: flex; flex-direction: column; border-radius: 12px; + word-break: break-all; + word-break: break-word; /* 非标准，但某些浏览器支持 */ + overflow-wrap: break-word; /* 标准写法 */ gap: 20px; .results-overview {