diff --git a/server/routers/knowledge_router.py b/server/routers/knowledge_router.py index a52bfe9a..211d83d3 100644 --- a/server/routers/knowledge_router.py +++ b/server/routers/knowledge_router.py @@ -17,8 +17,8 @@ from src import config, knowledge_base from src.knowledge.indexing import SUPPORTED_FILE_EXTENSIONS, is_supported_file_extension, process_file_to_markdown from src.knowledge.utils import calculate_content_hash, merge_processing_params from src.models.embed import test_embedding_model_status, test_all_embedding_models_status -from src.storage.minio.client import aupload_file_to_minio, get_minio_client -from src.utils import hashstr, logger +from src.storage.minio.client import aupload_file_to_minio, get_minio_client, StorageError +from src.utils import logger knowledge = APIRouter(prefix="/knowledge", tags=["knowledge"]) @@ -328,17 +328,6 @@ async def add_documents( "failed": failed_count, } message = f"{item_type}处理完成，失败 {failed_count} 个" if failed_count else f"{item_type}处理完成" - - for success_item in success_items: - # 使用异步上传到minio的对应知识库,同名文件会被覆盖 - async with aiofiles.open(success_item["path"], "rb") as f: - file_bytes = await f.read() - # 上传的bucket名为ref-{refdb},refdb中的_替换为- - refdb = db_id.replace("_", "-") - url = await aupload_file_to_minio( - f"ref-{refdb}", success_item["filename"], file_bytes, success_item["file_type"] - ) - logger.info(f"上传文件成功: {url}") await context.set_result(summary | {"items": processed_items}) await context.set_progress(100.0, message) return summary | {"items": processed_items} @@ -536,12 +525,16 @@ async def rechunks_documents( @knowledge.get("/databases/{db_id}/documents/{doc_id}/download") async def download_document(db_id: str, doc_id: str, request: Request, current_user: User = Depends(get_admin_user)): - """下载原始文件""" + """下载原始文件 - 根据path类型选择本地或MinIO下载""" logger.debug(f"Download document {doc_id} from {db_id}") try: file_info = await knowledge_base.get_file_basic_info(db_id, doc_id) - # 获取文件扩展名和MIME类型，解码URL编码的文件名 - filename = file_info.get("meta", {}).get("filename", "file") + file_meta = file_info.get("meta", {}) + + # 获取文件路径和文件名 + file_path = file_meta.get("path", "") + filename = file_meta.get("filename", "file") + logger.debug(f"File path from database: {file_path}") logger.debug(f"Original filename from database: {filename}") # 解码URL编码的文件名（如果有的话） @@ -553,45 +546,103 @@ async def download_document(db_id: str, doc_id: str, request: Request, current_u decoded_filename = filename # 如果解码失败，使用原文件名 _, ext = os.path.splitext(decoded_filename) - media_type = media_types.get(ext.lower(), "application/octet-stream") - minio_client = get_minio_client() - minio_response = await minio_client.adownload_response( - bucket_name="ref-" + db_id.replace("_", "-"), - object_name=filename, - ) + # 根据path类型选择下载方式 + from src.knowledge.utils.kb_utils import is_minio_url + if is_minio_url(file_path): + # MinIO下载 + logger.debug(f"Downloading from MinIO: {file_path}") - # 创建流式生成器 - async def minio_stream(): try: - while True: - chunk = await asyncio.to_thread(minio_response.read, 8192) - if not chunk: - break - yield chunk - finally: - minio_response.close() - minio_response.release_conn() + # 使用通用函数解析MinIO URL + from src.knowledge.utils.kb_utils import parse_minio_url + bucket_name, object_name = parse_minio_url(file_path) - # 创建StreamingResponse - response = StreamingResponse( - minio_stream(), - media_type=media_type, - ) - # 正确处理中文文件名的HTTP头部设置 - # HTTP头部只能包含ASCII字符，所以需要对中文文件名进行编码 - try: - # 尝试使用ASCII编码（适用于英文文件名） - decoded_filename.encode("ascii") - # 如果成功，直接使用简单格式 - response.headers["Content-Disposition"] = f'attachment; filename="{decoded_filename}"' - except UnicodeEncodeError: - # 如果包含非ASCII字符（如中文），使用RFC 2231格式 - encoded_filename = quote(decoded_filename.encode("utf-8")) - response.headers["Content-Disposition"] = f"attachment; filename*=UTF-8''{encoded_filename}" + logger.debug(f"Parsed bucket_name: {bucket_name}, object_name: {object_name}") + + minio_client = get_minio_client() + + # 直接使用解析出的完整对象名称下载 + minio_response = await minio_client.adownload_response( + bucket_name=bucket_name, + object_name=object_name, + ) + logger.debug(f"Successfully downloaded object: {object_name}") + + except Exception as e: + logger.error(f"Failed to download MinIO file: {e}") + raise StorageError(f"下载文件失败: {e}") + + # 创建流式生成器 + async def minio_stream(): + try: + while True: + chunk = await asyncio.to_thread(minio_response.read, 8192) + if not chunk: + break + yield chunk + finally: + minio_response.close() + minio_response.release_conn() + + # 创建StreamingResponse + response = StreamingResponse( + minio_stream(), + media_type=media_type, + ) + # 正确处理中文文件名的HTTP头部设置 + try: + # 尝试使用ASCII编码（适用于英文文件名） + decoded_filename.encode("ascii") + # 如果成功，直接使用简单格式 + response.headers["Content-Disposition"] = f'attachment; filename="{decoded_filename}"' + except UnicodeEncodeError: + # 如果包含非ASCII字符（如中文），使用RFC 2231格式 + encoded_filename = quote(decoded_filename.encode("utf-8")) + response.headers["Content-Disposition"] = f"attachment; filename*=UTF-8''{encoded_filename}" + + return response + + else: + # 本地文件下载 + logger.debug(f"Downloading from local filesystem: {file_path}") + + if not os.path.exists(file_path): + raise StorageError(f"文件不存在: {file_path}") + + # 获取文件大小 + file_size = os.path.getsize(file_path) + + # 创建文件流式生成器 + async def file_stream(): + async with aiofiles.open(file_path, "rb") as f: + while True: + chunk = await f.read(8192) + if not chunk: + break + yield chunk + + # 创建StreamingResponse + response = StreamingResponse( + file_stream(), + media_type=media_type, + ) + # 正确处理中文文件名的HTTP头部设置 + try: + # 尝试使用ASCII编码（适用于英文文件名） + decoded_filename.encode("ascii") + # 如果成功，直接使用简单格式 + response.headers["Content-Disposition"] = f'attachment; filename="{decoded_filename}"' + response.headers["Content-Length"] = str(file_size) + except UnicodeEncodeError: + # 如果包含非ASCII字符（如中文），使用RFC 2231格式 + encoded_filename = quote(decoded_filename.encode("utf-8")) + response.headers["Content-Disposition"] = f"attachment; filename*=UTF-8''{encoded_filename}" + response.headers["Content-Length"] = str(file_size) + + return response - return response except HTTPException: raise except Exception as e: @@ -1068,19 +1119,9 @@ async def upload_file( elif not (is_supported_file_extension(file.filename) or ext == ".zip"): raise HTTPException(status_code=400, detail=f"Unsupported file type: {ext}") - # 根据db_id获取上传路径，如果db_id为None则使用默认路径 - if db_id: - upload_dir = knowledge_base.get_db_upload_path(db_id) - else: - upload_dir = os.path.join(config.save_dir, "database", "uploads") - basename, ext = os.path.splitext(file.filename) - filename = f"{basename}_{hashstr(basename, 4, with_salt=True, salt='fixed_salt')}{ext}".lower() - - file_path = os.path.join(upload_dir, filename) - - # 在线程池中执行同步文件系统操作，避免阻塞事件循环 - await asyncio.to_thread(os.makedirs, upload_dir, exist_ok=True) + # 直接使用原始文件名（小写） + filename = f"{basename}{ext}".lower() file_bytes = await file.read() @@ -1093,15 +1134,36 @@ async def upload_file( detail="数据库中已经存在了相同内容文件，File with the same content already exists in this database", ) - # 使用异步文件写入，避免阻塞事件循环 - async with aiofiles.open(file_path, "wb") as buffer: - await buffer.write(file_bytes) + # 直接上传到MinIO，添加时间戳区分版本 + import time + timestamp = int(time.time() * 1000) + minio_filename = f"{basename}_{timestamp}{ext}" + + # 生成符合MinIO规范的存储桶名称（将下划线替换为连字符） + if db_id: + bucket_name = f"ref-{db_id.replace('_', '-')}" + else: + bucket_name = "default-uploads" + + # 上传到MinIO + minio_url = await aupload_file_to_minio(bucket_name, minio_filename, file_bytes, ext.lstrip('.')) + + # 检测同名文件（基于原始文件名） + same_name_files = await knowledge_base.get_same_name_files(db_id, filename) + has_same_name = len(same_name_files) > 0 return { "message": "File successfully uploaded", - "file_path": file_path, + "file_path": minio_url, # MinIO路径作为主要路径 + "minio_path": minio_url, # MinIO路径 "db_id": db_id, "content_hash": content_hash, + "filename": filename, # 原始文件名（小写） + "original_filename": basename, # 原始文件名（去掉后缀） + "minio_filename": minio_filename, # MinIO中的文件名（带时间戳） + "bucket_name": bucket_name, # MinIO存储桶名称 + "same_name_files": same_name_files, # 同名文件列表 + "has_same_name": has_same_name # 是否包含同名文件标志 } diff --git a/src/knowledge/indexing.py b/src/knowledge/indexing.py index 848ec4d3..49f10be4 100644 --- a/src/knowledge/indexing.py +++ b/src/knowledge/indexing.py @@ -388,10 +388,10 @@ async def parse_image_async(file, params=None): async def process_file_to_markdown(file_path: str, params: dict | None = None) -> str: """ - 将不同类型的文件转换为markdown格式 + 将不同类型的文件转换为markdown格式 - 支持本地文件和MinIO文件 Args: - file_path: 文件路径 + file_path: 文件路径或MinIO URL params: 处理参数，对于ZIP文件需要包含 db_id Returns: @@ -402,105 +402,161 @@ async def process_file_to_markdown(file_path: str, params: dict | None = None) - - params['_zip_images_info']: 图片信息列表 - params['_zip_content_hash']: 内容哈希值 """ - file_path_obj = Path(file_path) - file_ext = file_path_obj.suffix.lower() + import tempfile + import aiofiles + import os - if file_ext == ".pdf": - # 使用 OCR 处理 PDF - text = await parse_pdf_async(str(file_path_obj), params=params) - return f"# {file_path_obj.name}\n\n{text}" + # 检测是否是MinIO URL + from src.knowledge.utils.kb_utils import is_minio_url - elif file_ext in [".txt", ".md"]: - # 直接读取文本文件 - with open(file_path_obj, encoding="utf-8") as f: - content = f.read() - return f"# {file_path_obj.name}\n\n{content}" + if is_minio_url(file_path): + # 从MinIO下载文件到临时位置 + logger.debug(f"Downloading file from MinIO: {file_path}") - elif file_ext == ".docx": - text = _extract_docx_markdown_with_images(file_path_obj, params=params) - return f"# {file_path_obj.name}\n\n" + text + # 从MinIO URL中提取文件名 + if "?" in file_path: + file_path_clean = file_path.split("?")[0] + else: + file_path_clean = file_path - elif file_ext == ".doc": - text = _extract_word_text(file_path_obj) - return f"# {file_path_obj.name}\n\n{text}" + original_filename = file_path_clean.split("/")[-1] - elif file_ext in [".jpg", ".jpeg", ".png", ".bmp", ".tiff", ".tif"]: - # 使用 OCR 处理图片 - text = await parse_image_async(str(file_path_obj), params=params) - return f"# {file_path_obj.name}\n\n{text}" + # 创建临时文件 + with tempfile.NamedTemporaryFile(delete=False, suffix=Path(original_filename).suffix) as temp_file: + temp_path = temp_file.name - elif file_ext in [".html", ".htm"]: - # 使用 BeautifulSoup 处理 HTML 文件 - from markdownify import markdownify as md + try: + # 使用通用函数解析MinIO URL并下载文件 + from src.knowledge.utils.kb_utils import parse_minio_url + from src.storage.minio.client import get_minio_client - with open(file_path_obj, encoding="utf-8") as f: - content = f.read() - text = md(content, heading_style="ATX") - return f"# {file_path_obj.name}\n\n{text}" + # 解析MinIO URL获取bucket_name和object_name + bucket_name, object_name = parse_minio_url(file_path) - elif file_ext == ".csv": - # 处理 CSV 文件 - import pandas as pd + # 获取MinIO客户端并下载文件 + minio_client = get_minio_client() + file_content = await minio_client.adownload_file(bucket_name, object_name) - df = pd.read_csv(file_path_obj) - # 将每一行数据与表头组合成独立的表格 - markdown_content = f"# {file_path_obj.name}\n\n" + # 写入临时文件 + async with aiofiles.open(temp_path, "wb") as f: + await f.write(file_content) - for index, row in df.iterrows(): - # 创建包含表头和当前行的小表格 - row_df = pd.DataFrame([row], columns=df.columns) - markdown_table = row_df.to_markdown(index=False) - markdown_content += f"{markdown_table}\n\n" + logger.debug(f"File downloaded to temp path: {temp_path}") - return markdown_content.strip() + # 使用临时文件路径 + actual_file_path = temp_path - elif file_ext in [".xls", ".xlsx"]: - # 处理 Excel 文件 - import pandas as pd - from openpyxl import load_workbook + except Exception as e: + # 清理临时文件 + if os.path.exists(temp_path): + os.unlink(temp_path) + logger.error(f"Failed to download file from MinIO: {e}") + raise ValueError(f"无法从MinIO下载文件: {e}") + else: + # 本地文件 + actual_file_path = file_path - markdown_content = f"# {file_path_obj.name}\n\n" + try: + file_path_obj = Path(actual_file_path) + file_ext = file_path_obj.suffix.lower() + original_filename = file_path_obj.name - # 使用 openpyxl 加载工作簿以正确处理合并单元格 - wb = load_workbook(file_path_obj, data_only=True) + if file_ext == ".pdf": + # 使用 OCR 处理 PDF + text = await parse_pdf_async(str(file_path_obj), params=params) + result = f"{text}" - for sheet_name in wb.sheetnames: - ws = wb[sheet_name] + elif file_ext in [".txt", ".md"]: + # 直接读取文本文件 + with open(file_path_obj, encoding="utf-8") as f: + content = f.read() + result = f"{content}" - # 先取消所有合并单元格，并填充值 - merged_ranges = list(ws.merged_cells.ranges) + elif file_ext == ".docx": + text = _extract_docx_markdown_with_images(file_path_obj, params=params) + result = f"" + text - for merged_range in merged_ranges: - # 获取合并区域左上角单元格的值 - min_row, min_col, max_row, max_col = ( - merged_range.min_row, - merged_range.min_col, - merged_range.max_row, - merged_range.max_col, - ) + elif file_ext == ".doc": + text = _extract_word_text(file_path_obj) + result = f"{text}" - # 获取左上角单元格的值 - top_left_value = ws.cell(row=min_row, column=min_col).value + elif file_ext in [".jpg", ".jpeg", ".png", ".bmp", ".tiff", ".tif"]: + # 使用 OCR 处理图片 + text = await parse_image_async(str(file_path_obj), params=params) + result = f"{text}" - # 取消合并 - ws.unmerge_cells(start_row=min_row, start_column=min_col, end_row=max_row, end_column=max_col) + elif file_ext in [".html", ".htm"]: + # 使用 BeautifulSoup 处理 HTML 文件 + from markdownify import markdownify as md - # 在所有原合并单元格区域填充值 - for row in range(min_row, max_row + 1): - for col in range(min_col, max_col + 1): - ws.cell(row=row, column=col).value = top_left_value + with open(file_path_obj, encoding="utf-8") as f: + content = f.read() + text = md(content, heading_style="ATX") + result = f"{text}" - # 转换为DataFrame - data = [] - for row in ws.iter_rows(values_only=True): - data.append(row) + elif file_ext == ".csv": + # 处理 CSV 文件 + import pandas as pd - # 第一行作为列名 - columns = data[0] if data else [] - df_data = data[1:] if len(data) > 1 else [] + df = pd.read_csv(file_path_obj) + # 将每一行数据与表头组合成独立的表格 + markdown_content = f"" - # 处理重复的列名,给重复的列添加后缀 - columns = _make_unique_columns(columns) + for index, row in df.iterrows(): + # 创建包含表头和当前行的小表格 + row_df = pd.DataFrame([row], columns=df.columns) + markdown_table = row_df.to_markdown(index=False) + markdown_content += f"{markdown_table}\n\n" + + result = markdown_content.strip() + + elif file_ext in [".xls", ".xlsx"]: + # 处理 Excel 文件 + import pandas as pd + from openpyxl import load_workbook + + markdown_content = f"" + + # 使用 openpyxl 加载工作簿以正确处理合并单元格 + wb = load_workbook(file_path_obj, data_only=True) + + for sheet_name in wb.sheetnames: + ws = wb[sheet_name] + + # 先取消所有合并单元格，并填充值 + merged_ranges = list(ws.merged_cells.ranges) + + for merged_range in merged_ranges: + # 获取合并区域左上角单元格的值 + min_row, min_col, max_row, max_col = ( + merged_range.min_row, + merged_range.min_col, + merged_range.max_row, + merged_range.max_col, + ) + + # 获取左上角单元格的值 + top_left_value = ws.cell(row=min_row, column=min_col).value + + # 取消合并 + ws.unmerge_cells(start_row=min_row, start_column=min_col, end_row=max_row, end_column=max_col) + + # 在所有原合并单元格区域填充值 + for row in range(min_row, max_row + 1): + for col in range(min_col, max_col + 1): + ws.cell(row=row, column=col).value = top_left_value + + # 转换为DataFrame + data = [] + for row in ws.iter_rows(values_only=True): + data.append(row) + + # 第一行作为列名 + columns = data[0] if data else [] + df_data = data[1:] if len(data) > 1 else [] + + # 处理重复的列名,给重复的列添加后缀 + columns = _make_unique_columns(columns) df = pd.DataFrame(df_data, columns=columns) @@ -519,34 +575,55 @@ async def process_file_to_markdown(file_path: str, params: dict | None = None) - markdown_table = chunk_df.to_markdown(index=False) markdown_content += f"{markdown_table}\n\n" - return markdown_content.strip() + result = markdown_content.strip() - elif file_ext == ".json": - # 处理 JSON 文件 - import json + elif file_ext == ".json": + # 处理 JSON 文件 + import json - async with aiofiles.open(file_path_obj, encoding="utf-8") as f: - content = await f.read() - data = json.loads(content) - # 将 JSON 数据格式化为 markdown 代码块 - json_str = json.dumps(data, ensure_ascii=False, indent=2) - return f"# {file_path_obj.name}\n\n```json\n{json_str}\n```" + async with aiofiles.open(file_path_obj, encoding="utf-8") as f: + content = await f.read() + data = json.loads(content) + # 将 JSON 数据格式化为 markdown 代码块 + json_str = json.dumps(data, ensure_ascii=False, indent=2) + result = f"```json\n{json_str}\n```" - elif file_ext == ".zip": - if not params or "db_id" not in params: - raise ValueError("ZIP文件处理需要在params中提供db_id参数") + elif file_ext == ".zip": + if not params or "db_id" not in params: + raise ValueError("ZIP文件处理需要在params中提供db_id参数") - result = await asyncio.to_thread(_process_zip_file, str(file_path_obj), params["db_id"]) + zip_result = await asyncio.to_thread(_process_zip_file, str(file_path_obj), params["db_id"]) - # 将处理结果保存到params中供调用方使用 - params["_zip_images_info"] = result["images_info"] - params["_zip_content_hash"] = result["content_hash"] + # 将处理结果保存到params中供调用方使用 + params["_zip_images_info"] = zip_result["images_info"] + params["_zip_content_hash"] = zip_result["content_hash"] - return result["markdown_content"] + result = zip_result["markdown_content"] - else: - # 尝试作为文本文件读取 - raise ValueError(f"Unsupported file type: {file_ext}") + else: + # 尝试作为文本文件读取 + raise ValueError(f"Unsupported file type: {file_ext}") + + except Exception as e: + # 清理临时文件 + if is_minio_url(file_path) and os.path.exists(actual_file_path): + try: + os.unlink(actual_file_path) + logger.debug(f"Cleaned up temp file: {actual_file_path}") + except Exception as cleanup_e: + logger.warning(f"Failed to clean up temp file {actual_file_path}: {cleanup_e}") + raise + + finally: + # 清理临时文件 + if is_minio_url(file_path) and os.path.exists(actual_file_path): + try: + os.unlink(actual_file_path) + logger.debug(f"Cleaned up temp file: {actual_file_path}") + except Exception as e: + logger.warning(f"Failed to clean up temp file {actual_file_path}: {e}") + + return result def _process_zip_file(zip_path: str, db_id: str) -> dict: diff --git a/src/knowledge/manager.py b/src/knowledge/manager.py index e52d094f..47595e47 100644 --- a/src/knowledge/manager.py +++ b/src/knowledge/manager.py @@ -374,6 +374,52 @@ class KnowledgeBaseManager: return False + async def get_same_name_files(self, db_id: str, filename: str) -> list[dict]: + """获取同一知识库中同名文件列表 + 基于原始文件名直接比较 + 返回基础信息：文件名、大小、上传时间 + + Args: + db_id: 数据库ID + filename: 要检测的文件名（原始文件名） + + Returns: + 同名文件列表，每项包含： + - filename: 文件名 + - size: 文件大小 + - created_at: 上传时间 + - file_id: 文件ID（用于下载） + """ + if not db_id or not filename: + return [] + try: + kb_instance = self._get_kb_for_database(db_id) + except KBNotFoundError: + return [] + + same_name_files = [] + for file_id, file_info in kb_instance.files_meta.items(): + if file_info.get("database_id") != db_id: + continue + if file_info.get("status") == "failed": + continue + + # 直接比较文件名（现在就是原始文件名） + current_filename = file_info.get("filename", "") + + if current_filename.lower() == filename.lower(): + same_name_files.append({ + "file_id": file_id, + "filename": current_filename, + "size": file_info.get("size", 0), + "created_at": file_info.get("created_at", ""), + "content_hash": file_info.get("content_hash", "") + }) + + # 按上传时间降序排序 + same_name_files.sort(key=lambda x: x.get("created_at", ""), reverse=True) + return same_name_files + async def update_file(self, db_id: str, region_file_id: str, file_name: str, params: dict | None = None) -> dict: """对单个文件执行更新""" kb_instance = self._get_kb_for_database(db_id) diff --git a/src/knowledge/utils/kb_utils.py b/src/knowledge/utils/kb_utils.py index c8d9abcb..7b8c6bf8 100644 --- a/src/knowledge/utils/kb_utils.py +++ b/src/knowledge/utils/kb_utils.py @@ -13,10 +13,10 @@ from src.utils.datetime_utils import utc_isoformat def validate_file_path(file_path: str, db_id: str = None) -> str: """ - 验证文件路径安全性，防止路径遍历攻击 + 验证文件路径安全性，防止路径遍历攻击 - 支持本地文件和MinIO URL Args: - file_path: 要验证的文件路径 + file_path: 要验证的文件路径或MinIO URL db_id: 数据库ID，用于获取知识库特定的上传目录 Returns: @@ -26,7 +26,12 @@ def validate_file_path(file_path: str, db_id: str = None) -> str: ValueError: 如果路径不安全 """ try: - # 规范化路径 + # 检测是否是MinIO URL，如果是则直接返回（不进行路径遍历检查） + if is_minio_url(file_path): + logger.debug(f"MinIO URL detected, skipping path validation: {file_path}") + return file_path + + # 规范化路径（仅对本地文件） normalized_path = os.path.abspath(os.path.realpath(file_path)) # 获取允许的根目录 @@ -134,32 +139,71 @@ async def calculate_content_hash(data: bytes | bytearray | str | os.PathLike[str async def prepare_item_metadata(item: str, content_type: str, db_id: str, params: dict | None = None) -> dict: """ - 准备文件或URL的元数据 + 准备文件或URL的元数据 - 支持本地文件和MinIO文件 Args: - item: 文件路径或URL + item: 文件路径或MinIO URL content_type: 内容类型 ("file" 或 "url") db_id: 数据库ID params: 处理参数，可选 """ if content_type == "file": - file_path = Path(item) - file_id = f"file_{hashstr(str(file_path) + str(time.time()), 6)}" - file_type = file_path.suffix.lower().replace(".", "") - filename = file_path.name - item_path = os.path.relpath(file_path, Path.cwd()) - content_hash = None - try: - if file_path.exists(): - content_hash = await calculate_content_hash(file_path) - except Exception as exc: # noqa: BLE001 - logger.warning(f"Failed to calculate content hash for {file_path}: {exc}") + # 检测是否是MinIO URL还是本地文件路径 + if is_minio_url(item): + # MinIO文件处理 + logger.debug(f"Processing MinIO file: {item}") + # 从MinIO URL中提取文件名 + if "?" in item: + # URL可能包含查询参数，去掉它们 + item_clean = item.split("?")[0] + else: + item_clean = item + + # 获取文件名（从路径的最后部分） + filename = item_clean.split("/")[-1] + + # 如果文件名包含时间戳，提取原始文件名 + import re + timestamp_pattern = r'^(.+)_(\d{13})(\.[^.]+)$' + match = re.match(timestamp_pattern, filename) + if match: + original_filename = match.group(1) + match.group(3) + # 存储原始文件名用于显示 + filename_display = original_filename + else: + filename_display = filename + + file_type = filename.split(".")[-1].lower() if "." in filename else "" + item_path = item # 保持MinIO URL作为路径 + + # 从URL或params中获取content_hash（如果有的话） + content_hash = None + if params and "content_hash" in params: + content_hash = params["content_hash"] + + else: + # 本地文件处理 + file_path = Path(item) + file_type = file_path.suffix.lower().replace(".", "") + filename = file_path.name + filename_display = filename + item_path = os.path.relpath(file_path, Path.cwd()) + content_hash = None + try: + if file_path.exists(): + content_hash = await calculate_content_hash(file_path) + except Exception as exc: # noqa: BLE001 + logger.warning(f"Failed to calculate content hash for {file_path}: {exc}") + + # 生成文件ID + file_id = f"file_{hashstr(str(item_path) + str(time.time()), 6)}" + else: raise ValueError("URL 元数据生成已禁用") metadata = { "database_id": db_id, - "filename": filename, + "filename": filename_display, # 使用显示用的文件名 "path": item_path, "file_type": file_type, "status": "processing", @@ -283,3 +327,54 @@ def get_embedding_config(embed_info: dict) -> dict: logger.debug(f"Embedding config: {config_dict}") return config_dict + + +def is_minio_url(file_path: str) -> bool: + """ + 检测是否是MinIO URL + + Args: + file_path: 文件路径或URL + + Returns: + bool: 是否是MinIO URL + """ + return file_path.startswith(("http://", "https://", "s3://")) or "minio" in file_path.lower() + + +def parse_minio_url(file_path: str) -> tuple[str, str]: + """ + 解析MinIO URL，提取bucket名称和对象名称 + + Args: + file_path: MinIO文件URL + + Returns: + tuple[str, str]: (bucket_name, object_name) + + Raises: + ValueError: 如果无法解析URL + """ + try: + from urllib.parse import urlparse + + # 解析URL + parsed_url = urlparse(file_path) + + # 从URL路径中提取对象名称（去掉开头的斜杠） + object_name = parsed_url.path.lstrip('/') + + # 分离bucket名称和对象名称 + path_parts = object_name.split('/', 1) + if len(path_parts) > 1: + bucket_name = path_parts[0] + object_name = path_parts[1] + else: + raise ValueError(f"无法解析MinIO URL中的bucket名称: {file_path}") + + logger.debug(f"Parsed MinIO URL: bucket_name={bucket_name}, object_name={object_name}") + return bucket_name, object_name + + except Exception as e: + logger.error(f"Failed to parse MinIO URL {file_path}: {e}") + raise ValueError(f"无法解析MinIO URL: {file_path}") diff --git a/web/src/components/AgentChatComponent.vue b/web/src/components/AgentChatComponent.vue index 0d1e050f..cbad97a4 100644 --- a/web/src/components/AgentChatComponent.vue +++ b/web/src/components/AgentChatComponent.vue @@ -565,8 +565,8 @@ const resetOnGoingConv = (threadId = null, preserveMessages = false) => { if (preserveMessages) { setTimeout(() => { if (threadState.onGoingConv) { - threadState.onGoingConv = createOnGoingConvState(); - } + threadState.onGoingConv = createOnGoingConvState(); + } }, 100); } else { threadState.onGoingConv = createOnGoingConvState(); diff --git a/web/src/components/AgentConfigSidebar.vue b/web/src/components/AgentConfigSidebar.vue index 6689acd0..ba9fbdac 100644 --- a/web/src/components/AgentConfigSidebar.vue +++ b/web/src/components/AgentConfigSidebar.vue @@ -696,7 +696,7 @@ watch(() => props.isOpen, (newVal) => { } .config-item { - background-color: var(--gray-50); + background-color: var(--gray-25); padding: 12px; border-radius: 8px; box-shadow: 0px 1px 1px var(--shadow-2); diff --git a/web/src/components/FileTable.vue b/web/src/components/FileTable.vue index adee3de3..90e86e93 100644 --- a/web/src/components/FileTable.vue +++ b/web/src/components/FileTable.vue @@ -533,7 +533,7 @@ import ChunkParamsConfig from '@/components/ChunkParamsConfig.vue'; overflow: hidden; border-radius: 12px; border: 1px solid var(--gray-150); - padding-top: 6px; + /* padding-top: 6px; */ } .panel-header { @@ -542,7 +542,7 @@ import ChunkParamsConfig from '@/components/ChunkParamsConfig.vue'; align-items: center; margin-bottom: 4px; flex-shrink: 0; - padding: 4px 2px; + padding: 4px 4px; } .search-container { diff --git a/web/src/components/FileUploadModal.vue b/web/src/components/FileUploadModal.vue index 02fa1079..8f5993f3 100644 --- a/web/src/components/FileUploadModal.vue +++ b/web/src/components/FileUploadModal.vue @@ -102,7 +102,32 @@ - + +
+
+ + 当前知识库中已存在以下同名文件： +
+
+
+
+ {{ file.filename }} + {{ formatFileTime(file.created_at) }} +
+
+ + + 下载 + + + + 删除 + +
+
+
+
+ @@ -123,18 +148,20 @@