diff --git a/backend/package/yuxi/agents/common/toolkits/buildin/tools.py b/backend/package/yuxi/agents/common/toolkits/buildin/tools.py index 2b4e4555..6c76cdd4 100644 --- a/backend/package/yuxi/agents/common/toolkits/buildin/tools.py +++ b/backend/package/yuxi/agents/common/toolkits/buildin/tools.py @@ -172,6 +172,7 @@ async def text_to_img_qwen_image( negative_prompt: Annotated[str, "负面提示词,用于指定不想出现在图片中的元素"] = "", num_inference_steps: Annotated[int, "推理步数,范围1-100"] = 20, guidance_scale: Annotated[float, "引导强度,控制图片与提示词的匹配程度"] = 7.5, + user_id: Annotated[str, "用户ID,用于图片归档路径"] = "unknown", ) -> str: """使用 Qwen-Image 模型生成图片,返回图片的URL,需要注意的是,生成结果不会默认展示,需要将返回的URL进行展示处理。""" url = "https://api.siliconflow.cn/v1/images/generations" @@ -202,9 +203,10 @@ async def text_to_img_qwen_image( response = requests.get(image_url) file_data = response.content - file_name = f"{uuid.uuid4()}.jpg" + safe_user_id = str(user_id or "unknown").replace("/", "_").replace("\\", "_") + file_name = f"user/{safe_user_id}/generated-images/{uuid.uuid4()}.jpg" image_url = await aupload_file_to_minio( - bucket_name="generated-images", file_name=file_name, data=file_data, file_extension="jpg" + bucket_name="public", file_name=file_name, data=file_data, file_extension="jpg" ) logger.info(f"Image uploaded. URL: {image_url}") return image_url diff --git a/backend/package/yuxi/knowledge/base.py b/backend/package/yuxi/knowledge/base.py index bf70ccb8..e47f19c6 100644 --- a/backend/package/yuxi/knowledge/base.py +++ b/backend/package/yuxi/knowledge/base.py @@ -365,14 +365,10 @@ class KnowledgeBase(ABC): from yuxi.storage.minio import get_minio_client minio_client = get_minio_client() - bucket_name = "kb-documents" # Or reuse existing bucket strategy? - # Maybe store in 'kb-files' or 'kb-markdowns' - # Current uploads go to 'kb-files' usually? - # Let's use 'kb-parsed' - bucket_name = "kb-parsed" + bucket_name = minio_client.KB_BUCKETS["parsed"] await asyncio.to_thread(minio_client.ensure_bucket_exists, bucket_name) - object_name = f"{db_id}/{file_id}/parsed.md" + object_name = f"{db_id}/parsed/{file_id}.md" data = content.encode("utf-8") # Return standard HTTP URL from UploadResult @@ -494,22 +490,23 @@ class KnowledgeBase(ABC): except Exception as e: logger.warning(f"Failed to delete MinIO file {file_path}: {e}") - # 删除解析后的 markdown 文件 (kb-parsed/{db_id}/{file_id}/parsed.md) - parsed_object = f"{db_id}/{file_id}/parsed.md" + # 删除解析后的 markdown 文件 + parsed_object = f"{db_id}/parsed/{file_id}.md" await minio_client.adelete_file(minio_client.KB_BUCKETS["parsed"], parsed_object) del self.files_meta[file_id] # 2. 并行删除所有知识库 bucket 中该 db_id 下的文件 prefix = f"{db_id}/" - await asyncio.gather( - minio_client.adelete_objects_by_prefix(minio_client.KB_BUCKETS["parsed"], prefix), - minio_client.adelete_objects_by_prefix(minio_client.KB_BUCKETS["documents"], prefix), - minio_client.adelete_objects_by_prefix(minio_client.KB_BUCKETS["images"], prefix), - # 直接删除 ref bucket(两种命名格式) - minio_client.adelete_bucket(minio_client.get_ref_bucket_name(db_id)), - minio_client.adelete_bucket(minio_client.get_ref_bucket_name_full(db_id)), - ) + cleanup_buckets = { + minio_client.KB_BUCKETS["parsed"], + minio_client.KB_BUCKETS["documents"], + minio_client.KB_BUCKETS["images"], + } + cleanup_tasks = [ + minio_client.adelete_objects_by_prefix(bucket_name, prefix) for bucket_name in cleanup_buckets + ] + await asyncio.gather(*cleanup_tasks) # 3. 删除数据库记录 del self.databases_meta[db_id] diff --git a/backend/package/yuxi/knowledge/indexing.py b/backend/package/yuxi/knowledge/indexing.py index e0b05430..ec18cdeb 100644 --- a/backend/package/yuxi/knowledge/indexing.py +++ b/backend/package/yuxi/knowledge/indexing.py @@ -22,7 +22,7 @@ from markdownify import markdownify as md_convert from yuxi.plugins.parser.zip_utils import process_zip_file as _process_zip_file from yuxi.storage.minio import get_minio_client -from yuxi.utils import hashstr, logger +from yuxi.utils import logger SUPPORTED_FILE_EXTENSIONS: tuple[str, ...] = ( ".txt", @@ -69,14 +69,32 @@ def _get_docling_converter() -> DocumentConverter: return _docling_converter -def _upload_image_to_minio(image_data: bytes, filename: str, db_id: str) -> str: +def _resolve_image_storage_params(params: dict | None) -> tuple[str, str]: + params = params or {} + + image_bucket = params.get("image_bucket") or "public" + image_prefix = params.get("image_prefix") + if image_prefix: + normalized_prefix = str(image_prefix).strip("/") + if normalized_prefix: + return image_bucket, normalized_prefix + + db_id = params.get("db_id") + if db_id: + return image_bucket, f"{db_id}/kb-images" + + return image_bucket, "unknown/kb-images" + + +def _upload_image_to_minio(image_data: bytes, filename: str, bucket_name: str, object_prefix: str) -> str: """上传图片到 MinIO,返回 URL""" minio_client = get_minio_client() - minio_client.ensure_bucket_exists("kb-images") - file_id = hashstr(filename, length=16) + minio_client.ensure_bucket_exists(bucket_name) + + normalized_prefix = object_prefix.strip("/") or "unknown/kb-images" timestamp = int(time.time() * 1000000) suffix = Path(filename).suffix.lower() - object_name = f"{db_id}/{file_id}/images/{timestamp}_{Path(filename).name}" + object_name = f"{normalized_prefix}/{timestamp}_{Path(filename).name}" content_type_map = { ".jpg": "image/jpeg", ".jpeg": "image/jpeg", @@ -90,7 +108,7 @@ def _upload_image_to_minio(image_data: bytes, filename: str, db_id: str) -> str: content_type = content_type_map.get(suffix, "image/jpeg") result = minio_client.upload_file( - bucket_name="kb-images", + bucket_name=bucket_name, object_name=object_name, data=image_data, content_type=content_type, @@ -112,13 +130,13 @@ def _convert_with_docling(file_path: Path, params: dict | None = None) -> str: Args: file_path: 文件路径 - params: 参数,包含 db_id 用于图片上传 + params: 参数,可包含 image_bucket/image_prefix Returns: Markdown 字符串 """ params = params or {} - db_id = params.get("db_id") or "docling-docs" + image_bucket, image_prefix = _resolve_image_storage_params(params) converter = _get_docling_converter() result = converter.convert(file_path) @@ -145,7 +163,7 @@ def _convert_with_docling(file_path: Path, params: dict | None = None) -> str: image_urls: list[str] = [] for filename, image_data in image_refs: try: - url = _upload_image_to_minio(image_data, filename, db_id) + url = _upload_image_to_minio(image_data, filename, image_bucket, image_prefix) image_urls.append(f"![{filename}]({url})") except Exception as e: logger.error(f"上传图片失败 {filename}: {e}") @@ -334,6 +352,10 @@ def parse_pdf(file, params=None): if opt_ocr == "disable": return pdfreader(file, params=params) + image_bucket, image_prefix = _resolve_image_storage_params(params) + params.setdefault("image_bucket", image_bucket) + params.setdefault("image_prefix", image_prefix) + try: return DocumentProcessorFactory.process_file(opt_ocr, file, params) @@ -373,6 +395,10 @@ def parse_image(file, params=None): "请选择OCR方式 (rapid_ocr/mineru_ocr/mineru_official/pp_structure_v3_ocr) 或移除该文件。" ) + image_bucket, image_prefix = _resolve_image_storage_params(params) + params.setdefault("image_bucket", image_bucket) + params.setdefault("image_prefix", image_prefix) + try: return DocumentProcessorFactory.process_file(opt_ocr, file, params) @@ -543,14 +569,19 @@ async def process_file_to_markdown(file_path: str, params: dict | None = None) - result = f"```json\n{json_str}\n```" elif file_ext == ".zip": - if not params or "db_id" not in params: - raise ValueError("ZIP文件处理需要在params中提供db_id参数") - - zip_result = await _process_zip_file(str(file_path_obj), params["db_id"]) + image_bucket, image_prefix = _resolve_image_storage_params(params) + zip_result = await _process_zip_file( + str(file_path_obj), + image_bucket=image_bucket, + image_prefix=image_prefix, + ) # 将处理结果保存到params中供调用方使用 - params["_zip_images_info"] = zip_result["images_info"] - params["_zip_content_hash"] = zip_result["content_hash"] + if params is not None: + params["_zip_images_info"] = zip_result["images_info"] + params["_zip_content_hash"] = zip_result["content_hash"] + params["_zip_image_bucket"] = image_bucket + params["_zip_image_prefix"] = image_prefix result = zip_result["markdown_content"] diff --git a/backend/package/yuxi/plugins/parser/mineru.py b/backend/package/yuxi/plugins/parser/mineru.py index aabe61e1..15389cea 100644 --- a/backend/package/yuxi/plugins/parser/mineru.py +++ b/backend/package/yuxi/plugins/parser/mineru.py @@ -189,7 +189,14 @@ class MinerUParser(BaseDocumentProcessor): tmp_zip.flush() try: - processed = process_zip_file_sync(tmp_zip.name, params.get("db_id") or "ocr-temp") + image_bucket = params.get("image_bucket") or "public" + image_prefix = params.get("image_prefix") or "unknown/kb-images" + + processed = process_zip_file_sync( + tmp_zip.name, + image_bucket=image_bucket, + image_prefix=image_prefix, + ) text = processed["markdown_content"] finally: os.unlink(tmp_zip.name) diff --git a/backend/package/yuxi/plugins/parser/mineru_official.py b/backend/package/yuxi/plugins/parser/mineru_official.py index 11450afa..2e530b71 100644 --- a/backend/package/yuxi/plugins/parser/mineru_official.py +++ b/backend/package/yuxi/plugins/parser/mineru_official.py @@ -156,7 +156,14 @@ class MinerUOfficialParser(BaseDocumentProcessor): return text try: - processed = process_zip_file_sync(zip_path, params.get("db_id") or "ocr-test") + image_bucket = params.get("image_bucket") or "public" + image_prefix = params.get("image_prefix") or "unknown/kb-images" + + processed = process_zip_file_sync( + zip_path, + image_bucket=image_bucket, + image_prefix=image_prefix, + ) text = processed["markdown_content"] except Exception: import zipfile diff --git a/backend/package/yuxi/plugins/parser/zip_utils.py b/backend/package/yuxi/plugins/parser/zip_utils.py index 717952e1..7cd23292 100644 --- a/backend/package/yuxi/plugins/parser/zip_utils.py +++ b/backend/package/yuxi/plugins/parser/zip_utils.py @@ -7,16 +7,29 @@ import zipfile from pathlib import Path from yuxi.storage.minio import get_minio_client -from yuxi.utils import hashstr, logger +from yuxi.utils import logger + +DEFAULT_IMAGE_BUCKET = "public" +DEFAULT_IMAGE_PREFIX = "unknown/kb-images" -async def process_zip_file(zip_path: str, db_id: str) -> dict: +def _normalize_object_prefix(prefix: str | None) -> str: + normalized = (prefix or DEFAULT_IMAGE_PREFIX).strip("/") + return normalized or DEFAULT_IMAGE_PREFIX + + +async def process_zip_file( + zip_path: str, + image_bucket: str = DEFAULT_IMAGE_BUCKET, + image_prefix: str = DEFAULT_IMAGE_PREFIX, +) -> dict: """ 处理ZIP文件,提取markdown内容和图片 Args: zip_path: ZIP文件路径 - db_id: 数据库ID + image_bucket: 图片上传的目标 bucket + image_prefix: 图片上传对象前缀 Returns: dict: { @@ -43,9 +56,15 @@ async def process_zip_file(zip_path: str, db_id: str) -> dict: images_info = [] images_dir = find_images_directory(zf, md_file) + normalized_prefix = _normalize_object_prefix(image_prefix) if images_dir: - images_info = await process_images(zf, images_dir, db_id, md_file) + images_info = await process_images( + zf, + images_dir, + image_bucket=image_bucket, + image_prefix=normalized_prefix, + ) markdown_content = replace_image_links(markdown_content, images_info) content_hash = hashlib.sha256(markdown_content.encode("utf-8")).hexdigest() @@ -57,12 +76,16 @@ async def process_zip_file(zip_path: str, db_id: str) -> dict: } -def process_zip_file_sync(zip_path: str, db_id: str) -> dict: +def process_zip_file_sync( + zip_path: str, + image_bucket: str = DEFAULT_IMAGE_BUCKET, + image_prefix: str = DEFAULT_IMAGE_PREFIX, +) -> dict: """同步调用 ZIP 处理,供同步解析器使用。""" try: asyncio.get_running_loop() except RuntimeError: - return asyncio.run(process_zip_file(zip_path, db_id)) + return asyncio.run(process_zip_file(zip_path, image_bucket=image_bucket, image_prefix=image_prefix)) result: dict | None = None error: Exception | None = None @@ -70,7 +93,7 @@ def process_zip_file_sync(zip_path: str, db_id: str) -> dict: def runner() -> None: nonlocal result, error try: - result = asyncio.run(process_zip_file(zip_path, db_id)) + result = asyncio.run(process_zip_file(zip_path, image_bucket=image_bucket, image_prefix=image_prefix)) except Exception as exc: # pragma: no cover - pass through outer raise error = exc @@ -106,7 +129,12 @@ def find_images_directory(zip_file: zipfile.ZipFile, md_file_path: str) -> str | return None -async def process_images(zip_file: zipfile.ZipFile, images_dir: str, db_id: str, md_file_path: str) -> list[dict]: +async def process_images( + zip_file: zipfile.ZipFile, + images_dir: str, + image_bucket: str, + image_prefix: str, +) -> list[dict]: """处理图片:上传到MinIO并返回信息""" supported_extensions = {".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp"} content_type_map = { @@ -120,12 +148,10 @@ async def process_images(zip_file: zipfile.ZipFile, images_dir: str, db_id: str, images = [] image_names = [n for n in zip_file.namelist() if n.startswith(images_dir + "/")] + normalized_prefix = _normalize_object_prefix(image_prefix) minio_client = get_minio_client() - bucket_name = "kb-images" - await asyncio.to_thread(minio_client.ensure_bucket_exists, bucket_name) - - file_id = hashstr(Path(md_file_path).name, length=16) + await asyncio.to_thread(minio_client.ensure_bucket_exists, image_bucket) for img_name in image_names: suffix = Path(img_name).suffix.lower() @@ -137,11 +163,11 @@ async def process_images(zip_file: zipfile.ZipFile, images_dir: str, db_id: str, data = f.read() timestamp = int(time.time() * 1000000) - object_name = f"{db_id}/{file_id}/images/{timestamp}_{Path(img_name).name}" + object_name = f"{normalized_prefix}/{timestamp}_{Path(img_name).name}" content_type = content_type_map.get(suffix, "image/jpeg") result = await minio_client.aupload_file( - bucket_name=bucket_name, + bucket_name=image_bucket, object_name=object_name, data=data, content_type=content_type, diff --git a/backend/package/yuxi/services/conversation_service.py b/backend/package/yuxi/services/conversation_service.py index 3805fb35..2ed43072 100644 --- a/backend/package/yuxi/services/conversation_service.py +++ b/backend/package/yuxi/services/conversation_service.py @@ -15,7 +15,7 @@ from yuxi.utils.datetime_utils import utc_isoformat from yuxi.utils.logging_config import logger # 附件存储桶名称 -ATTACHMENTS_BUCKET = "chat-attachments" +ATTACHMENTS_BUCKET = "user" async def require_user_conversation(conv_repo: ConversationRepository, thread_id: str, user_id: str): @@ -246,14 +246,15 @@ async def upload_thread_attachment_view( file_content = await file.read() await file.seek(0) client = get_minio_client() - object_name = f"attachments/{thread_id}/{conversion.file_name}" + safe_filename = conversion.file_name.replace("/", "_").replace("\\", "_") + object_name = f"user/{current_user_id}/chat_attachments/{thread_id}/{safe_filename}" result = client.upload_file( bucket_name=ATTACHMENTS_BUCKET, object_name=object_name, data=file_content, content_type=conversion.file_type or "application/octet-stream", ) - minio_url = result.public_url + minio_url = result.url logger.info(f"Uploaded attachment to MinIO: {object_name}") except Exception as e: logger.error(f"Failed to upload attachment to MinIO: {e}") diff --git a/backend/package/yuxi/storage/minio/client.py b/backend/package/yuxi/storage/minio/client.py index aa14b6ed..c8ccdc89 100644 --- a/backend/package/yuxi/storage/minio/client.py +++ b/backend/package/yuxi/storage/minio/client.py @@ -41,25 +41,15 @@ class MinIOClient: 简化的 MinIO 客户端类 """ - PUBLIC_READ_BUCKETS = {"generated-images", "avatar", "kb-images"} + PUBLIC_READ_BUCKETS = {"public"} # 知识库相关的 bucket 名称 KB_BUCKETS = { - "documents": "kb-documents", - "parsed": "kb-parsed", - "images": "kb-images", + "documents": "knowledgebases", + "parsed": "knowledgebases", + "images": "public", } - @staticmethod - def get_ref_bucket_name(db_id: str) -> str: - """获取 ref bucket 名称(截断32位)""" - return f"ref-{db_id[:32].replace('_', '-')}" - - @staticmethod - def get_ref_bucket_name_full(db_id: str) -> str: - """获取 ref bucket 名称(不截断)""" - return f"ref-{db_id.replace('_', '-')}" - def __init__(self): """初始化 MinIO 客户端""" self.endpoint = os.getenv("MINIO_URI") or "http://milvus-minio:9000" diff --git a/backend/server/routers/auth_router.py b/backend/server/routers/auth_router.py index 243a09be..08467eb4 100644 --- a/backend/server/routers/auth_router.py +++ b/backend/server/routers/auth_router.py @@ -757,8 +757,8 @@ async def upload_user_avatar( file_extension = file.filename.split(".")[-1].lower() if file.filename and "." in file.filename else "jpg" # 上传到MinIO - file_name = f"{uuid.uuid4()}.{file_extension}" - avatar_url = await aupload_file_to_minio("avatar", file_name, file_content, file_extension) + file_name = f"avatar/{current_user.id}/{uuid.uuid4()}.{file_extension}" + avatar_url = await aupload_file_to_minio("public", file_name, file_content, file_extension) # 更新用户头像 current_user.avatar = avatar_url diff --git a/backend/server/routers/knowledge_router.py b/backend/server/routers/knowledge_router.py index 53cdfdb5..35a2fdfa 100644 --- a/backend/server/routers/knowledge_router.py +++ b/backend/server/routers/knowledge_router.py @@ -16,6 +16,7 @@ from yuxi import config, knowledge_base from yuxi.knowledge.chunking.ragflow_like.presets import ensure_chunk_defaults_in_additional_params from yuxi.knowledge.indexing import SUPPORTED_FILE_EXTENSIONS, is_supported_file_extension, process_file_to_markdown from yuxi.knowledge.utils import calculate_content_hash +from yuxi.knowledge.utils.kb_utils import parse_minio_url from yuxi.models.embed import test_all_embedding_models_status, test_embedding_model_status from yuxi.storage.postgres.models_business import User from yuxi.storage.minio.client import MinIOClient, StorageError, aupload_file_to_minio, get_minio_client @@ -680,15 +681,18 @@ async def batch_delete_documents( deleted_count += 1 continue - file_name = file_meta_info.get("meta", {}).get("filename") + file_path = file_meta_info.get("meta", {}).get("path", "") - # 尝试从MinIO删除文件,如果失败(例如旧知识库没有MinIO实例),则忽略 + # 尝试从 MinIO 删除文件对象与解析结果 try: minio_client = get_minio_client() - await minio_client.adelete_file(MinIOClient.get_ref_bucket_name(db_id), file_name) - logger.debug(f"成功从MinIO删除文件: {file_name}") + if file_path.startswith(("http://", "https://")): + bucket_name, object_name = parse_minio_url(file_path) + await minio_client.adelete_file(bucket_name, object_name) + await minio_client.adelete_file(minio_client.KB_BUCKETS["parsed"], f"{db_id}/parsed/{doc_id}.md") + logger.debug(f"成功从MinIO删除文件: {file_path}") except Exception as minio_error: - logger.warning(f"从MinIO删除文件失败(可能是旧知识库): {minio_error}") + logger.warning(f"从MinIO删除文件失败: {minio_error}") # 无论MinIO删除是否成功,都继续从知识库删除 await knowledge_base.delete_file(db_id, doc_id) @@ -723,17 +727,18 @@ async def delete_document(db_id: str, doc_id: str, current_user: User = Depends( await knowledge_base.delete_folder(db_id, doc_id) return {"message": "文件夹删除成功"} - file_name = file_meta_info.get("meta", {}).get("path", "").split("/")[-1] + file_path = file_meta_info.get("meta", {}).get("path", "") - # 尝试从MinIO删除文件,如果失败(例如旧知识库没有MinIO实例),则忽略 + # 尝试从 MinIO 删除文件对象与解析结果 try: minio_client = get_minio_client() - await minio_client.adelete_file(MinIOClient.get_ref_bucket_name(db_id), file_name) - # 同时删除 parsed bucket 中的 parsed.md 文件 - await minio_client.adelete_file(minio_client.KB_BUCKETS["parsed"], f"{db_id}/{doc_id}/parsed.md") - logger.debug(f"成功从MinIO删除文件: {file_name}") + if file_path.startswith(("http://", "https://")): + bucket_name, object_name = parse_minio_url(file_path) + await minio_client.adelete_file(bucket_name, object_name) + await minio_client.adelete_file(minio_client.KB_BUCKETS["parsed"], f"{db_id}/parsed/{doc_id}.md") + logger.debug(f"成功从MinIO删除文件: {file_path}") except Exception as minio_error: - logger.warning(f"从MinIO删除文件失败(可能是旧知识库): {minio_error}") + logger.warning(f"从MinIO删除文件失败: {minio_error}") # 无论MinIO删除是否成功,都继续从知识库删除 await knowledge_base.delete_file(db_id, doc_id) @@ -1240,13 +1245,11 @@ async def fetch_url( # 3. 上传到 MinIO minio_client = get_minio_client() - # 确保存储 bucket 存在 - bucket_name = "kb-html-archives" + bucket_name = MinIOClient.KB_BUCKETS["documents"] await asyncio.to_thread(minio_client.ensure_bucket_exists, bucket_name) - # 如果没有提供 db_id,使用 default - folder = db_id if db_id else "default" - object_name = f"{folder}/{content_hash}.html" + folder = db_id if db_id else "unknown" + object_name = f"{folder}/upload/{content_hash}.html" upload_result = await minio_client.aupload_file( bucket_name=bucket_name, @@ -1326,14 +1329,12 @@ async def upload_file( timestamp = int(time.time() * 1000) minio_filename = f"{basename}_{timestamp}{ext}" - # 生成符合MinIO规范的存储桶名称(将下划线替换为连字符,截取长度以满足MinIO的63字符限制) - if db_id: - bucket_name = f"ref-{db_id[:32].replace('_', '-')}" - else: - bucket_name = "default-uploads" + bucket_name = MinIOClient.KB_BUCKETS["documents"] + folder = db_id if db_id else "unknown" + object_name = f"{folder}/upload/{minio_filename}" # 上传到MinIO - minio_url = await aupload_file_to_minio(bucket_name, minio_filename, file_bytes, ext.lstrip(".")) + minio_url = await aupload_file_to_minio(bucket_name, object_name, file_bytes, ext.lstrip(".")) # 检测同名文件(基于原始文件名) same_name_files = await knowledge_base.get_same_name_files(db_id, filename) @@ -1348,6 +1349,7 @@ async def upload_file( "filename": filename, # 原始文件名(小写) "original_filename": basename, # 原始文件名(去掉后缀) "minio_filename": minio_filename, # MinIO中的文件名(带时间戳) + "object_name": object_name, "bucket_name": bucket_name, # MinIO存储桶名称 "same_name_files": same_name_files, # 同名文件列表 "has_same_name": has_same_name, # 是否包含同名文件标志 diff --git a/backend/test/test_kb_minio_cleanup.py b/backend/test/test_kb_minio_cleanup.py index b11d537e..d8121924 100644 --- a/backend/test/test_kb_minio_cleanup.py +++ b/backend/test/test_kb_minio_cleanup.py @@ -24,6 +24,8 @@ async def test_delete_knowledge_base_cleanup(): # 初始化 minio_client = get_minio_client() kb_manager = knowledge_base + doc_bucket = minio_client.KB_BUCKETS["documents"] + parsed_bucket = minio_client.KB_BUCKETS["parsed"] # 测试参数 test_db_name = "test_cleanup_db" @@ -44,35 +46,35 @@ async def test_delete_knowledge_base_cleanup(): print("2. 上传测试文件到 MinIO...") # 确保 bucket 存在 - minio_client.ensure_bucket_exists("kb-documents") - minio_client.ensure_bucket_exists("kb-parsed") + minio_client.ensure_bucket_exists(doc_bucket) + minio_client.ensure_bucket_exists(parsed_bucket) - # 上传到 kb-documents (原始文件) - test_object_name = f"{db_id}/test_file_{db_id[:8]}.txt" + # 上传到知识库文档路径 (knowledgebases/{db_id}/upload/...) + test_object_name = f"{db_id}/upload/test_file_{db_id[:8]}.txt" minio_client.upload_file( - bucket_name="kb-documents", + bucket_name=doc_bucket, object_name=test_object_name, data=test_file_content, content_type="text/plain", ) - print(f" 文件上传到 kb-documents: {test_object_name}") + print(f" 文件上传到 {doc_bucket}: {test_object_name}") - # 上传到 kb-parsed (解析后的 markdown) + # 上传到知识库解析路径 (knowledgebases/{db_id}/parsed/...) test_file_id = f"file_test_{db_id[:8]}" - parsed_object_name = f"{db_id}/{test_file_id}/parsed.md" + parsed_object_name = f"{db_id}/parsed/{test_file_id}.md" parsed_content = b"# Test Parsed Content\n\nThis is parsed markdown." minio_client.upload_file( - bucket_name="kb-parsed", + bucket_name=parsed_bucket, object_name=parsed_object_name, data=parsed_content, content_type="text/markdown", ) - print(f" 文件上传到 kb-parsed: {parsed_object_name}") + print(f" 文件上传到 {parsed_bucket}: {parsed_object_name}") # 3. 验证文件存在 print("3. 验证文件存在...") - assert minio_client.file_exists("kb-documents", test_object_name), "原始文件应该存在" - assert minio_client.file_exists("kb-parsed", parsed_object_name), "解析后文件应该存在" + assert minio_client.file_exists(doc_bucket, test_object_name), "原始文件应该存在" + assert minio_client.file_exists(parsed_bucket, parsed_object_name), "解析后文件应该存在" print(" 文件存在验证通过") # 4. 删除知识库 @@ -83,13 +85,13 @@ async def test_delete_knowledge_base_cleanup(): # 5. 验证 MinIO 文件已清理 print("5. 验证 MinIO 文件已清理...") - # 检查 kb-documents 中的文件 - doc_exists = minio_client.file_exists("kb-documents", test_object_name) - print(f" kb-documents/{test_object_name} 存在: {doc_exists}") + # 检查文档对象 + doc_exists = minio_client.file_exists(doc_bucket, test_object_name) + print(f" {doc_bucket}/{test_object_name} 存在: {doc_exists}") - # 检查 kb-parsed 中的文件 - parsed_exists = minio_client.file_exists("kb-parsed", parsed_object_name) - print(f" kb-parsed/{parsed_object_name} 存在: {parsed_exists}") + # 检查解析对象 + parsed_exists = minio_client.file_exists(parsed_bucket, parsed_object_name) + print(f" {parsed_bucket}/{parsed_object_name} 存在: {parsed_exists}") # 6. 输出测试结果 print("\n" + "=" * 50) @@ -99,9 +101,9 @@ async def test_delete_knowledge_base_cleanup(): else: print("❌ 测试失败!以下文件未被清理:") if doc_exists: - print(f" - kb-documents/{test_object_name}") + print(f" - {doc_bucket}/{test_object_name}") if parsed_exists: - print(f" - kb-parsed/{parsed_object_name}") + print(f" - {parsed_bucket}/{parsed_object_name}") return False except Exception as e: @@ -116,9 +118,8 @@ async def test_delete_knowledge_base_cleanup(): print("\n清理测试数据...") try: # 尝试删除可能残留的文件(忽略错误) - for obj in [test_object_name, parsed_object_name]: + for bucket, obj in [(doc_bucket, test_object_name), (parsed_bucket, parsed_object_name)]: try: - bucket = "kb-documents" if "kb-documents" in str(obj) else "kb-parsed" await minio_client.adelete_file(bucket, obj) except Exception: pass