kris/ForcePilot
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

refactor: 更新 MinIO 存储逻辑，优化文件路径和桶名称（不兼容）

Browse Source
This commit is contained in:
Wenjie Zhang 2026-03-18 00:54:27 +08:00
parent fe2153ad76
commit 095f0db8e6
11 changed files with 177 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
backend/package/yuxi/agents/common/toolkits/buildin/tools.py
View File

@ -172,6 +172,7 @@ async def text_to_img_qwen_image(
negative_prompt: Annotated[str, "负面提示词，用于指定不想出现在图片中的元素"] = "",
num_inference_steps: Annotated[int, "推理步数，范围1-100"] = 20,
guidance_scale: Annotated[float, "引导强度，控制图片与提示词的匹配程度"] = 7.5,
user_id: Annotated[str, "用户ID，用于图片归档路径"] = "unknown",
) -> str:
"""使用 Qwen-Image 模型生成图片，返回图片的URL，需要注意的是，生成结果不会默认展示，需要将返回的URL进行展示处理。"""
url = "https://api.siliconflow.cn/v1/images/generations"
@ -202,9 +203,10 @@ async def text_to_img_qwen_image(
response = requests.get(image_url)
file_data = response.content
file_name = f"{uuid.uuid4()}.jpg"
safe_user_id = str(user_id or "unknown").replace("/", "_").replace("\\", "_")
file_name = f"user/{safe_user_id}/generated-images/{uuid.uuid4()}.jpg"
image_url = await aupload_file_to_minio(
bucket_name="generated-images", file_name=file_name, data=file_data, file_extension="jpg"
bucket_name="public", file_name=file_name, data=file_data, file_extension="jpg"
)
logger.info(f"Image uploaded. URL: {image_url}")
return image_url

29
backend/package/yuxi/knowledge/base.py
View File

@ -365,14 +365,10 @@ class KnowledgeBase(ABC):
from yuxi.storage.minio import get_minio_client
minio_client = get_minio_client()
bucket_name = "kb-documents" # Or reuse existing bucket strategy?
# Maybe store in 'kb-files' or 'kb-markdowns'
# Current uploads go to 'kb-files' usually?
# Let's use 'kb-parsed'
bucket_name = "kb-parsed"
bucket_name = minio_client.KB_BUCKETS["parsed"]
await asyncio.to_thread(minio_client.ensure_bucket_exists, bucket_name)
object_name = f"{db_id}/{file_id}/parsed.md"
object_name = f"{db_id}/parsed/{file_id}.md"
data = content.encode("utf-8")
# Return standard HTTP URL from UploadResult
@ -494,22 +490,23 @@ class KnowledgeBase(ABC):
except Exception as e:
logger.warning(f"Failed to delete MinIO file {file_path}: {e}")
# 删除解析后的 markdown 文件 (kb-parsed/{db_id}/{file_id}/parsed.md)
parsed_object = f"{db_id}/{file_id}/parsed.md"
# 删除解析后的 markdown 文件
parsed_object = f"{db_id}/parsed/{file_id}.md"
await minio_client.adelete_file(minio_client.KB_BUCKETS["parsed"], parsed_object)
del self.files_meta[file_id]
# 2. 并行删除所有知识库 bucket 中该 db_id 下的文件
prefix = f"{db_id}/"
await asyncio.gather(
minio_client.adelete_objects_by_prefix(minio_client.KB_BUCKETS["parsed"], prefix),
minio_client.adelete_objects_by_prefix(minio_client.KB_BUCKETS["documents"], prefix),
minio_client.adelete_objects_by_prefix(minio_client.KB_BUCKETS["images"], prefix),
# 直接删除 ref bucket（两种命名格式）
minio_client.adelete_bucket(minio_client.get_ref_bucket_name(db_id)),
minio_client.adelete_bucket(minio_client.get_ref_bucket_name_full(db_id)),
)
cleanup_buckets = {
minio_client.KB_BUCKETS["parsed"],
minio_client.KB_BUCKETS["documents"],
minio_client.KB_BUCKETS["images"],
}
cleanup_tasks = [
minio_client.adelete_objects_by_prefix(bucket_name, prefix) for bucket_name in cleanup_buckets
]
await asyncio.gather(*cleanup_tasks)
# 3. 删除数据库记录
del self.databases_meta[db_id]

61
backend/package/yuxi/knowledge/indexing.py
View File

@ -22,7 +22,7 @@ from markdownify import markdownify as md_convert
from yuxi.plugins.parser.zip_utils import process_zip_file as _process_zip_file
from yuxi.storage.minio import get_minio_client
from yuxi.utils import hashstr, logger
from yuxi.utils import logger
SUPPORTED_FILE_EXTENSIONS: tuple[str, ...] = (
".txt",
@ -69,14 +69,32 @@ def _get_docling_converter() -> DocumentConverter:
return _docling_converter
def _upload_image_to_minio(image_data: bytes, filename: str, db_id: str) -> str:
def _resolve_image_storage_params(params: dict | None) -> tuple[str, str]:
params = params or {}
image_bucket = params.get("image_bucket") or "public"
image_prefix = params.get("image_prefix")
if image_prefix:
normalized_prefix = str(image_prefix).strip("/")
if normalized_prefix:
return image_bucket, normalized_prefix
db_id = params.get("db_id")
if db_id:
return image_bucket, f"{db_id}/kb-images"
return image_bucket, "unknown/kb-images"
def _upload_image_to_minio(image_data: bytes, filename: str, bucket_name: str, object_prefix: str) -> str:
"""上传图片到 MinIO，返回 URL"""
minio_client = get_minio_client()
minio_client.ensure_bucket_exists("kb-images")
file_id = hashstr(filename, length=16)
minio_client.ensure_bucket_exists(bucket_name)
normalized_prefix = object_prefix.strip("/") or "unknown/kb-images"
timestamp = int(time.time() * 1000000)
suffix = Path(filename).suffix.lower()
object_name = f"{db_id}/{file_id}/images/{timestamp}_{Path(filename).name}"
object_name = f"{normalized_prefix}/{timestamp}_{Path(filename).name}"
content_type_map = {
".jpg": "image/jpeg",
".jpeg": "image/jpeg",
@ -90,7 +108,7 @@ def _upload_image_to_minio(image_data: bytes, filename: str, db_id: str) -> str:
content_type = content_type_map.get(suffix, "image/jpeg")
result = minio_client.upload_file(
bucket_name="kb-images",
bucket_name=bucket_name,
object_name=object_name,
data=image_data,
content_type=content_type,
@ -112,13 +130,13 @@ def _convert_with_docling(file_path: Path, params: dict | None = None) -> str:
Args:
file_path: 文件路径
params: 参数，包含 db_id 用于图片上传
params: 参数，可包含 image_bucket/image_prefix
Returns:
Markdown 字符串
"""
params = params or {}
db_id = params.get("db_id") or "docling-docs"
image_bucket, image_prefix = _resolve_image_storage_params(params)
converter = _get_docling_converter()
result = converter.convert(file_path)
@ -145,7 +163,7 @@ def _convert_with_docling(file_path: Path, params: dict | None = None) -> str:
image_urls: list[str] = []
for filename, image_data in image_refs:
try:
url = _upload_image_to_minio(image_data, filename, db_id)
url = _upload_image_to_minio(image_data, filename, image_bucket, image_prefix)
image_urls.append(f"![{filename}]({url})")
except Exception as e:
logger.error(f"上传图片失败 {filename}: {e}")
@ -334,6 +352,10 @@ def parse_pdf(file, params=None):
if opt_ocr == "disable":
return pdfreader(file, params=params)
image_bucket, image_prefix = _resolve_image_storage_params(params)
params.setdefault("image_bucket", image_bucket)
params.setdefault("image_prefix", image_prefix)
try:
return DocumentProcessorFactory.process_file(opt_ocr, file, params)
@ -373,6 +395,10 @@ def parse_image(file, params=None):
"请选择OCR方式 (rapid_ocr/mineru_ocr/mineru_official/pp_structure_v3_ocr) 或移除该文件。"
)
image_bucket, image_prefix = _resolve_image_storage_params(params)
params.setdefault("image_bucket", image_bucket)
params.setdefault("image_prefix", image_prefix)
try:
return DocumentProcessorFactory.process_file(opt_ocr, file, params)
@ -543,14 +569,19 @@ async def process_file_to_markdown(file_path: str, params: dict | None = None) -
result = f"```json\n{json_str}\n```"
elif file_ext == ".zip":
if not params or "db_id" not in params:
raise ValueError("ZIP文件处理需要在params中提供db_id参数")
zip_result = await _process_zip_file(str(file_path_obj), params["db_id"])
image_bucket, image_prefix = _resolve_image_storage_params(params)
zip_result = await _process_zip_file(
str(file_path_obj),
image_bucket=image_bucket,
image_prefix=image_prefix,
)
# 将处理结果保存到params中供调用方使用
params["_zip_images_info"] = zip_result["images_info"]
params["_zip_content_hash"] = zip_result["content_hash"]
if params is not None:
params["_zip_images_info"] = zip_result["images_info"]
params["_zip_content_hash"] = zip_result["content_hash"]
params["_zip_image_bucket"] = image_bucket
params["_zip_image_prefix"] = image_prefix
result = zip_result["markdown_content"]

9
backend/package/yuxi/plugins/parser/mineru.py
View File

@ -189,7 +189,14 @@ class MinerUParser(BaseDocumentProcessor):
tmp_zip.flush()
try:
processed = process_zip_file_sync(tmp_zip.name, params.get("db_id") or "ocr-temp")
image_bucket = params.get("image_bucket") or "public"
image_prefix = params.get("image_prefix") or "unknown/kb-images"
processed = process_zip_file_sync(
tmp_zip.name,
image_bucket=image_bucket,
image_prefix=image_prefix,
)
text = processed["markdown_content"]
finally:
os.unlink(tmp_zip.name)

9
backend/package/yuxi/plugins/parser/mineru_official.py
View File

@ -156,7 +156,14 @@ class MinerUOfficialParser(BaseDocumentProcessor):
return text
try:
processed = process_zip_file_sync(zip_path, params.get("db_id") or "ocr-test")
image_bucket = params.get("image_bucket") or "public"
image_prefix = params.get("image_prefix") or "unknown/kb-images"
processed = process_zip_file_sync(
zip_path,
image_bucket=image_bucket,
image_prefix=image_prefix,
)
text = processed["markdown_content"]
except Exception:
import zipfile

54
backend/package/yuxi/plugins/parser/zip_utils.py
View File

@ -7,16 +7,29 @@ import zipfile
from pathlib import Path
from yuxi.storage.minio import get_minio_client
from yuxi.utils import hashstr, logger
from yuxi.utils import logger
DEFAULT_IMAGE_BUCKET = "public"
DEFAULT_IMAGE_PREFIX = "unknown/kb-images"
async def process_zip_file(zip_path: str, db_id: str) -> dict:
def _normalize_object_prefix(prefix: str | None) -> str:
normalized = (prefix or DEFAULT_IMAGE_PREFIX).strip("/")
return normalized or DEFAULT_IMAGE_PREFIX
async def process_zip_file(
zip_path: str,
image_bucket: str = DEFAULT_IMAGE_BUCKET,
image_prefix: str = DEFAULT_IMAGE_PREFIX,
) -> dict:
"""
处理ZIP文件，提取markdown内容和图片
Args:
zip_path: ZIP文件路径
db_id: 数据库ID
image_bucket: 图片上传的目标 bucket
image_prefix: 图片上传对象前缀
Returns:
dict: {
@ -43,9 +56,15 @@ async def process_zip_file(zip_path: str, db_id: str) -> dict:
images_info = []
images_dir = find_images_directory(zf, md_file)
normalized_prefix = _normalize_object_prefix(image_prefix)
if images_dir:
images_info = await process_images(zf, images_dir, db_id, md_file)
images_info = await process_images(
zf,
images_dir,
image_bucket=image_bucket,
image_prefix=normalized_prefix,
)
markdown_content = replace_image_links(markdown_content, images_info)
content_hash = hashlib.sha256(markdown_content.encode("utf-8")).hexdigest()
@ -57,12 +76,16 @@ async def process_zip_file(zip_path: str, db_id: str) -> dict:
}
def process_zip_file_sync(zip_path: str, db_id: str) -> dict:
def process_zip_file_sync(
zip_path: str,
image_bucket: str = DEFAULT_IMAGE_BUCKET,
image_prefix: str = DEFAULT_IMAGE_PREFIX,
) -> dict:
"""同步调用 ZIP 处理，供同步解析器使用。"""
try:
asyncio.get_running_loop()
except RuntimeError:
return asyncio.run(process_zip_file(zip_path, db_id))
return asyncio.run(process_zip_file(zip_path, image_bucket=image_bucket, image_prefix=image_prefix))
result: dict | None = None
error: Exception | None = None
@ -70,7 +93,7 @@ def process_zip_file_sync(zip_path: str, db_id: str) -> dict:
def runner() -> None:
nonlocal result, error
try:
result = asyncio.run(process_zip_file(zip_path, db_id))
result = asyncio.run(process_zip_file(zip_path, image_bucket=image_bucket, image_prefix=image_prefix))
except Exception as exc: # pragma: no cover - pass through outer raise
error = exc
@ -106,7 +129,12 @@ def find_images_directory(zip_file: zipfile.ZipFile, md_file_path: str) -> str |
return None
async def process_images(zip_file: zipfile.ZipFile, images_dir: str, db_id: str, md_file_path: str) -> list[dict]:
async def process_images(
zip_file: zipfile.ZipFile,
images_dir: str,
image_bucket: str,
image_prefix: str,
) -> list[dict]:
"""处理图片：上传到MinIO并返回信息"""
supported_extensions = {".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp"}
content_type_map = {
@ -120,12 +148,10 @@ async def process_images(zip_file: zipfile.ZipFile, images_dir: str, db_id: str,
images = []
image_names = [n for n in zip_file.namelist() if n.startswith(images_dir + "/")]
normalized_prefix = _normalize_object_prefix(image_prefix)
minio_client = get_minio_client()
bucket_name = "kb-images"
await asyncio.to_thread(minio_client.ensure_bucket_exists, bucket_name)
file_id = hashstr(Path(md_file_path).name, length=16)
await asyncio.to_thread(minio_client.ensure_bucket_exists, image_bucket)
for img_name in image_names:
suffix = Path(img_name).suffix.lower()
@ -137,11 +163,11 @@ async def process_images(zip_file: zipfile.ZipFile, images_dir: str, db_id: str,
data = f.read()
timestamp = int(time.time() * 1000000)
object_name = f"{db_id}/{file_id}/images/{timestamp}_{Path(img_name).name}"
object_name = f"{normalized_prefix}/{timestamp}_{Path(img_name).name}"
content_type = content_type_map.get(suffix, "image/jpeg")
result = await minio_client.aupload_file(
bucket_name=bucket_name,
bucket_name=image_bucket,
object_name=object_name,
data=data,
content_type=content_type,

7
backend/package/yuxi/services/conversation_service.py
View File

@ -15,7 +15,7 @@ from yuxi.utils.datetime_utils import utc_isoformat
from yuxi.utils.logging_config import logger
# 附件存储桶名称
ATTACHMENTS_BUCKET = "chat-attachments"
ATTACHMENTS_BUCKET = "user"
async def require_user_conversation(conv_repo: ConversationRepository, thread_id: str, user_id: str):
@ -246,14 +246,15 @@ async def upload_thread_attachment_view(
file_content = await file.read()
await file.seek(0)
client = get_minio_client()
object_name = f"attachments/{thread_id}/{conversion.file_name}"
safe_filename = conversion.file_name.replace("/", "_").replace("\\", "_")
object_name = f"user/{current_user_id}/chat_attachments/{thread_id}/{safe_filename}"
result = client.upload_file(
bucket_name=ATTACHMENTS_BUCKET,
object_name=object_name,
data=file_content,
content_type=conversion.file_type or "application/octet-stream",
)
minio_url = result.public_url
minio_url = result.url
logger.info(f"Uploaded attachment to MinIO: {object_name}")
except Exception as e:
logger.error(f"Failed to upload attachment to MinIO: {e}")

18
backend/package/yuxi/storage/minio/client.py
View File

@ -41,25 +41,15 @@ class MinIOClient:
简化的 MinIO 客户端类
"""
PUBLIC_READ_BUCKETS = {"generated-images", "avatar", "kb-images"}
PUBLIC_READ_BUCKETS = {"public"}
# 知识库相关的 bucket 名称
KB_BUCKETS = {
"documents": "kb-documents",
"parsed": "kb-parsed",
"images": "kb-images",
"documents": "knowledgebases",
"parsed": "knowledgebases",
"images": "public",
}
@staticmethod
def get_ref_bucket_name(db_id: str) -> str:
"""获取 ref bucket 名称（截断32位）"""
return f"ref-{db_id[:32].replace('_', '-')}"
@staticmethod
def get_ref_bucket_name_full(db_id: str) -> str:
"""获取 ref bucket 名称（不截断）"""
return f"ref-{db_id.replace('_', '-')}"
def __init__(self):
"""初始化 MinIO 客户端"""
self.endpoint = os.getenv("MINIO_URI") or "http://milvus-minio:9000"

4
backend/server/routers/auth_router.py
View File

@ -757,8 +757,8 @@ async def upload_user_avatar(
file_extension = file.filename.split(".")[-1].lower() if file.filename and "." in file.filename else "jpg"
# 上传到MinIO
file_name = f"{uuid.uuid4()}.{file_extension}"
avatar_url = await aupload_file_to_minio("avatar", file_name, file_content, file_extension)
file_name = f"avatar/{current_user.id}/{uuid.uuid4()}.{file_extension}"
avatar_url = await aupload_file_to_minio("public", file_name, file_content, file_extension)
# 更新用户头像
current_user.avatar = avatar_url

48
backend/server/routers/knowledge_router.py
View File

@ -16,6 +16,7 @@ from yuxi import config, knowledge_base
from yuxi.knowledge.chunking.ragflow_like.presets import ensure_chunk_defaults_in_additional_params
from yuxi.knowledge.indexing import SUPPORTED_FILE_EXTENSIONS, is_supported_file_extension, process_file_to_markdown
from yuxi.knowledge.utils import calculate_content_hash
from yuxi.knowledge.utils.kb_utils import parse_minio_url
from yuxi.models.embed import test_all_embedding_models_status, test_embedding_model_status
from yuxi.storage.postgres.models_business import User
from yuxi.storage.minio.client import MinIOClient, StorageError, aupload_file_to_minio, get_minio_client
@ -680,15 +681,18 @@ async def batch_delete_documents(
deleted_count += 1
continue
file_name = file_meta_info.get("meta", {}).get("filename")
file_path = file_meta_info.get("meta", {}).get("path", "")
# 尝试从MinIO删除文件，如果失败（例如旧知识库没有MinIO实例），则忽略
# 尝试从 MinIO 删除文件对象与解析结果
try:
minio_client = get_minio_client()
await minio_client.adelete_file(MinIOClient.get_ref_bucket_name(db_id), file_name)
logger.debug(f"成功从MinIO删除文件: {file_name}")
if file_path.startswith(("http://", "https://")):
bucket_name, object_name = parse_minio_url(file_path)
await minio_client.adelete_file(bucket_name, object_name)
await minio_client.adelete_file(minio_client.KB_BUCKETS["parsed"], f"{db_id}/parsed/{doc_id}.md")
logger.debug(f"成功从MinIO删除文件: {file_path}")
except Exception as minio_error:
logger.warning(f"从MinIO删除文件失败（可能是旧知识库）: {minio_error}")
logger.warning(f"从MinIO删除文件失败: {minio_error}")
# 无论MinIO删除是否成功，都继续从知识库删除
await knowledge_base.delete_file(db_id, doc_id)
@ -723,17 +727,18 @@ async def delete_document(db_id: str, doc_id: str, current_user: User = Depends(
await knowledge_base.delete_folder(db_id, doc_id)
return {"message": "文件夹删除成功"}
file_name = file_meta_info.get("meta", {}).get("path", "").split("/")[-1]
file_path = file_meta_info.get("meta", {}).get("path", "")
# 尝试从MinIO删除文件，如果失败（例如旧知识库没有MinIO实例），则忽略
# 尝试从 MinIO 删除文件对象与解析结果
try:
minio_client = get_minio_client()
await minio_client.adelete_file(MinIOClient.get_ref_bucket_name(db_id), file_name)
# 同时删除 parsed bucket 中的 parsed.md 文件
await minio_client.adelete_file(minio_client.KB_BUCKETS["parsed"], f"{db_id}/{doc_id}/parsed.md")
logger.debug(f"成功从MinIO删除文件: {file_name}")
if file_path.startswith(("http://", "https://")):
bucket_name, object_name = parse_minio_url(file_path)
await minio_client.adelete_file(bucket_name, object_name)
await minio_client.adelete_file(minio_client.KB_BUCKETS["parsed"], f"{db_id}/parsed/{doc_id}.md")
logger.debug(f"成功从MinIO删除文件: {file_path}")
except Exception as minio_error:
logger.warning(f"从MinIO删除文件失败（可能是旧知识库）: {minio_error}")
logger.warning(f"从MinIO删除文件失败: {minio_error}")
# 无论MinIO删除是否成功，都继续从知识库删除
await knowledge_base.delete_file(db_id, doc_id)
@ -1240,13 +1245,11 @@ async def fetch_url(
# 3. 上传到 MinIO
minio_client = get_minio_client()
# 确保存储 bucket 存在
bucket_name = "kb-html-archives"
bucket_name = MinIOClient.KB_BUCKETS["documents"]
await asyncio.to_thread(minio_client.ensure_bucket_exists, bucket_name)
# 如果没有提供 db_id，使用 default
folder = db_id if db_id else "default"
object_name = f"{folder}/{content_hash}.html"
folder = db_id if db_id else "unknown"
object_name = f"{folder}/upload/{content_hash}.html"
upload_result = await minio_client.aupload_file(
bucket_name=bucket_name,
@ -1326,14 +1329,12 @@ async def upload_file(
timestamp = int(time.time() * 1000)
minio_filename = f"{basename}_{timestamp}{ext}"
# 生成符合MinIO规范的存储桶名称（将下划线替换为连字符，截取长度以满足MinIO的63字符限制）
if db_id:
bucket_name = f"ref-{db_id[:32].replace('_', '-')}"
else:
bucket_name = "default-uploads"
bucket_name = MinIOClient.KB_BUCKETS["documents"]
folder = db_id if db_id else "unknown"
object_name = f"{folder}/upload/{minio_filename}"
# 上传到MinIO
minio_url = await aupload_file_to_minio(bucket_name, minio_filename, file_bytes, ext.lstrip("."))
minio_url = await aupload_file_to_minio(bucket_name, object_name, file_bytes, ext.lstrip("."))
# 检测同名文件（基于原始文件名）
same_name_files = await knowledge_base.get_same_name_files(db_id, filename)
@ -1348,6 +1349,7 @@ async def upload_file(
"filename": filename, # 原始文件名（小写）
"original_filename": basename, # 原始文件名（去掉后缀）
"minio_filename": minio_filename, # MinIO中的文件名（带时间戳）
"object_name": object_name,
"bucket_name": bucket_name, # MinIO存储桶名称
"same_name_files": same_name_files, # 同名文件列表
"has_same_name": has_same_name, # 是否包含同名文件标志

45
backend/test/test_kb_minio_cleanup.py
View File

@ -24,6 +24,8 @@ async def test_delete_knowledge_base_cleanup():
# 初始化
minio_client = get_minio_client()
kb_manager = knowledge_base
doc_bucket = minio_client.KB_BUCKETS["documents"]
parsed_bucket = minio_client.KB_BUCKETS["parsed"]
# 测试参数
test_db_name = "test_cleanup_db"
@ -44,35 +46,35 @@ async def test_delete_knowledge_base_cleanup():
print("2. 上传测试文件到 MinIO...")
# 确保 bucket 存在
minio_client.ensure_bucket_exists("kb-documents")
minio_client.ensure_bucket_exists("kb-parsed")
minio_client.ensure_bucket_exists(doc_bucket)
minio_client.ensure_bucket_exists(parsed_bucket)
# 上传到 kb-documents (原始文件)
test_object_name = f"{db_id}/test_file_{db_id[:8]}.txt"
# 上传到知识库文档路径 (knowledgebases/{db_id}/upload/...)
test_object_name = f"{db_id}/upload/test_file_{db_id[:8]}.txt"
minio_client.upload_file(
bucket_name="kb-documents",
bucket_name=doc_bucket,
object_name=test_object_name,
data=test_file_content,
content_type="text/plain",
)
print(f" 文件上传到 kb-documents: {test_object_name}")
print(f" 文件上传到 {doc_bucket}: {test_object_name}")
# 上传到 kb-parsed (解析后的 markdown)
# 上传到知识库解析路径 (knowledgebases/{db_id}/parsed/...)
test_file_id = f"file_test_{db_id[:8]}"
parsed_object_name = f"{db_id}/{test_file_id}/parsed.md"
parsed_object_name = f"{db_id}/parsed/{test_file_id}.md"
parsed_content = b"# Test Parsed Content\n\nThis is parsed markdown."
minio_client.upload_file(
bucket_name="kb-parsed",
bucket_name=parsed_bucket,
object_name=parsed_object_name,
data=parsed_content,
content_type="text/markdown",
)
print(f" 文件上传到 kb-parsed: {parsed_object_name}")
print(f" 文件上传到 {parsed_bucket}: {parsed_object_name}")
# 3. 验证文件存在
print("3. 验证文件存在...")
assert minio_client.file_exists("kb-documents", test_object_name), "原始文件应该存在"
assert minio_client.file_exists("kb-parsed", parsed_object_name), "解析后文件应该存在"
assert minio_client.file_exists(doc_bucket, test_object_name), "原始文件应该存在"
assert minio_client.file_exists(parsed_bucket, parsed_object_name), "解析后文件应该存在"
print(" 文件存在验证通过")
# 4. 删除知识库
@ -83,13 +85,13 @@ async def test_delete_knowledge_base_cleanup():
# 5. 验证 MinIO 文件已清理
print("5. 验证 MinIO 文件已清理...")
# 检查 kb-documents 中的文件
doc_exists = minio_client.file_exists("kb-documents", test_object_name)
print(f" kb-documents/{test_object_name} 存在: {doc_exists}")
# 检查文档对象
doc_exists = minio_client.file_exists(doc_bucket, test_object_name)
print(f" {doc_bucket}/{test_object_name} 存在: {doc_exists}")
# 检查 kb-parsed 中的文件
parsed_exists = minio_client.file_exists("kb-parsed", parsed_object_name)
print(f" kb-parsed/{parsed_object_name} 存在: {parsed_exists}")
# 检查解析对象
parsed_exists = minio_client.file_exists(parsed_bucket, parsed_object_name)
print(f" {parsed_bucket}/{parsed_object_name} 存在: {parsed_exists}")
# 6. 输出测试结果
print("\n" + "=" * 50)
@ -99,9 +101,9 @@ async def test_delete_knowledge_base_cleanup():
else:
print("❌ 测试失败！以下文件未被清理:")
if doc_exists:
print(f" - kb-documents/{test_object_name}")
print(f" - {doc_bucket}/{test_object_name}")
if parsed_exists:
print(f" - kb-parsed/{parsed_object_name}")
print(f" - {parsed_bucket}/{parsed_object_name}")
return False
except Exception as e:
@ -116,9 +118,8 @@ async def test_delete_knowledge_base_cleanup():
print("\n清理测试数据...")
try:
# 尝试删除可能残留的文件（忽略错误）
for obj in [test_object_name, parsed_object_name]:
for bucket, obj in [(doc_bucket, test_object_name), (parsed_bucket, parsed_object_name)]:
try:
bucket = "kb-documents" if "kb-documents" in str(obj) else "kb-parsed"
await minio_client.adelete_file(bucket, obj)
except Exception:
pass