feat(database): 实现同名文件检测与MinIO存储支持
- 添加同名文件检测功能，当上传文件时检查知识库中是否存在同名文件并提示用户 - 重构文件存储逻辑，使用MinIO作为主要存储后端，支持本地和MinIO文件的统一处理 - 优化文件下载功能，根据文件路径类型自动选择本地或MinIO下载方式 - 新增同名文件列表展示组件，支持下载和删除操作 Co-authored-by: YuChuan <xerrors@qq.com>
This commit is contained in:
Wenjie Zhang
parent
6003563adb
commit
7750a5f106
@ -17,8 +17,8 @@ from src import config, knowledge_base
|
||||
from src.knowledge.indexing import SUPPORTED_FILE_EXTENSIONS, is_supported_file_extension, process_file_to_markdown
|
||||
from src.knowledge.utils import calculate_content_hash, merge_processing_params
|
||||
from src.models.embed import test_embedding_model_status, test_all_embedding_models_status
|
||||
from src.storage.minio.client import aupload_file_to_minio, get_minio_client
|
||||
from src.utils import hashstr, logger
|
||||
from src.storage.minio.client import aupload_file_to_minio, get_minio_client, StorageError
|
||||
from src.utils import logger
|
||||
|
||||
knowledge = APIRouter(prefix="/knowledge", tags=["knowledge"])
|
||||
|
||||
@ -328,17 +328,6 @@ async def add_documents(
|
||||
"failed": failed_count,
|
||||
}
|
||||
message = f"{item_type}处理完成，失败 {failed_count} 个" if failed_count else f"{item_type}处理完成"
|
||||
|
||||
for success_item in success_items:
|
||||
# 使用异步上传到minio的对应知识库,同名文件会被覆盖
|
||||
async with aiofiles.open(success_item["path"], "rb") as f:
|
||||
file_bytes = await f.read()
|
||||
# 上传的bucket名为ref-{refdb},refdb中的_替换为-
|
||||
refdb = db_id.replace("_", "-")
|
||||
url = await aupload_file_to_minio(
|
||||
f"ref-{refdb}", success_item["filename"], file_bytes, success_item["file_type"]
|
||||
)
|
||||
logger.info(f"上传文件成功: {url}")
|
||||
await context.set_result(summary | {"items": processed_items})
|
||||
await context.set_progress(100.0, message)
|
||||
return summary | {"items": processed_items}
|
||||
@ -536,12 +525,16 @@ async def rechunks_documents(
|
||||
|
||||
@knowledge.get("/databases/{db_id}/documents/{doc_id}/download")
|
||||
async def download_document(db_id: str, doc_id: str, request: Request, current_user: User = Depends(get_admin_user)):
|
||||
"""下载原始文件"""
|
||||
"""下载原始文件 - 根据path类型选择本地或MinIO下载"""
|
||||
logger.debug(f"Download document {doc_id} from {db_id}")
|
||||
try:
|
||||
file_info = await knowledge_base.get_file_basic_info(db_id, doc_id)
|
||||
# 获取文件扩展名和MIME类型，解码URL编码的文件名
|
||||
filename = file_info.get("meta", {}).get("filename", "file")
|
||||
file_meta = file_info.get("meta", {})
|
||||
|
||||
# 获取文件路径和文件名
|
||||
file_path = file_meta.get("path", "")
|
||||
filename = file_meta.get("filename", "file")
|
||||
logger.debug(f"File path from database: {file_path}")
|
||||
logger.debug(f"Original filename from database: {filename}")
|
||||
|
||||
# 解码URL编码的文件名（如果有的话）
|
||||
@ -553,45 +546,103 @@ async def download_document(db_id: str, doc_id: str, request: Request, current_u
|
||||
decoded_filename = filename # 如果解码失败，使用原文件名
|
||||
|
||||
_, ext = os.path.splitext(decoded_filename)
|
||||
|
||||
media_type = media_types.get(ext.lower(), "application/octet-stream")
|
||||
|
||||
minio_client = get_minio_client()
|
||||
minio_response = await minio_client.adownload_response(
|
||||
bucket_name="ref-" + db_id.replace("_", "-"),
|
||||
object_name=filename,
|
||||
)
|
||||
# 根据path类型选择下载方式
|
||||
from src.knowledge.utils.kb_utils import is_minio_url
|
||||
if is_minio_url(file_path):
|
||||
# MinIO下载
|
||||
logger.debug(f"Downloading from MinIO: {file_path}")
|
||||
|
||||
# 创建流式生成器
|
||||
async def minio_stream():
|
||||
try:
|
||||
while True:
|
||||
chunk = await asyncio.to_thread(minio_response.read, 8192)
|
||||
if not chunk:
|
||||
break
|
||||
yield chunk
|
||||
finally:
|
||||
minio_response.close()
|
||||
minio_response.release_conn()
|
||||
# 使用通用函数解析MinIO URL
|
||||
from src.knowledge.utils.kb_utils import parse_minio_url
|
||||
bucket_name, object_name = parse_minio_url(file_path)
|
||||
|
||||
# 创建StreamingResponse
|
||||
response = StreamingResponse(
|
||||
minio_stream(),
|
||||
media_type=media_type,
|
||||
)
|
||||
# 正确处理中文文件名的HTTP头部设置
|
||||
# HTTP头部只能包含ASCII字符，所以需要对中文文件名进行编码
|
||||
try:
|
||||
# 尝试使用ASCII编码（适用于英文文件名）
|
||||
decoded_filename.encode("ascii")
|
||||
# 如果成功，直接使用简单格式
|
||||
response.headers["Content-Disposition"] = f'attachment; filename="{decoded_filename}"'
|
||||
except UnicodeEncodeError:
|
||||
# 如果包含非ASCII字符（如中文），使用RFC 2231格式
|
||||
encoded_filename = quote(decoded_filename.encode("utf-8"))
|
||||
response.headers["Content-Disposition"] = f"attachment; filename*=UTF-8''{encoded_filename}"
|
||||
logger.debug(f"Parsed bucket_name: {bucket_name}, object_name: {object_name}")
|
||||
|
||||
minio_client = get_minio_client()
|
||||
|
||||
# 直接使用解析出的完整对象名称下载
|
||||
minio_response = await minio_client.adownload_response(
|
||||
bucket_name=bucket_name,
|
||||
object_name=object_name,
|
||||
)
|
||||
logger.debug(f"Successfully downloaded object: {object_name}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to download MinIO file: {e}")
|
||||
raise StorageError(f"下载文件失败: {e}")
|
||||
|
||||
# 创建流式生成器
|
||||
async def minio_stream():
|
||||
try:
|
||||
while True:
|
||||
chunk = await asyncio.to_thread(minio_response.read, 8192)
|
||||
if not chunk:
|
||||
break
|
||||
yield chunk
|
||||
finally:
|
||||
minio_response.close()
|
||||
minio_response.release_conn()
|
||||
|
||||
# 创建StreamingResponse
|
||||
response = StreamingResponse(
|
||||
minio_stream(),
|
||||
media_type=media_type,
|
||||
)
|
||||
# 正确处理中文文件名的HTTP头部设置
|
||||
try:
|
||||
# 尝试使用ASCII编码（适用于英文文件名）
|
||||
decoded_filename.encode("ascii")
|
||||
# 如果成功，直接使用简单格式
|
||||
response.headers["Content-Disposition"] = f'attachment; filename="{decoded_filename}"'
|
||||
except UnicodeEncodeError:
|
||||
# 如果包含非ASCII字符（如中文），使用RFC 2231格式
|
||||
encoded_filename = quote(decoded_filename.encode("utf-8"))
|
||||
response.headers["Content-Disposition"] = f"attachment; filename*=UTF-8''{encoded_filename}"
|
||||
|
||||
return response
|
||||
|
||||
else:
|
||||
# 本地文件下载
|
||||
logger.debug(f"Downloading from local filesystem: {file_path}")
|
||||
|
||||
if not os.path.exists(file_path):
|
||||
raise StorageError(f"文件不存在: {file_path}")
|
||||
|
||||
# 获取文件大小
|
||||
file_size = os.path.getsize(file_path)
|
||||
|
||||
# 创建文件流式生成器
|
||||
async def file_stream():
|
||||
async with aiofiles.open(file_path, "rb") as f:
|
||||
while True:
|
||||
chunk = await f.read(8192)
|
||||
if not chunk:
|
||||
break
|
||||
yield chunk
|
||||
|
||||
# 创建StreamingResponse
|
||||
response = StreamingResponse(
|
||||
file_stream(),
|
||||
media_type=media_type,
|
||||
)
|
||||
# 正确处理中文文件名的HTTP头部设置
|
||||
try:
|
||||
# 尝试使用ASCII编码（适用于英文文件名）
|
||||
decoded_filename.encode("ascii")
|
||||
# 如果成功，直接使用简单格式
|
||||
response.headers["Content-Disposition"] = f'attachment; filename="{decoded_filename}"'
|
||||
response.headers["Content-Length"] = str(file_size)
|
||||
except UnicodeEncodeError:
|
||||
# 如果包含非ASCII字符（如中文），使用RFC 2231格式
|
||||
encoded_filename = quote(decoded_filename.encode("utf-8"))
|
||||
response.headers["Content-Disposition"] = f"attachment; filename*=UTF-8''{encoded_filename}"
|
||||
response.headers["Content-Length"] = str(file_size)
|
||||
|
||||
return response
|
||||
|
||||
return response
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
@ -1068,19 +1119,9 @@ async def upload_file(
|
||||
elif not (is_supported_file_extension(file.filename) or ext == ".zip"):
|
||||
raise HTTPException(status_code=400, detail=f"Unsupported file type: {ext}")
|
||||
|
||||
# 根据db_id获取上传路径，如果db_id为None则使用默认路径
|
||||
if db_id:
|
||||
upload_dir = knowledge_base.get_db_upload_path(db_id)
|
||||
else:
|
||||
upload_dir = os.path.join(config.save_dir, "database", "uploads")
|
||||
|
||||
basename, ext = os.path.splitext(file.filename)
|
||||
filename = f"{basename}_{hashstr(basename, 4, with_salt=True, salt='fixed_salt')}{ext}".lower()
|
||||
|
||||
file_path = os.path.join(upload_dir, filename)
|
||||
|
||||
# 在线程池中执行同步文件系统操作，避免阻塞事件循环
|
||||
await asyncio.to_thread(os.makedirs, upload_dir, exist_ok=True)
|
||||
# 直接使用原始文件名（小写）
|
||||
filename = f"{basename}{ext}".lower()
|
||||
|
||||
file_bytes = await file.read()
|
||||
|
||||
@ -1093,15 +1134,36 @@ async def upload_file(
|
||||
detail="数据库中已经存在了相同内容文件，File with the same content already exists in this database",
|
||||
)
|
||||
|
||||
# 使用异步文件写入，避免阻塞事件循环
|
||||
async with aiofiles.open(file_path, "wb") as buffer:
|
||||
await buffer.write(file_bytes)
|
||||
# 直接上传到MinIO，添加时间戳区分版本
|
||||
import time
|
||||
timestamp = int(time.time() * 1000)
|
||||
minio_filename = f"{basename}_{timestamp}{ext}"
|
||||
|
||||
# 生成符合MinIO规范的存储桶名称（将下划线替换为连字符）
|
||||
if db_id:
|
||||
bucket_name = f"ref-{db_id.replace('_', '-')}"
|
||||
else:
|
||||
bucket_name = "default-uploads"
|
||||
|
||||
# 上传到MinIO
|
||||
minio_url = await aupload_file_to_minio(bucket_name, minio_filename, file_bytes, ext.lstrip('.'))
|
||||
|
||||
# 检测同名文件（基于原始文件名）
|
||||
same_name_files = await knowledge_base.get_same_name_files(db_id, filename)
|
||||
has_same_name = len(same_name_files) > 0
|
||||
|
||||
return {
|
||||
"message": "File successfully uploaded",
|
||||
"file_path": file_path,
|
||||
"file_path": minio_url, # MinIO路径作为主要路径
|
||||
"minio_path": minio_url, # MinIO路径
|
||||
"db_id": db_id,
|
||||
"content_hash": content_hash,
|
||||
"filename": filename, # 原始文件名（小写）
|
||||
"original_filename": basename, # 原始文件名（去掉后缀）
|
||||
"minio_filename": minio_filename, # MinIO中的文件名（带时间戳）
|
||||
"bucket_name": bucket_name, # MinIO存储桶名称
|
||||
"same_name_files": same_name_files, # 同名文件列表
|
||||
"has_same_name": has_same_name # 是否包含同名文件标志
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -388,10 +388,10 @@ async def parse_image_async(file, params=None):
|
||||
|
||||
async def process_file_to_markdown(file_path: str, params: dict | None = None) -> str:
|
||||
"""
|
||||
将不同类型的文件转换为markdown格式
|
||||
将不同类型的文件转换为markdown格式 - 支持本地文件和MinIO文件
|
||||
|
||||
Args:
|
||||
file_path: 文件路径
|
||||
file_path: 文件路径或MinIO URL
|
||||
params: 处理参数，对于ZIP文件需要包含 db_id
|
||||
|
||||
Returns:
|
||||
@ -402,105 +402,161 @@ async def process_file_to_markdown(file_path: str, params: dict | None = None) -
|
||||
- params['_zip_images_info']: 图片信息列表
|
||||
- params['_zip_content_hash']: 内容哈希值
|
||||
"""
|
||||
file_path_obj = Path(file_path)
|
||||
file_ext = file_path_obj.suffix.lower()
|
||||
import tempfile
|
||||
import aiofiles
|
||||
import os
|
||||
|
||||
if file_ext == ".pdf":
|
||||
# 使用 OCR 处理 PDF
|
||||
text = await parse_pdf_async(str(file_path_obj), params=params)
|
||||
return f"# {file_path_obj.name}\n\n{text}"
|
||||
# 检测是否是MinIO URL
|
||||
from src.knowledge.utils.kb_utils import is_minio_url
|
||||
|
||||
elif file_ext in [".txt", ".md"]:
|
||||
# 直接读取文本文件
|
||||
with open(file_path_obj, encoding="utf-8") as f:
|
||||
content = f.read()
|
||||
return f"# {file_path_obj.name}\n\n{content}"
|
||||
if is_minio_url(file_path):
|
||||
# 从MinIO下载文件到临时位置
|
||||
logger.debug(f"Downloading file from MinIO: {file_path}")
|
||||
|
||||
elif file_ext == ".docx":
|
||||
text = _extract_docx_markdown_with_images(file_path_obj, params=params)
|
||||
return f"# {file_path_obj.name}\n\n" + text
|
||||
# 从MinIO URL中提取文件名
|
||||
if "?" in file_path:
|
||||
file_path_clean = file_path.split("?")[0]
|
||||
else:
|
||||
file_path_clean = file_path
|
||||
|
||||
elif file_ext == ".doc":
|
||||
text = _extract_word_text(file_path_obj)
|
||||
return f"# {file_path_obj.name}\n\n{text}"
|
||||
original_filename = file_path_clean.split("/")[-1]
|
||||
|
||||
elif file_ext in [".jpg", ".jpeg", ".png", ".bmp", ".tiff", ".tif"]:
|
||||
# 使用 OCR 处理图片
|
||||
text = await parse_image_async(str(file_path_obj), params=params)
|
||||
return f"# {file_path_obj.name}\n\n{text}"
|
||||
# 创建临时文件
|
||||
with tempfile.NamedTemporaryFile(delete=False, suffix=Path(original_filename).suffix) as temp_file:
|
||||
temp_path = temp_file.name
|
||||
|
||||
elif file_ext in [".html", ".htm"]:
|
||||
# 使用 BeautifulSoup 处理 HTML 文件
|
||||
from markdownify import markdownify as md
|
||||
try:
|
||||
# 使用通用函数解析MinIO URL并下载文件
|
||||
from src.knowledge.utils.kb_utils import parse_minio_url
|
||||
from src.storage.minio.client import get_minio_client
|
||||
|
||||
with open(file_path_obj, encoding="utf-8") as f:
|
||||
content = f.read()
|
||||
text = md(content, heading_style="ATX")
|
||||
return f"# {file_path_obj.name}\n\n{text}"
|
||||
# 解析MinIO URL获取bucket_name和object_name
|
||||
bucket_name, object_name = parse_minio_url(file_path)
|
||||
|
||||
elif file_ext == ".csv":
|
||||
# 处理 CSV 文件
|
||||
import pandas as pd
|
||||
# 获取MinIO客户端并下载文件
|
||||
minio_client = get_minio_client()
|
||||
file_content = await minio_client.adownload_file(bucket_name, object_name)
|
||||
|
||||
df = pd.read_csv(file_path_obj)
|
||||
# 将每一行数据与表头组合成独立的表格
|
||||
markdown_content = f"# {file_path_obj.name}\n\n"
|
||||
# 写入临时文件
|
||||
async with aiofiles.open(temp_path, "wb") as f:
|
||||
await f.write(file_content)
|
||||
|
||||
for index, row in df.iterrows():
|
||||
# 创建包含表头和当前行的小表格
|
||||
row_df = pd.DataFrame([row], columns=df.columns)
|
||||
markdown_table = row_df.to_markdown(index=False)
|
||||
markdown_content += f"{markdown_table}\n\n"
|
||||
logger.debug(f"File downloaded to temp path: {temp_path}")
|
||||
|
||||
return markdown_content.strip()
|
||||
# 使用临时文件路径
|
||||
actual_file_path = temp_path
|
||||
|
||||
elif file_ext in [".xls", ".xlsx"]:
|
||||
# 处理 Excel 文件
|
||||
import pandas as pd
|
||||
from openpyxl import load_workbook
|
||||
except Exception as e:
|
||||
# 清理临时文件
|
||||
if os.path.exists(temp_path):
|
||||
os.unlink(temp_path)
|
||||
logger.error(f"Failed to download file from MinIO: {e}")
|
||||
raise ValueError(f"无法从MinIO下载文件: {e}")
|
||||
else:
|
||||
# 本地文件
|
||||
actual_file_path = file_path
|
||||
|
||||
markdown_content = f"# {file_path_obj.name}\n\n"
|
||||
try:
|
||||
file_path_obj = Path(actual_file_path)
|
||||
file_ext = file_path_obj.suffix.lower()
|
||||
original_filename = file_path_obj.name
|
||||
|
||||
# 使用 openpyxl 加载工作簿以正确处理合并单元格
|
||||
wb = load_workbook(file_path_obj, data_only=True)
|
||||
if file_ext == ".pdf":
|
||||
# 使用 OCR 处理 PDF
|
||||
text = await parse_pdf_async(str(file_path_obj), params=params)
|
||||
result = f"{text}"
|
||||
|
||||
for sheet_name in wb.sheetnames:
|
||||
ws = wb[sheet_name]
|
||||
elif file_ext in [".txt", ".md"]:
|
||||
# 直接读取文本文件
|
||||
with open(file_path_obj, encoding="utf-8") as f:
|
||||
content = f.read()
|
||||
result = f"{content}"
|
||||
|
||||
# 先取消所有合并单元格，并填充值
|
||||
merged_ranges = list(ws.merged_cells.ranges)
|
||||
elif file_ext == ".docx":
|
||||
text = _extract_docx_markdown_with_images(file_path_obj, params=params)
|
||||
result = f"" + text
|
||||
|
||||
for merged_range in merged_ranges:
|
||||
# 获取合并区域左上角单元格的值
|
||||
min_row, min_col, max_row, max_col = (
|
||||
merged_range.min_row,
|
||||
merged_range.min_col,
|
||||
merged_range.max_row,
|
||||
merged_range.max_col,
|
||||
)
|
||||
elif file_ext == ".doc":
|
||||
text = _extract_word_text(file_path_obj)
|
||||
result = f"{text}"
|
||||
|
||||
# 获取左上角单元格的值
|
||||
top_left_value = ws.cell(row=min_row, column=min_col).value
|
||||
elif file_ext in [".jpg", ".jpeg", ".png", ".bmp", ".tiff", ".tif"]:
|
||||
# 使用 OCR 处理图片
|
||||
text = await parse_image_async(str(file_path_obj), params=params)
|
||||
result = f"{text}"
|
||||
|
||||
# 取消合并
|
||||
ws.unmerge_cells(start_row=min_row, start_column=min_col, end_row=max_row, end_column=max_col)
|
||||
elif file_ext in [".html", ".htm"]:
|
||||
# 使用 BeautifulSoup 处理 HTML 文件
|
||||
from markdownify import markdownify as md
|
||||
|
||||
# 在所有原合并单元格区域填充值
|
||||
for row in range(min_row, max_row + 1):
|
||||
for col in range(min_col, max_col + 1):
|
||||
ws.cell(row=row, column=col).value = top_left_value
|
||||
with open(file_path_obj, encoding="utf-8") as f:
|
||||
content = f.read()
|
||||
text = md(content, heading_style="ATX")
|
||||
result = f"{text}"
|
||||
|
||||
# 转换为DataFrame
|
||||
data = []
|
||||
for row in ws.iter_rows(values_only=True):
|
||||
data.append(row)
|
||||
elif file_ext == ".csv":
|
||||
# 处理 CSV 文件
|
||||
import pandas as pd
|
||||
|
||||
# 第一行作为列名
|
||||
columns = data[0] if data else []
|
||||
df_data = data[1:] if len(data) > 1 else []
|
||||
df = pd.read_csv(file_path_obj)
|
||||
# 将每一行数据与表头组合成独立的表格
|
||||
markdown_content = f""
|
||||
|
||||
# 处理重复的列名,给重复的列添加后缀
|
||||
columns = _make_unique_columns(columns)
|
||||
for index, row in df.iterrows():
|
||||
# 创建包含表头和当前行的小表格
|
||||
row_df = pd.DataFrame([row], columns=df.columns)
|
||||
markdown_table = row_df.to_markdown(index=False)
|
||||
markdown_content += f"{markdown_table}\n\n"
|
||||
|
||||
result = markdown_content.strip()
|
||||
|
||||
elif file_ext in [".xls", ".xlsx"]:
|
||||
# 处理 Excel 文件
|
||||
import pandas as pd
|
||||
from openpyxl import load_workbook
|
||||
|
||||
markdown_content = f""
|
||||
|
||||
# 使用 openpyxl 加载工作簿以正确处理合并单元格
|
||||
wb = load_workbook(file_path_obj, data_only=True)
|
||||
|
||||
for sheet_name in wb.sheetnames:
|
||||
ws = wb[sheet_name]
|
||||
|
||||
# 先取消所有合并单元格，并填充值
|
||||
merged_ranges = list(ws.merged_cells.ranges)
|
||||
|
||||
for merged_range in merged_ranges:
|
||||
# 获取合并区域左上角单元格的值
|
||||
min_row, min_col, max_row, max_col = (
|
||||
merged_range.min_row,
|
||||
merged_range.min_col,
|
||||
merged_range.max_row,
|
||||
merged_range.max_col,
|
||||
)
|
||||
|
||||
# 获取左上角单元格的值
|
||||
top_left_value = ws.cell(row=min_row, column=min_col).value
|
||||
|
||||
# 取消合并
|
||||
ws.unmerge_cells(start_row=min_row, start_column=min_col, end_row=max_row, end_column=max_col)
|
||||
|
||||
# 在所有原合并单元格区域填充值
|
||||
for row in range(min_row, max_row + 1):
|
||||
for col in range(min_col, max_col + 1):
|
||||
ws.cell(row=row, column=col).value = top_left_value
|
||||
|
||||
# 转换为DataFrame
|
||||
data = []
|
||||
for row in ws.iter_rows(values_only=True):
|
||||
data.append(row)
|
||||
|
||||
# 第一行作为列名
|
||||
columns = data[0] if data else []
|
||||
df_data = data[1:] if len(data) > 1 else []
|
||||
|
||||
# 处理重复的列名,给重复的列添加后缀
|
||||
columns = _make_unique_columns(columns)
|
||||
|
||||
df = pd.DataFrame(df_data, columns=columns)
|
||||
|
||||
@ -519,34 +575,55 @@ async def process_file_to_markdown(file_path: str, params: dict | None = None) -
|
||||
markdown_table = chunk_df.to_markdown(index=False)
|
||||
markdown_content += f"{markdown_table}\n\n"
|
||||
|
||||
return markdown_content.strip()
|
||||
result = markdown_content.strip()
|
||||
|
||||
elif file_ext == ".json":
|
||||
# 处理 JSON 文件
|
||||
import json
|
||||
elif file_ext == ".json":
|
||||
# 处理 JSON 文件
|
||||
import json
|
||||
|
||||
async with aiofiles.open(file_path_obj, encoding="utf-8") as f:
|
||||
content = await f.read()
|
||||
data = json.loads(content)
|
||||
# 将 JSON 数据格式化为 markdown 代码块
|
||||
json_str = json.dumps(data, ensure_ascii=False, indent=2)
|
||||
return f"# {file_path_obj.name}\n\n```json\n{json_str}\n```"
|
||||
async with aiofiles.open(file_path_obj, encoding="utf-8") as f:
|
||||
content = await f.read()
|
||||
data = json.loads(content)
|
||||
# 将 JSON 数据格式化为 markdown 代码块
|
||||
json_str = json.dumps(data, ensure_ascii=False, indent=2)
|
||||
result = f"```json\n{json_str}\n```"
|
||||
|
||||
elif file_ext == ".zip":
|
||||
if not params or "db_id" not in params:
|
||||
raise ValueError("ZIP文件处理需要在params中提供db_id参数")
|
||||
elif file_ext == ".zip":
|
||||
if not params or "db_id" not in params:
|
||||
raise ValueError("ZIP文件处理需要在params中提供db_id参数")
|
||||
|
||||
result = await asyncio.to_thread(_process_zip_file, str(file_path_obj), params["db_id"])
|
||||
zip_result = await asyncio.to_thread(_process_zip_file, str(file_path_obj), params["db_id"])
|
||||
|
||||
# 将处理结果保存到params中供调用方使用
|
||||
params["_zip_images_info"] = result["images_info"]
|
||||
params["_zip_content_hash"] = result["content_hash"]
|
||||
# 将处理结果保存到params中供调用方使用
|
||||
params["_zip_images_info"] = zip_result["images_info"]
|
||||
params["_zip_content_hash"] = zip_result["content_hash"]
|
||||
|
||||
return result["markdown_content"]
|
||||
result = zip_result["markdown_content"]
|
||||
|
||||
else:
|
||||
# 尝试作为文本文件读取
|
||||
raise ValueError(f"Unsupported file type: {file_ext}")
|
||||
else:
|
||||
# 尝试作为文本文件读取
|
||||
raise ValueError(f"Unsupported file type: {file_ext}")
|
||||
|
||||
except Exception as e:
|
||||
# 清理临时文件
|
||||
if is_minio_url(file_path) and os.path.exists(actual_file_path):
|
||||
try:
|
||||
os.unlink(actual_file_path)
|
||||
logger.debug(f"Cleaned up temp file: {actual_file_path}")
|
||||
except Exception as cleanup_e:
|
||||
logger.warning(f"Failed to clean up temp file {actual_file_path}: {cleanup_e}")
|
||||
raise
|
||||
|
||||
finally:
|
||||
# 清理临时文件
|
||||
if is_minio_url(file_path) and os.path.exists(actual_file_path):
|
||||
try:
|
||||
os.unlink(actual_file_path)
|
||||
logger.debug(f"Cleaned up temp file: {actual_file_path}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to clean up temp file {actual_file_path}: {e}")
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _process_zip_file(zip_path: str, db_id: str) -> dict:
|
||||
|
||||
@ -374,6 +374,52 @@ class KnowledgeBaseManager:
|
||||
|
||||
return False
|
||||
|
||||
async def get_same_name_files(self, db_id: str, filename: str) -> list[dict]:
|
||||
"""获取同一知识库中同名文件列表
|
||||
基于原始文件名直接比较
|
||||
返回基础信息：文件名、大小、上传时间
|
||||
|
||||
Args:
|
||||
db_id: 数据库ID
|
||||
filename: 要检测的文件名（原始文件名）
|
||||
|
||||
Returns:
|
||||
同名文件列表，每项包含：
|
||||
- filename: 文件名
|
||||
- size: 文件大小
|
||||
- created_at: 上传时间
|
||||
- file_id: 文件ID（用于下载）
|
||||
"""
|
||||
if not db_id or not filename:
|
||||
return []
|
||||
try:
|
||||
kb_instance = self._get_kb_for_database(db_id)
|
||||
except KBNotFoundError:
|
||||
return []
|
||||
|
||||
same_name_files = []
|
||||
for file_id, file_info in kb_instance.files_meta.items():
|
||||
if file_info.get("database_id") != db_id:
|
||||
continue
|
||||
if file_info.get("status") == "failed":
|
||||
continue
|
||||
|
||||
# 直接比较文件名（现在就是原始文件名）
|
||||
current_filename = file_info.get("filename", "")
|
||||
|
||||
if current_filename.lower() == filename.lower():
|
||||
same_name_files.append({
|
||||
"file_id": file_id,
|
||||
"filename": current_filename,
|
||||
"size": file_info.get("size", 0),
|
||||
"created_at": file_info.get("created_at", ""),
|
||||
"content_hash": file_info.get("content_hash", "")
|
||||
})
|
||||
|
||||
# 按上传时间降序排序
|
||||
same_name_files.sort(key=lambda x: x.get("created_at", ""), reverse=True)
|
||||
return same_name_files
|
||||
|
||||
async def update_file(self, db_id: str, region_file_id: str, file_name: str, params: dict | None = None) -> dict:
|
||||
"""对单个文件执行更新"""
|
||||
kb_instance = self._get_kb_for_database(db_id)
|
||||
|
||||
@ -13,10 +13,10 @@ from src.utils.datetime_utils import utc_isoformat
|
||||
|
||||
def validate_file_path(file_path: str, db_id: str = None) -> str:
|
||||
"""
|
||||
验证文件路径安全性，防止路径遍历攻击
|
||||
验证文件路径安全性，防止路径遍历攻击 - 支持本地文件和MinIO URL
|
||||
|
||||
Args:
|
||||
file_path: 要验证的文件路径
|
||||
file_path: 要验证的文件路径或MinIO URL
|
||||
db_id: 数据库ID，用于获取知识库特定的上传目录
|
||||
|
||||
Returns:
|
||||
@ -26,7 +26,12 @@ def validate_file_path(file_path: str, db_id: str = None) -> str:
|
||||
ValueError: 如果路径不安全
|
||||
"""
|
||||
try:
|
||||
# 规范化路径
|
||||
# 检测是否是MinIO URL，如果是则直接返回（不进行路径遍历检查）
|
||||
if is_minio_url(file_path):
|
||||
logger.debug(f"MinIO URL detected, skipping path validation: {file_path}")
|
||||
return file_path
|
||||
|
||||
# 规范化路径（仅对本地文件）
|
||||
normalized_path = os.path.abspath(os.path.realpath(file_path))
|
||||
|
||||
# 获取允许的根目录
|
||||
@ -134,32 +139,71 @@ async def calculate_content_hash(data: bytes | bytearray | str | os.PathLike[str
|
||||
|
||||
async def prepare_item_metadata(item: str, content_type: str, db_id: str, params: dict | None = None) -> dict:
|
||||
"""
|
||||
准备文件或URL的元数据
|
||||
准备文件或URL的元数据 - 支持本地文件和MinIO文件
|
||||
|
||||
Args:
|
||||
item: 文件路径或URL
|
||||
item: 文件路径或MinIO URL
|
||||
content_type: 内容类型 ("file" 或 "url")
|
||||
db_id: 数据库ID
|
||||
params: 处理参数，可选
|
||||
"""
|
||||
if content_type == "file":
|
||||
file_path = Path(item)
|
||||
file_id = f"file_{hashstr(str(file_path) + str(time.time()), 6)}"
|
||||
file_type = file_path.suffix.lower().replace(".", "")
|
||||
filename = file_path.name
|
||||
item_path = os.path.relpath(file_path, Path.cwd())
|
||||
content_hash = None
|
||||
try:
|
||||
if file_path.exists():
|
||||
content_hash = await calculate_content_hash(file_path)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning(f"Failed to calculate content hash for {file_path}: {exc}")
|
||||
# 检测是否是MinIO URL还是本地文件路径
|
||||
if is_minio_url(item):
|
||||
# MinIO文件处理
|
||||
logger.debug(f"Processing MinIO file: {item}")
|
||||
# 从MinIO URL中提取文件名
|
||||
if "?" in item:
|
||||
# URL可能包含查询参数，去掉它们
|
||||
item_clean = item.split("?")[0]
|
||||
else:
|
||||
item_clean = item
|
||||
|
||||
# 获取文件名（从路径的最后部分）
|
||||
filename = item_clean.split("/")[-1]
|
||||
|
||||
# 如果文件名包含时间戳，提取原始文件名
|
||||
import re
|
||||
timestamp_pattern = r'^(.+)_(\d{13})(\.[^.]+)$'
|
||||
match = re.match(timestamp_pattern, filename)
|
||||
if match:
|
||||
original_filename = match.group(1) + match.group(3)
|
||||
# 存储原始文件名用于显示
|
||||
filename_display = original_filename
|
||||
else:
|
||||
filename_display = filename
|
||||
|
||||
file_type = filename.split(".")[-1].lower() if "." in filename else ""
|
||||
item_path = item # 保持MinIO URL作为路径
|
||||
|
||||
# 从URL或params中获取content_hash（如果有的话）
|
||||
content_hash = None
|
||||
if params and "content_hash" in params:
|
||||
content_hash = params["content_hash"]
|
||||
|
||||
else:
|
||||
# 本地文件处理
|
||||
file_path = Path(item)
|
||||
file_type = file_path.suffix.lower().replace(".", "")
|
||||
filename = file_path.name
|
||||
filename_display = filename
|
||||
item_path = os.path.relpath(file_path, Path.cwd())
|
||||
content_hash = None
|
||||
try:
|
||||
if file_path.exists():
|
||||
content_hash = await calculate_content_hash(file_path)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning(f"Failed to calculate content hash for {file_path}: {exc}")
|
||||
|
||||
# 生成文件ID
|
||||
file_id = f"file_{hashstr(str(item_path) + str(time.time()), 6)}"
|
||||
|
||||
else:
|
||||
raise ValueError("URL 元数据生成已禁用")
|
||||
|
||||
metadata = {
|
||||
"database_id": db_id,
|
||||
"filename": filename,
|
||||
"filename": filename_display, # 使用显示用的文件名
|
||||
"path": item_path,
|
||||
"file_type": file_type,
|
||||
"status": "processing",
|
||||
@ -283,3 +327,54 @@ def get_embedding_config(embed_info: dict) -> dict:
|
||||
|
||||
logger.debug(f"Embedding config: {config_dict}")
|
||||
return config_dict
|
||||
|
||||
|
||||
def is_minio_url(file_path: str) -> bool:
|
||||
"""
|
||||
检测是否是MinIO URL
|
||||
|
||||
Args:
|
||||
file_path: 文件路径或URL
|
||||
|
||||
Returns:
|
||||
bool: 是否是MinIO URL
|
||||
"""
|
||||
return file_path.startswith(("http://", "https://", "s3://")) or "minio" in file_path.lower()
|
||||
|
||||
|
||||
def parse_minio_url(file_path: str) -> tuple[str, str]:
|
||||
"""
|
||||
解析MinIO URL，提取bucket名称和对象名称
|
||||
|
||||
Args:
|
||||
file_path: MinIO文件URL
|
||||
|
||||
Returns:
|
||||
tuple[str, str]: (bucket_name, object_name)
|
||||
|
||||
Raises:
|
||||
ValueError: 如果无法解析URL
|
||||
"""
|
||||
try:
|
||||
from urllib.parse import urlparse
|
||||
|
||||
# 解析URL
|
||||
parsed_url = urlparse(file_path)
|
||||
|
||||
# 从URL路径中提取对象名称（去掉开头的斜杠）
|
||||
object_name = parsed_url.path.lstrip('/')
|
||||
|
||||
# 分离bucket名称和对象名称
|
||||
path_parts = object_name.split('/', 1)
|
||||
if len(path_parts) > 1:
|
||||
bucket_name = path_parts[0]
|
||||
object_name = path_parts[1]
|
||||
else:
|
||||
raise ValueError(f"无法解析MinIO URL中的bucket名称: {file_path}")
|
||||
|
||||
logger.debug(f"Parsed MinIO URL: bucket_name={bucket_name}, object_name={object_name}")
|
||||
return bucket_name, object_name
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to parse MinIO URL {file_path}: {e}")
|
||||
raise ValueError(f"无法解析MinIO URL: {file_path}")
|
||||
|
||||
@ -565,8 +565,8 @@ const resetOnGoingConv = (threadId = null, preserveMessages = false) => {
|
||||
if (preserveMessages) {
|
||||
setTimeout(() => {
|
||||
if (threadState.onGoingConv) {
|
||||
threadState.onGoingConv = createOnGoingConvState();
|
||||
}
|
||||
threadState.onGoingConv = createOnGoingConvState();
|
||||
}
|
||||
}, 100);
|
||||
} else {
|
||||
threadState.onGoingConv = createOnGoingConvState();
|
||||
|
||||
@ -696,7 +696,7 @@ watch(() => props.isOpen, (newVal) => {
|
||||
}
|
||||
|
||||
.config-item {
|
||||
background-color: var(--gray-50);
|
||||
background-color: var(--gray-25);
|
||||
padding: 12px;
|
||||
border-radius: 8px;
|
||||
box-shadow: 0px 1px 1px var(--shadow-2);
|
||||
|
||||
@ -533,7 +533,7 @@ import ChunkParamsConfig from '@/components/ChunkParamsConfig.vue';
|
||||
overflow: hidden;
|
||||
border-radius: 12px;
|
||||
border: 1px solid var(--gray-150);
|
||||
padding-top: 6px;
|
||||
/* padding-top: 6px; */
|
||||
}
|
||||
|
||||
.panel-header {
|
||||
@ -542,7 +542,7 @@ import ChunkParamsConfig from '@/components/ChunkParamsConfig.vue';
|
||||
align-items: center;
|
||||
margin-bottom: 4px;
|
||||
flex-shrink: 0;
|
||||
padding: 4px 2px;
|
||||
padding: 4px 4px;
|
||||
}
|
||||
|
||||
.search-container {
|
||||
|
||||
@ -102,7 +102,32 @@
|
||||
</a-upload-dragger>
|
||||
</div>
|
||||
|
||||
|
||||
<!-- 同名文件提示 -->
|
||||
<div v-if="sameNameFiles.length > 0" class="same-name-files-section">
|
||||
<div class="same-name-files-header">
|
||||
<InfoCircleOutlined />
|
||||
<span>当前知识库中已存在以下同名文件：</span>
|
||||
</div>
|
||||
<div class="same-name-files-list">
|
||||
<div v-for="file in sameNameFiles" :key="file.file_id" class="same-name-file-item">
|
||||
<div class="same-name-file-info">
|
||||
<span class="same-name-file-name">{{ file.filename }}</span>
|
||||
<span class="same-name-file-time">{{ formatFileTime(file.created_at) }}</span>
|
||||
</div>
|
||||
<div class="same-name-file-actions">
|
||||
<a-button size="small" type="link" class="download-btn" @click="downloadSameNameFile(file)">
|
||||
<template #icon><DownloadOutlined /></template>
|
||||
下载
|
||||
</a-button>
|
||||
<a-button size="small" type="link" danger @click="deleteSameNameFile(file)">
|
||||
<template #icon><DeleteOutlined /></template>
|
||||
删除
|
||||
</a-button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</a-modal>
|
||||
|
||||
@ -123,18 +148,20 @@
|
||||
|
||||
<script setup>
|
||||
import { ref, computed, onMounted, watch } from 'vue';
|
||||
import { message, Upload, Tooltip } from 'ant-design-vue';
|
||||
import { message, Upload, Tooltip, Modal } from 'ant-design-vue';
|
||||
import { useUserStore } from '@/stores/user';
|
||||
import { useDatabaseStore } from '@/stores/database';
|
||||
import { ocrApi } from '@/apis/system_api';
|
||||
import { fileApi } from '@/apis/knowledge_api';
|
||||
import { fileApi, documentApi } from '@/apis/knowledge_api';
|
||||
import ChunkParamsConfig from '@/components/ChunkParamsConfig.vue';
|
||||
import {
|
||||
FileOutlined,
|
||||
LinkOutlined,
|
||||
SettingOutlined,
|
||||
CheckCircleOutlined,
|
||||
ExclamationCircleOutlined,
|
||||
InfoCircleOutlined,
|
||||
DownloadOutlined,
|
||||
DeleteOutlined,
|
||||
} from '@ant-design/icons-vue';
|
||||
import { h } from 'vue';
|
||||
|
||||
@ -265,6 +292,9 @@ const uploadModeOptions = computed(() => [
|
||||
// 文件列表
|
||||
const fileList = ref([]);
|
||||
|
||||
// 同名文件列表（用于显示提示）
|
||||
const sameNameFiles = ref([]);
|
||||
|
||||
|
||||
// URL相关功能已移除
|
||||
|
||||
@ -507,11 +537,109 @@ const beforeUpload = (file) => {
|
||||
return true;
|
||||
};
|
||||
|
||||
const formatFileSize = (bytes) => {
|
||||
if (bytes === 0 || !bytes) return '0 B';
|
||||
const k = 1024;
|
||||
const sizes = ['B', 'KB', 'MB', 'GB'];
|
||||
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
||||
return `${parseFloat((bytes / Math.pow(k, i)).toFixed(1))} ${sizes[i]}`;
|
||||
};
|
||||
|
||||
const formatFileTime = (timestamp) => {
|
||||
if (!timestamp) return '';
|
||||
try {
|
||||
const date = new Date(timestamp);
|
||||
return date.toLocaleString();
|
||||
} catch (e) {
|
||||
return timestamp;
|
||||
}
|
||||
};
|
||||
|
||||
const showSameNameFilesInUploadArea = (files) => {
|
||||
sameNameFiles.value = files;
|
||||
// 可以在这里添加其他逻辑，比如自动滚动到提示区域
|
||||
};
|
||||
|
||||
const downloadSameNameFile = async (file) => {
|
||||
try {
|
||||
// 获取当前数据库ID
|
||||
const currentDbId = databaseId.value;
|
||||
if (!currentDbId) {
|
||||
message.error('知识库ID不存在');
|
||||
return;
|
||||
}
|
||||
|
||||
message.loading('正在下载文件...', 0);
|
||||
const response = await documentApi.downloadDocument(currentDbId, file.file_id);
|
||||
message.destroy();
|
||||
|
||||
// 创建下载链接
|
||||
const blob = await response.blob(); // 从 Response 对象中提取 Blob 数据
|
||||
const url = window.URL.createObjectURL(blob);
|
||||
const link = document.createElement('a');
|
||||
link.href = url;
|
||||
link.download = file.filename;
|
||||
document.body.appendChild(link);
|
||||
link.click();
|
||||
document.body.removeChild(link);
|
||||
window.URL.revokeObjectURL(url);
|
||||
|
||||
message.success(`文件 ${file.filename} 下载成功`);
|
||||
} catch (error) {
|
||||
message.destroy();
|
||||
console.error('下载文件失败:', error);
|
||||
message.error(`下载文件失败: ${error.message || '未知错误'}`);
|
||||
}
|
||||
};
|
||||
|
||||
const deleteSameNameFile = (file) => {
|
||||
Modal.confirm({
|
||||
title: '确认删除文件',
|
||||
content: `确定要删除文件 "${file.filename}" 吗？此操作不可恢复。`,
|
||||
okText: '删除',
|
||||
okType: 'danger',
|
||||
cancelText: '取消',
|
||||
onOk: async () => {
|
||||
try {
|
||||
// 获取当前数据库ID
|
||||
const currentDbId = databaseId.value;
|
||||
if (!currentDbId) {
|
||||
message.error('知识库ID不存在');
|
||||
return;
|
||||
}
|
||||
|
||||
message.loading('正在删除文件...', 0);
|
||||
await documentApi.deleteDocument(currentDbId, file.file_id);
|
||||
message.destroy();
|
||||
|
||||
// 从同名文件列表中移除
|
||||
sameNameFiles.value = sameNameFiles.value.filter(f => f.file_id !== file.file_id);
|
||||
|
||||
message.success(`文件 ${file.filename} 删除成功`);
|
||||
} catch (error) {
|
||||
message.destroy();
|
||||
console.error('删除文件失败:', error);
|
||||
message.error(`删除文件失败: ${error.message || '未知错误'}`);
|
||||
}
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
const handleFileUpload = (info) => {
|
||||
if (info?.file?.status === 'error') {
|
||||
const errorMessage = info.file?.response?.detail || `文件上传失败：${info.file.name}`;
|
||||
message.error(errorMessage);
|
||||
}
|
||||
|
||||
// 检查是否有同名文件提示
|
||||
if (info?.file?.status === 'done' && info.file.response) {
|
||||
const response = info.file.response;
|
||||
if (response.has_same_name && response.same_name_files && response.same_name_files.length > 0) {
|
||||
// 显示同名文件提示
|
||||
showSameNameFilesInUploadArea(response.same_name_files);
|
||||
}
|
||||
}
|
||||
|
||||
fileList.value = info?.fileList ?? [];
|
||||
};
|
||||
|
||||
@ -613,6 +741,7 @@ const chunkData = async () => {
|
||||
if (success) {
|
||||
emit('update:visible', false);
|
||||
fileList.value = [];
|
||||
sameNameFiles.value = []; // 清空同名文件列表
|
||||
}
|
||||
};
|
||||
|
||||
@ -735,4 +864,82 @@ const chunkData = async () => {
|
||||
font-size: 12px;
|
||||
color: var(--color-warning-500);
|
||||
}
|
||||
|
||||
// 同名文件提示样式
|
||||
.same-name-files-section {
|
||||
margin-top: 16px;
|
||||
padding: 12px;
|
||||
background: var(--main-50);
|
||||
border: 1px solid var(--main-200);
|
||||
border-radius: 6px;
|
||||
}
|
||||
|
||||
.same-name-files-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
margin-bottom: 12px;
|
||||
color: var(--main-700);
|
||||
font-weight: 500;
|
||||
font-size: 14px;
|
||||
}
|
||||
|
||||
.same-name-files-list {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.same-name-file-item {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
padding: 8px 12px;
|
||||
background: #fff;
|
||||
border: 1px solid var(--gray-300);
|
||||
border-radius: 6px;
|
||||
}
|
||||
|
||||
.same-name-file-info {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.same-name-file-name {
|
||||
font-weight: 500;
|
||||
color: var(--gray-800);
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.same-name-file-size {
|
||||
font-size: 12px;
|
||||
color: var(--gray-500);
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.same-name-file-time {
|
||||
font-size: 12px;
|
||||
color: var(--gray-500);
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.same-name-file-actions {
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.download-btn {
|
||||
color: var(--main-600);
|
||||
}
|
||||
|
||||
.download-btn:hover {
|
||||
color: var(--main-700);
|
||||
background-color: var(--main-50);
|
||||
}
|
||||
</style>
|
||||
|
||||
@ -837,9 +837,9 @@ onMounted(() => {
|
||||
|
||||
.dbcard, .database {
|
||||
width: 100%;
|
||||
padding: 24px;
|
||||
padding: 16px;
|
||||
border-radius: 16px;
|
||||
height: 180px;
|
||||
height: 156px;
|
||||
cursor: pointer;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user