fix: 修复 doc 解析的问题,并增加文件类型支持检查和相关API
- 在文件上传功能中添加对不支持文件类型的检查,返回400错误。 - 新增获取支持文件类型的API接口。 - 更新前端文件上传组件,动态加载支持的文件类型并提供用户提示。
This commit is contained in:
parent
ceb676d79c
commit
c3a8360a77
@ -6,8 +6,8 @@
|
||||
|
||||
|
||||
🐛**BUGs**
|
||||
- [ ] 部分 doc 格式的文件支持有问题
|
||||
- [ ] 当出现不支持的文件类型的时候,前端没有限制
|
||||
- [x] 部分 doc 格式的文件支持有问题
|
||||
- [x] 当出现不支持的文件类型的时候,前端没有限制
|
||||
- [ ] 当消息生成的时候有报错的时候,前端无显示
|
||||
- [ ] 另外一个智能体的历史对话无法显示
|
||||
- [ ] 调用统计的统计结果有问题(Token 计算方法可能也不对)
|
||||
|
||||
@ -9,7 +9,7 @@ from starlette.responses import FileResponse as StarletteFileResponse
|
||||
from src.storage.db.models import User
|
||||
from server.utils.auth_middleware import get_admin_user
|
||||
from src import config, knowledge_base
|
||||
from src.knowledge.indexing import process_file_to_markdown
|
||||
from src.knowledge.indexing import SUPPORTED_FILE_EXTENSIONS, is_supported_file_extension, process_file_to_markdown
|
||||
from src.models.embed import test_embedding_model_status, test_all_embedding_models_status
|
||||
from src.utils import hashstr, logger
|
||||
|
||||
@ -522,6 +522,10 @@ async def upload_file(
|
||||
|
||||
logger.debug(f"Received upload file with filename: {file.filename}")
|
||||
|
||||
if not is_supported_file_extension(file.filename):
|
||||
ext = os.path.splitext(file.filename)[1].lower()
|
||||
raise HTTPException(status_code=400, detail=f"Unsupported file type: {ext}")
|
||||
|
||||
# 根据db_id获取上传路径,如果db_id为None则使用默认路径
|
||||
if db_id:
|
||||
upload_dir = knowledge_base.get_db_upload_path(db_id)
|
||||
@ -539,6 +543,12 @@ async def upload_file(
|
||||
return {"message": "File successfully uploaded", "file_path": file_path, "db_id": db_id}
|
||||
|
||||
|
||||
@knowledge.get("/files/supported-types")
|
||||
async def get_supported_file_types(current_user: User = Depends(get_admin_user)):
|
||||
"""获取当前支持的文件类型"""
|
||||
return {"message": "success", "file_types": sorted(SUPPORTED_FILE_EXTENSIONS)}
|
||||
|
||||
|
||||
@knowledge.post("/files/markdown")
|
||||
async def mark_it_down(file: UploadFile = File(...), current_user: User = Depends(get_admin_user)):
|
||||
"""调用 src.knowledge.indexing 下面的 process_file_to_markdown 解析为 markdown,参数是文件,需要管理员权限"""
|
||||
|
||||
@ -5,17 +5,69 @@ from pathlib import Path
|
||||
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
||||
from langchain_community.document_loaders import (
|
||||
CSVLoader,
|
||||
Docx2txtLoader,
|
||||
JSONLoader,
|
||||
PyPDFLoader,
|
||||
TextLoader,
|
||||
UnstructuredHTMLLoader,
|
||||
UnstructuredMarkdownLoader,
|
||||
UnstructuredWordDocumentLoader,
|
||||
)
|
||||
|
||||
from src.utils import logger
|
||||
|
||||
|
||||
SUPPORTED_FILE_EXTENSIONS: tuple[str, ...] = (
|
||||
".txt",
|
||||
".md",
|
||||
".doc",
|
||||
".docx",
|
||||
".html",
|
||||
".htm",
|
||||
".json",
|
||||
".csv",
|
||||
".xls",
|
||||
".xlsx",
|
||||
".pdf",
|
||||
".jpg",
|
||||
".jpeg",
|
||||
".png",
|
||||
".bmp",
|
||||
".tiff",
|
||||
".tif",
|
||||
)
|
||||
|
||||
|
||||
def is_supported_file_extension(file_name: str | os.PathLike[str]) -> bool:
|
||||
"""Check whether the given file path has a supported extension."""
|
||||
return Path(file_name).suffix.lower() in SUPPORTED_FILE_EXTENSIONS
|
||||
|
||||
|
||||
def _extract_word_text(file_path: Path) -> str:
|
||||
"""
|
||||
Parse Word documents (.doc/.docx) into plain text.
|
||||
|
||||
Try python-docx first for docx files and fall back to the unstructured
|
||||
loader so legacy .doc files are still parsed when possible.
|
||||
"""
|
||||
try:
|
||||
from docx import Document # type: ignore
|
||||
|
||||
doc = Document(file_path)
|
||||
text = "\n".join(paragraph.text for paragraph in doc.paragraphs).strip()
|
||||
if text:
|
||||
return text
|
||||
except Exception as docx_error: # noqa: BLE001
|
||||
logger.warning(f"python-docx failed to parse {file_path.name}: {docx_error}")
|
||||
|
||||
try:
|
||||
loader = UnstructuredWordDocumentLoader(str(file_path))
|
||||
docs = loader.load()
|
||||
return "\n".join(doc.page_content for doc in docs).strip()
|
||||
except Exception as unstructured_error: # noqa: BLE001
|
||||
logger.error(f"Unstructured failed to parse {file_path.name}: {unstructured_error}")
|
||||
raise ValueError(f"无法解析 Word 文档: {file_path.name}") from unstructured_error
|
||||
|
||||
|
||||
def chunk_with_parser(file_path, params=None):
|
||||
"""
|
||||
使用文件解析器将文件切分成固定大小的块
|
||||
@ -38,7 +90,7 @@ def chunk_with_parser(file_path, params=None):
|
||||
loader = UnstructuredMarkdownLoader(file_path)
|
||||
|
||||
elif file_type in [".docx", ".doc"]:
|
||||
loader = Docx2txtLoader(file_path)
|
||||
loader = UnstructuredWordDocumentLoader(file_path)
|
||||
|
||||
elif file_type in [".html", ".htm"]:
|
||||
loader = UnstructuredHTMLLoader(file_path)
|
||||
@ -251,10 +303,7 @@ async def process_file_to_markdown(file_path: str, params: dict | None = None) -
|
||||
|
||||
elif file_ext in [".doc", ".docx"]:
|
||||
# 处理 Word 文档
|
||||
from docx import Document # type: ignore
|
||||
|
||||
doc = Document(file_path_obj)
|
||||
text = "\n".join([para.text for para in doc.paragraphs])
|
||||
text = _extract_word_text(file_path_obj)
|
||||
return f"# {file_path_obj.name}\n\n{text}"
|
||||
|
||||
elif file_ext in [".jpg", ".jpeg", ".png", ".bmp", ".tiff", ".tif"]:
|
||||
|
||||
@ -173,6 +173,14 @@ export const fileApi = {
|
||||
'Content-Type': 'multipart/form-data'
|
||||
}
|
||||
})
|
||||
},
|
||||
|
||||
/**
|
||||
* 获取支持的文件类型
|
||||
* @returns {Promise} - 文件类型列表
|
||||
*/
|
||||
getSupportedFileTypes: async () => {
|
||||
return apiAdminGet('/api/knowledge/files/supported-types')
|
||||
}
|
||||
}
|
||||
|
||||
@ -220,4 +228,3 @@ export const embeddingApi = {
|
||||
return apiAdminGet('/api/knowledge/embedding-models/status')
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -105,6 +105,8 @@
|
||||
name="file"
|
||||
:multiple="true"
|
||||
:disabled="chunkLoading"
|
||||
:accept="acceptedFileTypes"
|
||||
:before-upload="beforeUpload"
|
||||
:action="'/api/knowledge/files/upload?db_id=' + databaseId"
|
||||
:headers="getAuthHeaders()"
|
||||
@change="handleFileUpload"
|
||||
@ -112,7 +114,7 @@
|
||||
>
|
||||
<p class="ant-upload-text">点击或者把文件拖拽到这里上传</p>
|
||||
<p class="ant-upload-hint">
|
||||
目前仅支持上传文本、图片文件,如 .pdf, .txt, .md, .docx, png, jpg等。
|
||||
支持的文件类型:{{ uploadHint }}
|
||||
</p>
|
||||
</a-upload-dragger>
|
||||
</div>
|
||||
@ -174,11 +176,12 @@
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { ref, reactive, computed, watch } from 'vue';
|
||||
import { message } from 'ant-design-vue';
|
||||
import { ref, computed, onMounted } from 'vue';
|
||||
import { message, Upload } from 'ant-design-vue';
|
||||
import { useUserStore } from '@/stores/user';
|
||||
import { useDatabaseStore } from '@/stores/database';
|
||||
import { ocrApi } from '@/apis/system_api';
|
||||
import { fileApi } from '@/apis/knowledge_api';
|
||||
import {
|
||||
FileOutlined,
|
||||
LinkOutlined,
|
||||
@ -198,6 +201,83 @@ const emit = defineEmits(['update:visible']);
|
||||
|
||||
const store = useDatabaseStore();
|
||||
|
||||
const DEFAULT_SUPPORTED_TYPES = [
|
||||
'.txt',
|
||||
'.pdf',
|
||||
'.jpg',
|
||||
'.jpeg',
|
||||
'.md',
|
||||
'.docx',
|
||||
'.doc',
|
||||
];
|
||||
|
||||
const normalizeExtensions = (extensions) => {
|
||||
if (!Array.isArray(extensions)) {
|
||||
return [];
|
||||
}
|
||||
const normalized = extensions
|
||||
.map((ext) => (typeof ext === 'string' ? ext.trim().toLowerCase() : ''))
|
||||
.filter((ext) => ext.length > 0)
|
||||
.map((ext) => (ext.startsWith('.') ? ext : `.${ext}`));
|
||||
|
||||
return Array.from(new Set(normalized)).sort();
|
||||
};
|
||||
|
||||
const supportedFileTypes = ref(normalizeExtensions(DEFAULT_SUPPORTED_TYPES));
|
||||
|
||||
const applySupportedFileTypes = (extensions) => {
|
||||
const normalized = normalizeExtensions(extensions);
|
||||
if (normalized.length > 0) {
|
||||
supportedFileTypes.value = normalized;
|
||||
} else {
|
||||
supportedFileTypes.value = normalizeExtensions(DEFAULT_SUPPORTED_TYPES);
|
||||
}
|
||||
};
|
||||
|
||||
const acceptedFileTypes = computed(() => {
|
||||
if (!supportedFileTypes.value.length) {
|
||||
return '';
|
||||
}
|
||||
return supportedFileTypes.value.join(',');
|
||||
});
|
||||
|
||||
const uploadHint = computed(() => {
|
||||
if (!supportedFileTypes.value.length) {
|
||||
return '加载中...';
|
||||
}
|
||||
return supportedFileTypes.value.join(', ');
|
||||
});
|
||||
|
||||
const isSupportedExtension = (fileName) => {
|
||||
if (!fileName) {
|
||||
return true;
|
||||
}
|
||||
if (!supportedFileTypes.value.length) {
|
||||
return true;
|
||||
}
|
||||
const lastDotIndex = fileName.lastIndexOf('.');
|
||||
if (lastDotIndex === -1) {
|
||||
return false;
|
||||
}
|
||||
const ext = fileName.slice(lastDotIndex).toLowerCase();
|
||||
return supportedFileTypes.value.includes(ext);
|
||||
};
|
||||
|
||||
const loadSupportedFileTypes = async () => {
|
||||
try {
|
||||
const data = await fileApi.getSupportedFileTypes();
|
||||
applySupportedFileTypes(data?.file_types);
|
||||
} catch (error) {
|
||||
console.error('获取支持的文件类型失败:', error);
|
||||
message.warning('获取支持的文件类型失败,已使用默认配置');
|
||||
applySupportedFileTypes(DEFAULT_SUPPORTED_TYPES);
|
||||
}
|
||||
};
|
||||
|
||||
onMounted(() => {
|
||||
loadSupportedFileTypes();
|
||||
});
|
||||
|
||||
const visible = computed({
|
||||
get: () => props.visible,
|
||||
set: (value) => emit('update:visible', value)
|
||||
@ -370,14 +450,24 @@ const handleCancel = () => {
|
||||
emit('update:visible', false);
|
||||
};
|
||||
|
||||
const handleFileUpload = (event) => {
|
||||
console.log(event);
|
||||
const beforeUpload = (file) => {
|
||||
if (!isSupportedExtension(file?.name)) {
|
||||
message.error(`不支持的文件类型:${file?.name || '未知文件'}`);
|
||||
return Upload.LIST_IGNORE;
|
||||
}
|
||||
return true;
|
||||
};
|
||||
|
||||
const handleDrop = (event) => {
|
||||
console.log(event);
|
||||
const handleFileUpload = (info) => {
|
||||
if (info?.file?.status === 'error') {
|
||||
const errorMessage = info.file?.response?.detail || `文件上传失败:${info.file.name}`;
|
||||
message.error(errorMessage);
|
||||
}
|
||||
fileList.value = info?.fileList ?? [];
|
||||
};
|
||||
|
||||
const handleDrop = () => {};
|
||||
|
||||
const showChunkConfigModal = () => {
|
||||
tempChunkParams.value = {
|
||||
chunk_size: chunkParams.value.chunk_size,
|
||||
@ -547,4 +637,4 @@ const chunkData = async () => {
|
||||
.chunk-config-content .params-info {
|
||||
margin-bottom: 16px;
|
||||
}
|
||||
</style>
|
||||
</style>
|
||||
|
||||
Loading…
Reference in New Issue
Block a user