fix: 修复 doc 解析的问题,并增加文件类型支持检查和相关API

- 在文件上传功能中添加对不支持文件类型的检查,返回400错误。
- 新增获取支持文件类型的API接口。
- 更新前端文件上传组件,动态加载支持的文件类型并提供用户提示。
This commit is contained in:
Wenjie Zhang 2025-10-11 10:36:57 +08:00
parent ceb676d79c
commit c3a8360a77
5 changed files with 174 additions and 18 deletions

View File

@ -6,8 +6,8 @@
🐛**BUGs**
- [ ] 部分 doc 格式的文件支持有问题
- [ ] 当出现不支持的文件类型的时候,前端没有限制
- [x] 部分 doc 格式的文件支持有问题
- [x] 当出现不支持的文件类型的时候,前端没有限制
- [ ] 当消息生成的时候有报错的时候,前端无显示
- [ ] 另外一个智能体的历史对话无法显示
- [ ] 调用统计的统计结果有问题Token 计算方法可能也不对)

View File

@ -9,7 +9,7 @@ from starlette.responses import FileResponse as StarletteFileResponse
from src.storage.db.models import User
from server.utils.auth_middleware import get_admin_user
from src import config, knowledge_base
from src.knowledge.indexing import process_file_to_markdown
from src.knowledge.indexing import SUPPORTED_FILE_EXTENSIONS, is_supported_file_extension, process_file_to_markdown
from src.models.embed import test_embedding_model_status, test_all_embedding_models_status
from src.utils import hashstr, logger
@ -522,6 +522,10 @@ async def upload_file(
logger.debug(f"Received upload file with filename: {file.filename}")
if not is_supported_file_extension(file.filename):
ext = os.path.splitext(file.filename)[1].lower()
raise HTTPException(status_code=400, detail=f"Unsupported file type: {ext}")
# 根据db_id获取上传路径如果db_id为None则使用默认路径
if db_id:
upload_dir = knowledge_base.get_db_upload_path(db_id)
@ -539,6 +543,12 @@ async def upload_file(
return {"message": "File successfully uploaded", "file_path": file_path, "db_id": db_id}
@knowledge.get("/files/supported-types")
async def get_supported_file_types(current_user: User = Depends(get_admin_user)):
"""获取当前支持的文件类型"""
return {"message": "success", "file_types": sorted(SUPPORTED_FILE_EXTENSIONS)}
@knowledge.post("/files/markdown")
async def mark_it_down(file: UploadFile = File(...), current_user: User = Depends(get_admin_user)):
"""调用 src.knowledge.indexing 下面的 process_file_to_markdown 解析为 markdown参数是文件需要管理员权限"""

View File

@ -5,17 +5,69 @@ from pathlib import Path
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain_community.document_loaders import (
CSVLoader,
Docx2txtLoader,
JSONLoader,
PyPDFLoader,
TextLoader,
UnstructuredHTMLLoader,
UnstructuredMarkdownLoader,
UnstructuredWordDocumentLoader,
)
from src.utils import logger
SUPPORTED_FILE_EXTENSIONS: tuple[str, ...] = (
".txt",
".md",
".doc",
".docx",
".html",
".htm",
".json",
".csv",
".xls",
".xlsx",
".pdf",
".jpg",
".jpeg",
".png",
".bmp",
".tiff",
".tif",
)
def is_supported_file_extension(file_name: str | os.PathLike[str]) -> bool:
"""Check whether the given file path has a supported extension."""
return Path(file_name).suffix.lower() in SUPPORTED_FILE_EXTENSIONS
def _extract_word_text(file_path: Path) -> str:
"""
Parse Word documents (.doc/.docx) into plain text.
Try python-docx first for docx files and fall back to the unstructured
loader so legacy .doc files are still parsed when possible.
"""
try:
from docx import Document # type: ignore
doc = Document(file_path)
text = "\n".join(paragraph.text for paragraph in doc.paragraphs).strip()
if text:
return text
except Exception as docx_error: # noqa: BLE001
logger.warning(f"python-docx failed to parse {file_path.name}: {docx_error}")
try:
loader = UnstructuredWordDocumentLoader(str(file_path))
docs = loader.load()
return "\n".join(doc.page_content for doc in docs).strip()
except Exception as unstructured_error: # noqa: BLE001
logger.error(f"Unstructured failed to parse {file_path.name}: {unstructured_error}")
raise ValueError(f"无法解析 Word 文档: {file_path.name}") from unstructured_error
def chunk_with_parser(file_path, params=None):
"""
使用文件解析器将文件切分成固定大小的块
@ -38,7 +90,7 @@ def chunk_with_parser(file_path, params=None):
loader = UnstructuredMarkdownLoader(file_path)
elif file_type in [".docx", ".doc"]:
loader = Docx2txtLoader(file_path)
loader = UnstructuredWordDocumentLoader(file_path)
elif file_type in [".html", ".htm"]:
loader = UnstructuredHTMLLoader(file_path)
@ -251,10 +303,7 @@ async def process_file_to_markdown(file_path: str, params: dict | None = None) -
elif file_ext in [".doc", ".docx"]:
# 处理 Word 文档
from docx import Document # type: ignore
doc = Document(file_path_obj)
text = "\n".join([para.text for para in doc.paragraphs])
text = _extract_word_text(file_path_obj)
return f"# {file_path_obj.name}\n\n{text}"
elif file_ext in [".jpg", ".jpeg", ".png", ".bmp", ".tiff", ".tif"]:

View File

@ -173,6 +173,14 @@ export const fileApi = {
'Content-Type': 'multipart/form-data'
}
})
},
/**
* 获取支持的文件类型
* @returns {Promise} - 文件类型列表
*/
getSupportedFileTypes: async () => {
return apiAdminGet('/api/knowledge/files/supported-types')
}
}
@ -220,4 +228,3 @@ export const embeddingApi = {
return apiAdminGet('/api/knowledge/embedding-models/status')
}
}

View File

@ -105,6 +105,8 @@
name="file"
:multiple="true"
:disabled="chunkLoading"
:accept="acceptedFileTypes"
:before-upload="beforeUpload"
:action="'/api/knowledge/files/upload?db_id=' + databaseId"
:headers="getAuthHeaders()"
@change="handleFileUpload"
@ -112,7 +114,7 @@
>
<p class="ant-upload-text">点击或者把文件拖拽到这里上传</p>
<p class="ant-upload-hint">
目前仅支持上传文本图片文件 .pdf, .txt, .md, .docx, png, jpg等
支持的文件类型{{ uploadHint }}
</p>
</a-upload-dragger>
</div>
@ -174,11 +176,12 @@
</template>
<script setup>
import { ref, reactive, computed, watch } from 'vue';
import { message } from 'ant-design-vue';
import { ref, computed, onMounted } from 'vue';
import { message, Upload } from 'ant-design-vue';
import { useUserStore } from '@/stores/user';
import { useDatabaseStore } from '@/stores/database';
import { ocrApi } from '@/apis/system_api';
import { fileApi } from '@/apis/knowledge_api';
import {
FileOutlined,
LinkOutlined,
@ -198,6 +201,83 @@ const emit = defineEmits(['update:visible']);
const store = useDatabaseStore();
const DEFAULT_SUPPORTED_TYPES = [
'.txt',
'.pdf',
'.jpg',
'.jpeg',
'.md',
'.docx',
'.doc',
];
const normalizeExtensions = (extensions) => {
if (!Array.isArray(extensions)) {
return [];
}
const normalized = extensions
.map((ext) => (typeof ext === 'string' ? ext.trim().toLowerCase() : ''))
.filter((ext) => ext.length > 0)
.map((ext) => (ext.startsWith('.') ? ext : `.${ext}`));
return Array.from(new Set(normalized)).sort();
};
const supportedFileTypes = ref(normalizeExtensions(DEFAULT_SUPPORTED_TYPES));
const applySupportedFileTypes = (extensions) => {
const normalized = normalizeExtensions(extensions);
if (normalized.length > 0) {
supportedFileTypes.value = normalized;
} else {
supportedFileTypes.value = normalizeExtensions(DEFAULT_SUPPORTED_TYPES);
}
};
const acceptedFileTypes = computed(() => {
if (!supportedFileTypes.value.length) {
return '';
}
return supportedFileTypes.value.join(',');
});
const uploadHint = computed(() => {
if (!supportedFileTypes.value.length) {
return '加载中...';
}
return supportedFileTypes.value.join(', ');
});
const isSupportedExtension = (fileName) => {
if (!fileName) {
return true;
}
if (!supportedFileTypes.value.length) {
return true;
}
const lastDotIndex = fileName.lastIndexOf('.');
if (lastDotIndex === -1) {
return false;
}
const ext = fileName.slice(lastDotIndex).toLowerCase();
return supportedFileTypes.value.includes(ext);
};
const loadSupportedFileTypes = async () => {
try {
const data = await fileApi.getSupportedFileTypes();
applySupportedFileTypes(data?.file_types);
} catch (error) {
console.error('获取支持的文件类型失败:', error);
message.warning('获取支持的文件类型失败,已使用默认配置');
applySupportedFileTypes(DEFAULT_SUPPORTED_TYPES);
}
};
onMounted(() => {
loadSupportedFileTypes();
});
const visible = computed({
get: () => props.visible,
set: (value) => emit('update:visible', value)
@ -370,14 +450,24 @@ const handleCancel = () => {
emit('update:visible', false);
};
const handleFileUpload = (event) => {
console.log(event);
const beforeUpload = (file) => {
if (!isSupportedExtension(file?.name)) {
message.error(`不支持的文件类型:${file?.name || '未知文件'}`);
return Upload.LIST_IGNORE;
}
return true;
};
const handleDrop = (event) => {
console.log(event);
const handleFileUpload = (info) => {
if (info?.file?.status === 'error') {
const errorMessage = info.file?.response?.detail || `文件上传失败:${info.file.name}`;
message.error(errorMessage);
}
fileList.value = info?.fileList ?? [];
};
const handleDrop = () => {};
const showChunkConfigModal = () => {
tempChunkParams.value = {
chunk_size: chunkParams.value.chunk_size,
@ -547,4 +637,4 @@ const chunkData = async () => {
.chunk-config-content .params-info {
margin-bottom: 16px;
}
</style>
</style>