From c3a8360a773ae4e6a34c11c13172800e0b714490 Mon Sep 17 00:00:00 2001 From: Wenjie Zhang Date: Sat, 11 Oct 2025 10:36:57 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=20doc=20=E8=A7=A3?= =?UTF-8?q?=E6=9E=90=E7=9A=84=E9=97=AE=E9=A2=98=EF=BC=8C=E5=B9=B6=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=E6=96=87=E4=BB=B6=E7=B1=BB=E5=9E=8B=E6=94=AF=E6=8C=81?= =?UTF-8?q?=E6=A3=80=E6=9F=A5=E5=92=8C=E7=9B=B8=E5=85=B3API?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 在文件上传功能中添加对不支持文件类型的检查，返回400错误。 - 新增获取支持文件类型的API接口。 - 更新前端文件上传组件，动态加载支持的文件类型并提供用户提示。 --- docs/changelog/roadmap.md | 4 +- server/routers/knowledge_router.py | 12 ++- src/knowledge/indexing.py | 61 ++++++++++++-- web/src/apis/knowledge_api.js | 9 ++- web/src/components/FileUploadModal.vue | 106 +++++++++++++++++++++++-- 5 files changed, 174 insertions(+), 18 deletions(-) diff --git a/docs/changelog/roadmap.md b/docs/changelog/roadmap.md index 273557db..50d3218a 100644 --- a/docs/changelog/roadmap.md +++ b/docs/changelog/roadmap.md @@ -6,8 +6,8 @@ 🐛**BUGs** -- [ ] 部分 doc 格式的文件支持有问题 -- [ ] 当出现不支持的文件类型的时候，前端没有限制 +- [x] 部分 doc 格式的文件支持有问题 +- [x] 当出现不支持的文件类型的时候，前端没有限制 - [ ] 当消息生成的时候有报错的时候，前端无显示 - [ ] 另外一个智能体的历史对话无法显示 - [ ] 调用统计的统计结果有问题（Token 计算方法可能也不对） diff --git a/server/routers/knowledge_router.py b/server/routers/knowledge_router.py index b8d69fee..e417be6e 100644 --- a/server/routers/knowledge_router.py +++ b/server/routers/knowledge_router.py @@ -9,7 +9,7 @@ from starlette.responses import FileResponse as StarletteFileResponse from src.storage.db.models import User from server.utils.auth_middleware import get_admin_user from src import config, knowledge_base -from src.knowledge.indexing import process_file_to_markdown +from src.knowledge.indexing import SUPPORTED_FILE_EXTENSIONS, is_supported_file_extension, process_file_to_markdown from src.models.embed import test_embedding_model_status, test_all_embedding_models_status from src.utils import hashstr, logger @@ -522,6 +522,10 @@ async def upload_file( logger.debug(f"Received upload file with filename: {file.filename}") + if not is_supported_file_extension(file.filename): + ext = os.path.splitext(file.filename)[1].lower() + raise HTTPException(status_code=400, detail=f"Unsupported file type: {ext}") + # 根据db_id获取上传路径，如果db_id为None则使用默认路径 if db_id: upload_dir = knowledge_base.get_db_upload_path(db_id) @@ -539,6 +543,12 @@ async def upload_file( return {"message": "File successfully uploaded", "file_path": file_path, "db_id": db_id} +@knowledge.get("/files/supported-types") +async def get_supported_file_types(current_user: User = Depends(get_admin_user)): + """获取当前支持的文件类型""" + return {"message": "success", "file_types": sorted(SUPPORTED_FILE_EXTENSIONS)} + + @knowledge.post("/files/markdown") async def mark_it_down(file: UploadFile = File(...), current_user: User = Depends(get_admin_user)): """调用 src.knowledge.indexing 下面的 process_file_to_markdown 解析为 markdown，参数是文件，需要管理员权限""" diff --git a/src/knowledge/indexing.py b/src/knowledge/indexing.py index d672847f..600a59fe 100644 --- a/src/knowledge/indexing.py +++ b/src/knowledge/indexing.py @@ -5,17 +5,69 @@ from pathlib import Path from langchain.text_splitter import RecursiveCharacterTextSplitter from langchain_community.document_loaders import ( CSVLoader, - Docx2txtLoader, JSONLoader, PyPDFLoader, TextLoader, UnstructuredHTMLLoader, UnstructuredMarkdownLoader, + UnstructuredWordDocumentLoader, ) from src.utils import logger +SUPPORTED_FILE_EXTENSIONS: tuple[str, ...] = ( + ".txt", + ".md", + ".doc", + ".docx", + ".html", + ".htm", + ".json", + ".csv", + ".xls", + ".xlsx", + ".pdf", + ".jpg", + ".jpeg", + ".png", + ".bmp", + ".tiff", + ".tif", +) + + +def is_supported_file_extension(file_name: str | os.PathLike[str]) -> bool: + """Check whether the given file path has a supported extension.""" + return Path(file_name).suffix.lower() in SUPPORTED_FILE_EXTENSIONS + + +def _extract_word_text(file_path: Path) -> str: + """ + Parse Word documents (.doc/.docx) into plain text. + + Try python-docx first for docx files and fall back to the unstructured + loader so legacy .doc files are still parsed when possible. + """ + try: + from docx import Document # type: ignore + + doc = Document(file_path) + text = "\n".join(paragraph.text for paragraph in doc.paragraphs).strip() + if text: + return text + except Exception as docx_error: # noqa: BLE001 + logger.warning(f"python-docx failed to parse {file_path.name}: {docx_error}") + + try: + loader = UnstructuredWordDocumentLoader(str(file_path)) + docs = loader.load() + return "\n".join(doc.page_content for doc in docs).strip() + except Exception as unstructured_error: # noqa: BLE001 + logger.error(f"Unstructured failed to parse {file_path.name}: {unstructured_error}") + raise ValueError(f"无法解析 Word 文档: {file_path.name}") from unstructured_error + + def chunk_with_parser(file_path, params=None): """ 使用文件解析器将文件切分成固定大小的块 @@ -38,7 +90,7 @@ def chunk_with_parser(file_path, params=None): loader = UnstructuredMarkdownLoader(file_path) elif file_type in [".docx", ".doc"]: - loader = Docx2txtLoader(file_path) + loader = UnstructuredWordDocumentLoader(file_path) elif file_type in [".html", ".htm"]: loader = UnstructuredHTMLLoader(file_path) @@ -251,10 +303,7 @@ async def process_file_to_markdown(file_path: str, params: dict | None = None) - elif file_ext in [".doc", ".docx"]: # 处理 Word 文档 - from docx import Document # type: ignore - - doc = Document(file_path_obj) - text = "\n".join([para.text for para in doc.paragraphs]) + text = _extract_word_text(file_path_obj) return f"# {file_path_obj.name}\n\n{text}" elif file_ext in [".jpg", ".jpeg", ".png", ".bmp", ".tiff", ".tif"]: diff --git a/web/src/apis/knowledge_api.js b/web/src/apis/knowledge_api.js index 45b71c7c..4489201b 100644 --- a/web/src/apis/knowledge_api.js +++ b/web/src/apis/knowledge_api.js @@ -173,6 +173,14 @@ export const fileApi = { 'Content-Type': 'multipart/form-data' } }) + }, + + /** + * 获取支持的文件类型 + * @returns {Promise} - 文件类型列表 + */ + getSupportedFileTypes: async () => { + return apiAdminGet('/api/knowledge/files/supported-types') } } @@ -220,4 +228,3 @@ export const embeddingApi = { return apiAdminGet('/api/knowledge/embedding-models/status') } } - diff --git a/web/src/components/FileUploadModal.vue b/web/src/components/FileUploadModal.vue index 8075b73b..e10ac1ca 100644 --- a/web/src/components/FileUploadModal.vue +++ b/web/src/components/FileUploadModal.vue @@ -105,6 +105,8 @@ name="file" :multiple="true" :disabled="chunkLoading" + :accept="acceptedFileTypes" + :before-upload="beforeUpload" :action="'/api/knowledge/files/upload?db_id=' + databaseId" :headers="getAuthHeaders()" @change="handleFileUpload" @@ -112,7 +114,7 @@ >

点击或者把文件拖拽到这里上传

- 目前仅支持上传文本、图片文件，如 .pdf, .txt, .md, .docx, png, jpg等。 + 支持的文件类型：{{ uploadHint }}

@@ -174,11 +176,12 @@