diff --git a/scripts/paddlestructure/example_usage.py b/scripts/paddlestructure/example_usage.py index eea1b4e8..68b4d6cb 100644 --- a/scripts/paddlestructure/example_usage.py +++ b/scripts/paddlestructure/example_usage.py @@ -1,11 +1,10 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ PaddleX 文档分析使用示例 演示如何使用 analyze_document 函数分析文档 """ -from paddlex_layout_parser import analyze_document +from src.plugins.paddlex import analyze_document import json @@ -14,7 +13,7 @@ def analyze_custom_file(file_path: str): print(f"\n🔍 分析自定义文件: {file_path}") - result = analyze_document(file_path=file_path) + result = analyze_document(file_path=file_path, base_url="http://172.19.13.5:8080") if result["success"]: return result @@ -27,6 +26,7 @@ if __name__ == "__main__": # main() # 如果您想分析其他文件，可以取消注释下面的代码 - custom_file = "test/struct_pdf/P020241226617572090546.pdf" - custom_file = "test/data/PixPin_2025-06-19_23-42-17.png" - print(analyze_custom_file(custom_file)) \ No newline at end of file + custom_file = "test/data/ocr_test/1706.03762v7_扫描版.pdf" + custom_file = "test/data/ocr_test/PixPin_2025-06-19_23-42-17.png" + + print(analyze_custom_file(custom_file)) diff --git a/server/routers/data_router.py b/server/routers/data_router.py index 38c466cc..7f0e83c1 100644 --- a/server/routers/data_router.py +++ b/server/routers/data_router.py @@ -56,7 +56,9 @@ async def file_to_chunk(db_id: str = Body(...), files: list[str] = Body(...), pa logger.debug(f"File to chunk for db_id {db_id}: {files} {params=}") try: processed_files = await knowledge_base.save_files_for_pending_indexing(db_id, files, params) - return {"message": "Files processed and pending indexing", "files": processed_files, "status": "success"} + processed_failed_count = len([_p['status'] == 'failed' for _p in processed_files]) + processed_info = f"Processed {len(processed_files)} files for pending indexing, {processed_failed_count} files failed" + return {"message": processed_info, "files": processed_files, "status": "success"} except Exception as e: logger.error(f"Failed to process files for pending indexing: {e}, {traceback.format_exc()}") return {"message": f"Failed to process files for pending indexing: {e}", "status": "failed"} diff --git a/src/core/indexing.py b/src/core/indexing.py index fd783134..904d26fb 100644 --- a/src/core/indexing.py +++ b/src/core/indexing.py @@ -133,6 +133,10 @@ def parse_pdf(file, params=None): from src.plugins import ocr return ocr.process_pdf_mineru(file) + elif opt_ocr == "paddlex_ocr": + from src.plugins import ocr + return ocr.process_pdf_paddlex(file) + else: return pdfreader(file, params=params) diff --git a/src/plugins/_ocr.py b/src/plugins/_ocr.py index 5c36b697..d306ed79 100644 --- a/src/plugins/_ocr.py +++ b/src/plugins/_ocr.py @@ -159,11 +159,10 @@ class OCRPlugin: :param pdf_path: PDF文件路径 :return: 提取的文本 """ + import requests + from .mineru import parse_doc mineru_ocr_uri = os.getenv("MINERU_OCR_URI", "http://localhost:30000") mineru_ocr_uri_health = f"{mineru_ocr_uri}/health" - import requests - import json - from .mineru import parse_doc health_check_response = requests.get(mineru_ocr_uri_health, timeout=5) if health_check_response.status_code != 200: @@ -180,6 +179,28 @@ class OCRPlugin: logger.debug(f"Mineru OCR result: {pdf_text[:50]}(...) total {len(pdf_text)} characters.") return pdf_text + def process_pdf_paddlex(self, pdf_path): + """ + 使用Paddlex OCR处理PDF文件 + :param pdf_path: PDF文件路径 + :return: 提取的文本 + """ + from .paddlex import analyze_document, check_paddlex_health + + paddlex_uri = os.getenv("PADDLEX_URI", "http://localhost:8080") + health_check_response = check_paddlex_health(paddlex_uri) + if not health_check_response.ok: + logger.error(f"Paddlex OCR service health check failed with {paddlex_uri}: {health_check_response.json()}") + raise RuntimeError("Paddlex OCR service health check failed. Please check the log use `docker logs paddlex`") + + result = analyze_document(pdf_path, base_url=paddlex_uri) + + if not result["success"]: + logger.error(f"Paddlex OCR failed: {result['error']}") + return "" + + return result["full_text"] + def get_state(task_id): return GOLBAL_STATE.get(task_id, {}) diff --git a/scripts/paddlestructure/paddlex_layout_parser.py b/src/plugins/paddlex.py similarity index 76% rename from scripts/paddlestructure/paddlex_layout_parser.py rename to src/plugins/paddlex.py index 4161d71c..eebc08ad 100644 --- a/scripts/paddlestructure/paddlex_layout_parser.py +++ b/src/plugins/paddlex.py @@ -4,8 +4,8 @@ import json import base64 import os import time -from typing import Optional, Dict, Any - +from typing import Optional, Any +from src.utils import logger @@ -24,43 +24,43 @@ class PaddleXLayoutParser: def _process_file_input(self, file_input: str) -> str: # 检查是否为本地文件路径 if os.path.exists(file_input): - print(f"📁 检测到本地文件: {file_input}") - print(f"📏 文件大小: {os.path.getsize(file_input) / 1024 / 1024:.2f} MB") + logger.info(f"📁 检测到本地文件: {file_input}") + logger.info(f"📏 文件大小: {os.path.getsize(file_input) / 1024 / 1024:.2f} MB") try: # 将本地文件编码为Base64 encoded_content = self.encode_file_to_base64(file_input) - print(f"✅ 文件已编码为Base64，长度: {len(encoded_content)} 字符") + logger.info(f"✅ 文件已编码为Base64，长度: {len(encoded_content)} 字符") return encoded_content except Exception as e: - print(f"❌ 文件编码失败: {e}") + logger.error(f"❌ 文件编码失败: {e}") raise # 检查是否为URL elif file_input.startswith(('http://', 'https://')): - print(f"🌐 检测到URL: {file_input}") + logger.info(f"🌐 检测到URL: {file_input}") return file_input # 否则假设为Base64编码内容 else: - print(f"📝 假设为Base64编码内容，长度: {len(file_input)} 字符") + logger.info(f"📝 假设为Base64编码内容，长度: {len(file_input)} 字符") return file_input def layout_parsing(self, file_input: str, - file_type: Optional[int] = None, - use_textline_orientation: Optional[bool] = None, - use_seal_recognition: Optional[bool] = None, - use_table_recognition: Optional[bool] = None, - use_formula_recognition: Optional[bool] = None, - use_chart_recognition: Optional[bool] = None, - use_region_detection: Optional[bool] = None, - layout_threshold: Optional[float] = None, - layout_nms: Optional[bool] = None, - use_doc_orientation_classify: Optional[bool] = True, - use_doc_unwarping: Optional[bool] = False, - use_wired_table_cells_trans_to_html: Optional[bool] = True, # 是否启用无有线表单元格检测结果直转HTML，默认False，启用则直接基于有线表单元格检测结果的几何关系构建HTML。 - **kwargs) -> Dict[str, Any]: + file_type: int | None = None, + use_textline_orientation: bool | None = None, + use_seal_recognition: bool | None = None, + use_table_recognition: bool | None = None, + use_formula_recognition: bool | None = None, + use_chart_recognition: bool | None = None, + use_region_detection: bool | None = None, + layout_threshold: float | None = None, + layout_nms: bool | None = None, + use_doc_orientation_classify: bool = True, + use_doc_unwarping: bool | None = False, + use_wired_table_cells_trans_to_html: bool = True, # 启用则直接基于有线表单元格检测结果的几何关系构建HTML。 + **kwargs) -> dict[str, Any]: """ 调用版面解析API：https://paddlepaddle.github.io/PaddleX/latest/pipeline_usage/tutorials/ocr_pipelines/PP-StructureV3.html#22-python """ @@ -104,28 +104,30 @@ class PaddleXLayoutParser: if response.status_code == 200: result = response.json() - print("✅ 请求成功!") + logger.info("✅ 请求成功!") return result else: - print("❌ 请求失败!") + logger.error("❌ 请求失败!") try: error_result = response.json() - print(f"错误信息: {json.dumps(error_result, indent=2, ensure_ascii=False)}") + logger.error(f"错误信息: {json.dumps(error_result, indent=2, ensure_ascii=False)}") return error_result - except: - print(f"响应内容: {response.text}") - return {"error": response.text, "status_code": response.status_code} + except Exception as e: + logger.error(f"响应内容: {response.text}") + return {"error": f"{e}: {response.text}", "status_code": response.status_code} except requests.exceptions.RequestException as e: - print(f"❌ 网络请求异常: {e}") + health_check_response = requests.get(f"{self.base_url}/health", timeout=5) + logger.error(f"❌ 网络请求异常: {e}: {health_check_response.json()}") return {"error": str(e)} + except Exception as e: - print(f"❌ 其他异常: {e}") + logger.error(f"❌ 其他异常: {e}") return {"error": str(e)} -def _parse_recognition_result(api_result: Dict[str, Any], file_path: str) -> Dict[str, Any]: +def _parse_recognition_result(api_result: dict[str, Any], file_path: str) -> dict[str, Any]: # 基本信息 parsed_result = { "success": True, @@ -238,7 +240,7 @@ def _parse_recognition_result(api_result: Dict[str, Any], file_path: str) -> Dic return parsed_result -def analyze_document(file_path: str) -> Dict[str, Any]: +def analyze_document(file_path: str, base_url: str = "http://localhost:8080") -> dict[str, Any]: # 检查文件是否存在 if not os.path.exists(file_path): @@ -249,7 +251,7 @@ def analyze_document(file_path: str) -> Dict[str, Any]: } # 初始化客户端 - client = PaddleXLayoutParser() + client = PaddleXLayoutParser(base_url=base_url) # 判断文件类型 file_ext = os.path.splitext(file_path)[1].lower() @@ -264,9 +266,9 @@ def analyze_document(file_path: str) -> Dict[str, Any]: "file_path": file_path } - print(f"📄 开始分析文档: {os.path.basename(file_path)}") - print(f"📏 文件大小: {os.path.getsize(file_path) / 1024 / 1024:.2f} MB") - print(f"📋 文件类型: {'PDF' if file_type == 0 else '图片'}") + logger.info(f"📄 开始分析文档: {os.path.basename(file_path)}") + logger.info(f"📏 文件大小: {os.path.getsize(file_path) / 1024 / 1024:.2f} MB") + logger.info(f"📋 文件类型: {'PDF' if file_type == 0 else '图片'}") try: # 调用API进行识别 @@ -291,3 +293,7 @@ def analyze_document(file_path: str) -> Dict[str, Any]: "error": f"处理异常: {str(e)}", "file_path": file_path } + + +def check_paddlex_health(base_url: str = "http://localhost:8080") -> bool: + return requests.get(f"{base_url}/health", timeout=5) diff --git a/test/data/ocr_test/PixPin_2025-06-19_23-42-17.png b/test/data/ocr_test/PixPin_2025-06-19_23-42-17.png new file mode 100644 index 00000000..9fdaa62f Binary files /dev/null and b/test/data/ocr_test/PixPin_2025-06-19_23-42-17.png differ diff --git a/web/src/views/DataBaseInfoView.vue b/web/src/views/DataBaseInfoView.vue index 2c42f7bf..8996ee1f 100644 --- a/web/src/views/DataBaseInfoView.vue +++ b/web/src/views/DataBaseInfoView.vue @@ -445,6 +445,7 @@ const enable_ocr_options = ref([ { value: 'disable', payload: { title: '不启用' } }, { value: 'onnx_rapid_ocr', payload: { title: 'ONNX with RapidOCR' } }, { value: 'mineru_ocr', payload: { title: 'MinerU OCR' } }, + { value: 'paddlex_ocr', payload: { title: 'Paddlex OCR' } }, ]) const use_rewrite_queryOptions = ref([ @@ -811,7 +812,7 @@ const chunkFiles = () => { .then(data => { console.log('文件处理结果:', data) if (data.status === 'success') { - message.success(data.message || '文件已提交处理，请稍后在列表刷新查看状态'); + message.info(data.message || '文件已提交处理，请稍后在列表刷新查看状态'); fileList.value = []; // 清空已上传文件列表 addFilesModalVisible.value = false; // 关闭弹窗 getDatabaseInfo(); // 刷新数据库信息以显示新文件及其状态