From 292237e5e2a1552543bb75662460cfaeb16ba384 Mon Sep 17 00:00:00 2001 From: Wenjie Zhang Date: Mon, 17 Nov 2025 17:08:53 +0800 Subject: [PATCH] =?UTF-8?q?feat(mineru=5Fofficial=5Fparser):=20=E5=9F=BA?= =?UTF-8?q?=E4=BA=8Emineru=E5=AE=98=E6=96=B9API=E7=9A=84=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=9B=BE=E7=89=87=E8=87=AA=E5=8A=A8=E4=B8=8A?= =?UTF-8?q?=E4=BC=A0=E5=88=B0=E6=96=87=E4=BB=B6=E6=95=B0=E6=8D=AE=E5=BA=93?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/plugins/mineru_official_parser.py | 50 +++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/src/plugins/mineru_official_parser.py b/src/plugins/mineru_official_parser.py index f3f4ee5c..d312a487 100644 --- a/src/plugins/mineru_official_parser.py +++ b/src/plugins/mineru_official_parser.py @@ -142,8 +142,38 @@ class MinerUOfficialParser(BaseDocumentProcessor): result = self._poll_batch_result(batch_id) logger.info(f"任务完成,状态: {result['state']}") - # 步骤 3: 下载并解压结果 - text = self._download_and_extract(result.get("full_zip_url")) + zip_url = result.get("full_zip_url") + + try: + zip_path = self._download_zip(zip_url) + except Exception: + text = self._download_and_extract(zip_url) + processing_time = time.time() - start_time + logger.info( + f"MinerU Official 处理成功: {os.path.basename(file_path)} - {len(text)} 字符 ({processing_time:.2f}s)" + ) + return text + + from src.knowledge.indexing import _process_zip_file + + try: + processed = _process_zip_file(zip_path, params.get("db_id") or "ocr-test") + text = processed["markdown_content"] + except Exception: + import zipfile + text = "" + logger.error(f"从 zip 文件中提取 full.md 失败: {zip_path},使用第一个 md 文件") + with zipfile.ZipFile(zip_path, "r") as zf: + md_files = [n for n in zf.namelist() if n.lower().endswith(".md")] + if md_files: + md_file = next((n for n in md_files if Path(n).name == "full.md"), md_files[0]) + with zf.open(md_file) as f: + text = f.read().decode("utf-8") + finally: + try: + os.unlink(zip_path) + except Exception: + pass processing_time = time.time() - start_time logger.info( @@ -310,3 +340,19 @@ class MinerUOfficialParser(BaseDocumentProcessor): finally: os.unlink(tmp_file.name) + + def _download_zip(self, zip_url: str) -> str: + """下载结果ZIP到临时文件并返回路径""" + if not zip_url: + raise DocumentParserException("未获取到结果下载链接", self.get_service_name(), "no_download_url") + response = requests.get(zip_url, timeout=60) + if response.status_code != 200: + raise DocumentParserException( + f"下载结果失败: HTTP {response.status_code}", self.get_service_name(), "download_failed" + ) + import tempfile + + with tempfile.NamedTemporaryFile(delete=False, suffix=".zip") as tmp_file: + tmp_file.write(response.content) + tmp_file.flush() + return tmp_file.name