feat(mineru_parser): 添加对docker 部署的mineru ZIP格式响应和图片解析的支持

添加对MinerU解析器返回ZIP格式响应的处理逻辑,支持包含图片的文档解析。同时从环境变量读取MINERU_VL_SERVER配置,替代直接使用参数中的server_url,提高配置安全性。
This commit is contained in:
Wenjie Zhang 2025-11-27 12:41:28 +08:00
parent 18d46ca04a
commit e369f751ad

View File

@ -5,13 +5,16 @@ MinerU 文档解析器
"""
import os
import tempfile
import base64
import time
from pathlib import Path
import requests
from src.knowledge.indexing import _process_zip_file
from src.plugins.document_processor_base import BaseDocumentProcessor, DocumentParserException
from src.utils import logger
from src.utils import logger, hashstr
class MinerUParser(BaseDocumentProcessor):
@ -126,11 +129,16 @@ class MinerUParser(BaseDocumentProcessor):
"parse_method": params.get("parse_method", "auto"),
# 固定返回 markdown 格式
"return_md": True,
# 添加图片解析支持
"response_format_zip": True,
"return_images": True,
}
# vlm-http-client 后端需要 server_url
if data["backend"] == "vlm-http-client":
data["server_url"] = params.get("server_url")
mineru_vl_server = os.environ.get("MINERU_VL_SERVER")
assert mineru_vl_server, "MINERU_VL_SERVER 环境变量未配置"
data["server_url"] = mineru_vl_server
try:
start_time = time.time()
@ -170,8 +178,26 @@ class MinerUParser(BaseDocumentProcessor):
try:
result = response.json()
# 简化响应处理 - 专注于获取 markdown 内容
if isinstance(result, dict) and "results" in result:
# 检查ZIP格式响应
if isinstance(result, dict) and "zip_data" in result:
zip_data = result["zip_data"]
if isinstance(zip_data, str):
zip_data = base64.b64decode(zip_data)
# 保存到临时文件并处理
with tempfile.NamedTemporaryFile(suffix=".zip", delete=False) as tmp_zip:
tmp_zip.write(zip_data)
tmp_zip.flush()
try:
processed = _process_zip_file(tmp_zip.name, params.get("db_id"))
text = processed["markdown_content"]
finally:
os.unlink(tmp_zip.name)
# 标准JSON格式响应
elif isinstance(result, dict) and "results" in result:
file_result = list(result["results"].values())[0]
text = file_result.get("md") or file_result.get("markdown") or file_result.get("md_content", "")
else: