feat(mineru_parser): 添加对docker 部署的mineru ZIP格式响应和图片解析的支持

添加对MinerU解析器返回ZIP格式响应的处理逻辑,支持包含图片的文档解析。同时从环境变量读取MINERU_VL_SERVER配置,替代直接使用参数中的server_url,提高配置安全性。
This commit is contained in:
Wenjie Zhang 2025-11-27 12:41:28 +08:00
parent 18d46ca04a
commit e369f751ad

View File

@ -5,13 +5,16 @@ MinerU 文档解析器
""" """
import os import os
import tempfile
import base64
import time import time
from pathlib import Path from pathlib import Path
import requests import requests
from src.knowledge.indexing import _process_zip_file
from src.plugins.document_processor_base import BaseDocumentProcessor, DocumentParserException from src.plugins.document_processor_base import BaseDocumentProcessor, DocumentParserException
from src.utils import logger from src.utils import logger, hashstr
class MinerUParser(BaseDocumentProcessor): class MinerUParser(BaseDocumentProcessor):
@ -126,11 +129,16 @@ class MinerUParser(BaseDocumentProcessor):
"parse_method": params.get("parse_method", "auto"), "parse_method": params.get("parse_method", "auto"),
# 固定返回 markdown 格式 # 固定返回 markdown 格式
"return_md": True, "return_md": True,
# 添加图片解析支持
"response_format_zip": True,
"return_images": True,
} }
# vlm-http-client 后端需要 server_url # vlm-http-client 后端需要 server_url
if data["backend"] == "vlm-http-client": if data["backend"] == "vlm-http-client":
data["server_url"] = params.get("server_url") mineru_vl_server = os.environ.get("MINERU_VL_SERVER")
assert mineru_vl_server, "MINERU_VL_SERVER 环境变量未配置"
data["server_url"] = mineru_vl_server
try: try:
start_time = time.time() start_time = time.time()
@ -170,8 +178,26 @@ class MinerUParser(BaseDocumentProcessor):
try: try:
result = response.json() result = response.json()
# 简化响应处理 - 专注于获取 markdown 内容 # 检查ZIP格式响应
if isinstance(result, dict) and "results" in result: if isinstance(result, dict) and "zip_data" in result:
zip_data = result["zip_data"]
if isinstance(zip_data, str):
zip_data = base64.b64decode(zip_data)
# 保存到临时文件并处理
with tempfile.NamedTemporaryFile(suffix=".zip", delete=False) as tmp_zip:
tmp_zip.write(zip_data)
tmp_zip.flush()
try:
processed = _process_zip_file(tmp_zip.name, params.get("db_id"))
text = processed["markdown_content"]
finally:
os.unlink(tmp_zip.name)
# 标准JSON格式响应
elif isinstance(result, dict) and "results" in result:
file_result = list(result["results"].values())[0] file_result = list(result["results"].values())[0]
text = file_result.get("md") or file_result.get("markdown") or file_result.get("md_content", "") text = file_result.get("md") or file_result.get("markdown") or file_result.get("md_content", "")
else: else: