feat(mineru_parser): 添加对docker 部署的mineru ZIP格式响应和图片解析的支持
添加对MinerU解析器返回ZIP格式响应的处理逻辑,支持包含图片的文档解析。同时从环境变量读取MINERU_VL_SERVER配置,替代直接使用参数中的server_url,提高配置安全性。
This commit is contained in:
parent
18d46ca04a
commit
e369f751ad
@ -5,13 +5,16 @@ MinerU 文档解析器
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
import tempfile
|
||||||
|
import base64
|
||||||
import time
|
import time
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
|
|
||||||
|
from src.knowledge.indexing import _process_zip_file
|
||||||
from src.plugins.document_processor_base import BaseDocumentProcessor, DocumentParserException
|
from src.plugins.document_processor_base import BaseDocumentProcessor, DocumentParserException
|
||||||
from src.utils import logger
|
from src.utils import logger, hashstr
|
||||||
|
|
||||||
|
|
||||||
class MinerUParser(BaseDocumentProcessor):
|
class MinerUParser(BaseDocumentProcessor):
|
||||||
@ -126,11 +129,16 @@ class MinerUParser(BaseDocumentProcessor):
|
|||||||
"parse_method": params.get("parse_method", "auto"),
|
"parse_method": params.get("parse_method", "auto"),
|
||||||
# 固定返回 markdown 格式
|
# 固定返回 markdown 格式
|
||||||
"return_md": True,
|
"return_md": True,
|
||||||
|
# 添加图片解析支持
|
||||||
|
"response_format_zip": True,
|
||||||
|
"return_images": True,
|
||||||
}
|
}
|
||||||
|
|
||||||
# vlm-http-client 后端需要 server_url
|
# vlm-http-client 后端需要 server_url
|
||||||
if data["backend"] == "vlm-http-client":
|
if data["backend"] == "vlm-http-client":
|
||||||
data["server_url"] = params.get("server_url")
|
mineru_vl_server = os.environ.get("MINERU_VL_SERVER")
|
||||||
|
assert mineru_vl_server, "MINERU_VL_SERVER 环境变量未配置"
|
||||||
|
data["server_url"] = mineru_vl_server
|
||||||
|
|
||||||
try:
|
try:
|
||||||
start_time = time.time()
|
start_time = time.time()
|
||||||
@ -170,8 +178,26 @@ class MinerUParser(BaseDocumentProcessor):
|
|||||||
try:
|
try:
|
||||||
result = response.json()
|
result = response.json()
|
||||||
|
|
||||||
# 简化响应处理 - 专注于获取 markdown 内容
|
# 检查ZIP格式响应
|
||||||
if isinstance(result, dict) and "results" in result:
|
if isinstance(result, dict) and "zip_data" in result:
|
||||||
|
|
||||||
|
zip_data = result["zip_data"]
|
||||||
|
if isinstance(zip_data, str):
|
||||||
|
zip_data = base64.b64decode(zip_data)
|
||||||
|
|
||||||
|
# 保存到临时文件并处理
|
||||||
|
with tempfile.NamedTemporaryFile(suffix=".zip", delete=False) as tmp_zip:
|
||||||
|
tmp_zip.write(zip_data)
|
||||||
|
tmp_zip.flush()
|
||||||
|
|
||||||
|
try:
|
||||||
|
processed = _process_zip_file(tmp_zip.name, params.get("db_id"))
|
||||||
|
text = processed["markdown_content"]
|
||||||
|
finally:
|
||||||
|
os.unlink(tmp_zip.name)
|
||||||
|
|
||||||
|
# 标准JSON格式响应
|
||||||
|
elif isinstance(result, dict) and "results" in result:
|
||||||
file_result = list(result["results"].values())[0]
|
file_result = list(result["results"].values())[0]
|
||||||
text = file_result.get("md") or file_result.get("markdown") or file_result.get("md_content", "")
|
text = file_result.get("md") or file_result.get("markdown") or file_result.get("md_content", "")
|
||||||
else:
|
else:
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user