refactor(mineru_parser): 修复 zip 解析的 bug
移除对 base64 编码 ZIP 数据和 JSON 格式响应的支持,直接处理响应内容 添加调试日志记录响应状态和内容类型 优化错误处理日志记录
This commit is contained in:
parent
e369f751ad
commit
18f641ea39
@ -6,7 +6,6 @@ MinerU 文档解析器
|
|||||||
|
|
||||||
import os
|
import os
|
||||||
import tempfile
|
import tempfile
|
||||||
import base64
|
|
||||||
import time
|
import time
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
@ -14,7 +13,7 @@ import requests
|
|||||||
|
|
||||||
from src.knowledge.indexing import _process_zip_file
|
from src.knowledge.indexing import _process_zip_file
|
||||||
from src.plugins.document_processor_base import BaseDocumentProcessor, DocumentParserException
|
from src.plugins.document_processor_base import BaseDocumentProcessor, DocumentParserException
|
||||||
from src.utils import logger, hashstr
|
from src.utils import logger
|
||||||
|
|
||||||
|
|
||||||
class MinerUParser(BaseDocumentProcessor):
|
class MinerUParser(BaseDocumentProcessor):
|
||||||
@ -160,6 +159,8 @@ class MinerUParser(BaseDocumentProcessor):
|
|||||||
)
|
)
|
||||||
|
|
||||||
# 检查响应状态
|
# 检查响应状态
|
||||||
|
logger.debug(f"MinerU 响应状态: {response.status_code}, Content-Type: {response.headers.get('content-type', 'unknown')}")
|
||||||
|
|
||||||
if response.status_code != 200:
|
if response.status_code != 200:
|
||||||
error_detail = "未知错误"
|
error_detail = "未知错误"
|
||||||
try:
|
try:
|
||||||
@ -168,6 +169,7 @@ class MinerUParser(BaseDocumentProcessor):
|
|||||||
except Exception:
|
except Exception:
|
||||||
error_detail = response.text or f"HTTP {response.status_code}"
|
error_detail = response.text or f"HTTP {response.status_code}"
|
||||||
|
|
||||||
|
logger.error(f"MinerU HTTP错误 {response.status_code}: {error_detail}")
|
||||||
raise DocumentParserException(
|
raise DocumentParserException(
|
||||||
f"MinerU 处理失败: {error_detail}",
|
f"MinerU 处理失败: {error_detail}",
|
||||||
self.get_service_name(),
|
self.get_service_name(),
|
||||||
@ -176,36 +178,22 @@ class MinerUParser(BaseDocumentProcessor):
|
|||||||
|
|
||||||
# 解析响应
|
# 解析响应
|
||||||
try:
|
try:
|
||||||
result = response.json()
|
# 直接从响应内容获取 ZIP 数据
|
||||||
|
zip_data = response.content
|
||||||
|
|
||||||
# 检查ZIP格式响应
|
# 保存到临时文件并处理
|
||||||
if isinstance(result, dict) and "zip_data" in result:
|
with tempfile.NamedTemporaryFile(suffix=".zip", delete=False) as tmp_zip:
|
||||||
|
tmp_zip.write(zip_data)
|
||||||
|
tmp_zip.flush()
|
||||||
|
|
||||||
zip_data = result["zip_data"]
|
try:
|
||||||
if isinstance(zip_data, str):
|
processed = _process_zip_file(tmp_zip.name, params.get("db_id"))
|
||||||
zip_data = base64.b64decode(zip_data)
|
text = processed["markdown_content"]
|
||||||
|
finally:
|
||||||
# 保存到临时文件并处理
|
os.unlink(tmp_zip.name)
|
||||||
with tempfile.NamedTemporaryFile(suffix=".zip", delete=False) as tmp_zip:
|
|
||||||
tmp_zip.write(zip_data)
|
|
||||||
tmp_zip.flush()
|
|
||||||
|
|
||||||
try:
|
|
||||||
processed = _process_zip_file(tmp_zip.name, params.get("db_id"))
|
|
||||||
text = processed["markdown_content"]
|
|
||||||
finally:
|
|
||||||
os.unlink(tmp_zip.name)
|
|
||||||
|
|
||||||
# 标准JSON格式响应
|
|
||||||
elif isinstance(result, dict) and "results" in result:
|
|
||||||
file_result = list(result["results"].values())[0]
|
|
||||||
text = file_result.get("md") or file_result.get("markdown") or file_result.get("md_content", "")
|
|
||||||
else:
|
|
||||||
# 降级处理
|
|
||||||
text = str(result) if result else ""
|
|
||||||
|
|
||||||
if not text:
|
if not text:
|
||||||
logger.error(f"MinerU 未返回任何文本内容: {str(result)[:500]}")
|
logger.error("MinerU 未返回任何文本内容")
|
||||||
raise DocumentParserException(
|
raise DocumentParserException(
|
||||||
"MinerU 未返回任何文本内容",
|
"MinerU 未返回任何文本内容",
|
||||||
self.get_service_name(),
|
self.get_service_name(),
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user