feat(doc): 更新 MinerU 解析器参数，修改 Docker 配置，优化文档说明
This commit is contained in:
Wenjie Zhang
parent
947a861411
commit
86f92f7f3f
@ -98,13 +98,14 @@ class MinerUParser(BaseDocumentProcessor):
|
||||
file_path: 文件路径
|
||||
params: 处理参数
|
||||
- lang_list: 语言列表 (默认: ["ch"])
|
||||
- backend: 后端类型 (默认: "pipeline", 支持 "vlm-*" 系列)
|
||||
- backend: 后端类型 (默认: "hybrid-auto-engine")
|
||||
- parse_method: 解析方法 (默认: "auto")
|
||||
- start_page_id: 起始页码 (默认: 0)
|
||||
- end_page_id: 结束页码 (默认: 99999)
|
||||
- formula_enable: 启用公式解析 (默认: True)
|
||||
- table_enable: 启用表格解析 (默认: True)
|
||||
- server_url: VLM 服务器地址 (vlm-http-client 时需要)
|
||||
- image_analysis: 启用图像/图表解析 (默认: True)
|
||||
- server_url: OpenAI 兼容服务地址 (*-http-client 后端时可选)
|
||||
|
||||
Returns:
|
||||
str: 提取的 Markdown 文本
|
||||
@ -121,23 +122,23 @@ class MinerUParser(BaseDocumentProcessor):
|
||||
# 解析参数
|
||||
params = params or {}
|
||||
|
||||
# 构建请求数据 - 只保留核心参数
|
||||
data = {
|
||||
"lang_list": params.get("lang_list", ["ch"]),
|
||||
"backend": params.get("backend", "vlm-http-client"),
|
||||
"backend": params.get("backend", "hybrid-auto-engine"),
|
||||
"parse_method": params.get("parse_method", "auto"),
|
||||
# 固定返回 markdown 格式
|
||||
"formula_enable": params.get("formula_enable", True),
|
||||
"table_enable": params.get("table_enable", True),
|
||||
"image_analysis": params.get("image_analysis", True),
|
||||
"start_page_id": params.get("start_page_id", 0),
|
||||
"end_page_id": params.get("end_page_id", 99999),
|
||||
"return_md": True,
|
||||
# 添加图片解析支持
|
||||
"response_format_zip": True,
|
||||
"return_images": True,
|
||||
}
|
||||
|
||||
# vlm-http-client 后端需要 server_url
|
||||
if data["backend"] == "vlm-http-client":
|
||||
mineru_vl_server = os.environ.get("MINERU_VL_SERVER")
|
||||
assert mineru_vl_server, "MINERU_VL_SERVER 环境变量未配置"
|
||||
data["server_url"] = mineru_vl_server
|
||||
server_url = params.get("server_url")
|
||||
if server_url:
|
||||
data["server_url"] = server_url
|
||||
|
||||
try:
|
||||
start_time = time.time()
|
||||
@ -155,7 +156,7 @@ class MinerUParser(BaseDocumentProcessor):
|
||||
self.parse_endpoint,
|
||||
files=files,
|
||||
data=data,
|
||||
timeout=os.environ.get("MINERU_TIMEOUT", 1800), # 30分钟超时
|
||||
timeout=int(os.environ.get("MINERU_TIMEOUT", 1800)), # 30分钟超时
|
||||
)
|
||||
|
||||
# 检查响应状态
|
||||
|
||||
@ -3,7 +3,7 @@ x-api-worker-env: &api-worker-env
|
||||
LITE_MODE: ${LITE_MODE:-}
|
||||
RUNNING_IN_DOCKER: "true"
|
||||
YUXI_ENV: production
|
||||
MODEL_DIR_IN_DOCKER: /models
|
||||
MODEL_DIR_IN_DOCKER: /app/models
|
||||
PYTHONPATH: /app/package
|
||||
POSTGRES_URL: ${POSTGRES_URL:-postgresql+asyncpg://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@postgres:5432/${POSTGRES_DB:-yuxi_know}}
|
||||
REDIS_URL: ${REDIS_URL:-redis://redis:6379/0}
|
||||
@ -15,7 +15,6 @@ x-api-worker-env: &api-worker-env
|
||||
MILVUS_URI: ${MILVUS_URI:-http://milvus:19530}
|
||||
MILVUS_DB_NAME: ${MILVUS_DB_NAME:-default}
|
||||
MILVUS_TOKEN: ${MILVUS_TOKEN:-}
|
||||
MINERU_VL_SERVER: ${MINERU_VL_SERVER:-http://mineru-vllm-server:30000}
|
||||
MINERU_API_URI: ${MINERU_API_URI:-http://mineru-api:30001}
|
||||
PADDLEX_URI: ${PADDLEX_URI:-http://paddlex:8080}
|
||||
MINIO_URI: ${MINIO_URI:-http://minio:9000}
|
||||
@ -283,39 +282,6 @@ services:
|
||||
- app-network
|
||||
restart: unless-stopped
|
||||
|
||||
mineru-vllm-server:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: docker/mineru.Dockerfile
|
||||
image: mineru-vllm:latest
|
||||
container_name: mineru-vllm-server
|
||||
profiles:
|
||||
- all
|
||||
env_file:
|
||||
- .env.prod
|
||||
environment:
|
||||
MINERU_MODEL_SOURCE: local
|
||||
entrypoint: mineru-vllm-server
|
||||
command:
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
ulimits:
|
||||
memlock: -1
|
||||
stack: 67108864
|
||||
ipc: host
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -f http://localhost:30000/health || exit 1"]
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
device_ids: ["0"]
|
||||
capabilities: [gpu]
|
||||
networks:
|
||||
- app-network
|
||||
restart: unless-stopped
|
||||
|
||||
mineru-api:
|
||||
build:
|
||||
context: .
|
||||
@ -334,13 +300,25 @@ services:
|
||||
command:
|
||||
--host 0.0.0.0
|
||||
--port 30001
|
||||
# --allow-public-http-client # Disabled by default; when binding to 0.0.0.0 or ::, this re-enables *-http-client backends and server_url. Enable only if you accept the SSRF risk.
|
||||
# parameters for vllm-engine
|
||||
# --gpu-memory-utilization 0.5 # If encountering VRAM shortage, reduce the KV cache size by this parameter; if VRAM issues persist, try lowering it further to `0.4` or below.
|
||||
ulimits:
|
||||
memlock: -1
|
||||
stack: 67108864
|
||||
ipc: host
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -f http://localhost:30001/health || exit 1"]
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
device_ids: ["0"] # Modify for multiple GPUs: ["0", "1"]
|
||||
capabilities: [gpu]
|
||||
networks:
|
||||
- app-network
|
||||
restart: unless-stopped
|
||||
|
||||
paddlex:
|
||||
build:
|
||||
|
||||
@ -15,7 +15,6 @@ x-api-worker-env: &api-worker-env
|
||||
MILVUS_TOKEN: ${MILVUS_TOKEN:-}
|
||||
MINIO_URI: ${MINIO_URI:-http://minio:9000}
|
||||
# 文档解析
|
||||
MINERU_VL_SERVER: ${MINERU_VL_SERVER:-http://mineru-vllm-server:30000}
|
||||
MINERU_API_URI: ${MINERU_API_URI:-http://mineru-api:30001}
|
||||
PADDLEX_URI: ${PADDLEX_URI:-http://paddlex:8080}
|
||||
# Sandbox related
|
||||
@ -340,44 +339,6 @@ services:
|
||||
restart: unless-stopped
|
||||
|
||||
# lastest version: wget https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/docker/compose.yaml
|
||||
mineru-vllm-server:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: docker/mineru.Dockerfile
|
||||
image: mineru-vllm:latest
|
||||
container_name: mineru-vllm-server
|
||||
profiles:
|
||||
- all
|
||||
env_file:
|
||||
- .env
|
||||
ports:
|
||||
- 30000:30000
|
||||
environment:
|
||||
MINERU_MODEL_SOURCE: local
|
||||
entrypoint: mineru-vllm-server
|
||||
command:
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
# parameters for vllm-engine
|
||||
# --data-parallel-size 2 # If using multiple GPUs, increase throughput using vllm's multi-GPU parallel mode
|
||||
# --gpu-memory-utilization 0.5 # If running on a single GPU and encountering VRAM shortage, reduce the KV cache size by this parameter, if VRAM issues persist, try lowering it further to `0.4` or below.
|
||||
ulimits:
|
||||
memlock: -1
|
||||
stack: 67108864
|
||||
ipc: host
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -f http://localhost:30000/health || exit 1"]
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
device_ids: ["0"]
|
||||
capabilities: [gpu]
|
||||
networks:
|
||||
- app-network
|
||||
restart: unless-stopped
|
||||
|
||||
mineru-api:
|
||||
build:
|
||||
context: .
|
||||
@ -392,17 +353,30 @@ services:
|
||||
- 30001:30001
|
||||
environment:
|
||||
MINERU_MODEL_SOURCE: local
|
||||
VLLM_USE_V1: "1"
|
||||
entrypoint: mineru-api
|
||||
command:
|
||||
--host 0.0.0.0
|
||||
--port 30001
|
||||
# --allow-public-http-client # Disabled by default; when binding to 0.0.0.0 or ::, this re-enables *-http-client backends and server_url. Enable only if you accept the SSRF risk.
|
||||
# parameters for vllm-engine
|
||||
# --gpu-memory-utilization 0.5 # If encountering VRAM shortage, reduce the KV cache size by this parameter; if VRAM issues persist, try lowering it further to `0.4` or below.
|
||||
ulimits:
|
||||
memlock: -1
|
||||
stack: 67108864
|
||||
ipc: host
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -f http://localhost:30001/health || exit 1"]
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
device_ids: ["0"] # Modify for multiple GPUs: ["0", "1"]
|
||||
capabilities: [gpu]
|
||||
networks:
|
||||
- app-network
|
||||
restart: unless-stopped
|
||||
|
||||
paddlex:
|
||||
build:
|
||||
|
||||
@ -1,15 +1,8 @@
|
||||
# Use DaoCloud mirrored vllm image for China region for gpu with Ampere architecture and above (Compute Capability>=8.0)
|
||||
# https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/docker/china/Dockerfile
|
||||
# Use DaoCloud mirrored vllm image for China region for gpu with Volta、Turing、Ampere、Ada Lovelace、Hopper、Blackwell architecture (7.0 <= Compute Capability <= 12.0)
|
||||
# Compute Capability version query (https://developer.nvidia.com/cuda-gpus)
|
||||
FROM docker.m.daocloud.io/vllm/vllm-openai:v0.10.1.1
|
||||
|
||||
# Use the official vllm image
|
||||
# FROM vllm/vllm-openai:v0.10.1.1
|
||||
|
||||
# Use DaoCloud mirrored vllm image for China region for gpu with Turing architecture and below (Compute Capability<8.0)
|
||||
# FROM docker.m.daocloud.io/vllm/vllm-openai:v0.10.2
|
||||
|
||||
# Use the official vllm image
|
||||
# FROM vllm/vllm-openai:v0.10.2
|
||||
# support x86_64 architecture and ARM(AArch64) architecture
|
||||
FROM docker.m.daocloud.io/vllm/vllm-openai:v0.11.2
|
||||
|
||||
# Install libgl for opencv support & Noto fonts for Chinese characters
|
||||
RUN apt-get update && \
|
||||
@ -23,7 +16,7 @@ RUN apt-get update && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install mineru latest
|
||||
RUN python3 -m pip install -U 'mineru[core]' -i https://mirrors.aliyun.com/pypi/simple --break-system-packages && \
|
||||
RUN python3 -m pip install -U 'mineru[core]>=3.0.0' -i https://mirrors.aliyun.com/pypi/simple --break-system-packages && \
|
||||
python3 -m pip cache purge
|
||||
|
||||
# Download models and update the configuration file
|
||||
|
||||
@ -76,35 +76,39 @@ docker compose up -d api
|
||||
|
||||
### MinerU（高精度）
|
||||
|
||||
```env
|
||||
# .env 配置
|
||||
MINERU_VL_SERVER=http://localhost:30000
|
||||
MINERU_API_URI=http://localhost:30001
|
||||
首先从官网下载最新的 docker-compose 文件：
|
||||
|
||||
# 启动服务（需要 GPU）
|
||||
docker compose up mineru-vllm-server mineru-api -d
|
||||
```bash
|
||||
wget https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/docker/compose.yaml -O docker/mineru.compose.yml
|
||||
```
|
||||
|
||||
启动服务（需要 GPU）
|
||||
|
||||
```bash
|
||||
docker compose -f docker/mineru.compose.yml --profile openai-server up -d
|
||||
```
|
||||
|
||||
### MinerU Official（云服务）
|
||||
|
||||
从 [MinerU 官网](https://mineru.net) 获取 API 密钥，在 .env 配置环境变量
|
||||
|
||||
```env
|
||||
# .env 配置
|
||||
MINERU_API_KEY=your-api-key-here
|
||||
```
|
||||
|
||||
从 [MinerU 官网](https://mineru.net) 获取 API 密钥。
|
||||
|
||||
### PP-Structure-V3（结构化）
|
||||
|
||||
启动服务（需要 GPU）
|
||||
|
||||
```bash
|
||||
# 启动服务（需要 GPU）
|
||||
docker compose up paddlex -d
|
||||
```
|
||||
|
||||
### DeepSeek OCR（简单云服务）
|
||||
|
||||
在 .env 配置（使用已有的 SiliconFlow 密钥）
|
||||
|
||||
```env
|
||||
# .env 配置（使用已有的 SiliconFlow 密钥）
|
||||
SILICONFLOW_API_KEY=your-api-key-here
|
||||
```
|
||||
|
||||
@ -114,7 +118,7 @@ SILICONFLOW_API_KEY=your-api-key-here
|
||||
|
||||
在 `.env` 中设置服务器 IP：
|
||||
|
||||
```env
|
||||
```
|
||||
HOST_IP=your_server_ip
|
||||
```
|
||||
|
||||
|
||||
@ -66,7 +66,7 @@
|
||||
</div>
|
||||
<div class="col-item" v-if="uploadMode !== 'url'">
|
||||
<div class="setting-label">
|
||||
OCR 引擎
|
||||
OCR 引擎（仅应用于 PDF/图片文件）
|
||||
<a-tooltip title="检查服务状态">
|
||||
<ReloadOutlined
|
||||
class="action-icon refresh-icon"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user