feat(doc): 更新 MinerU 解析器参数,修改 Docker 配置,优化文档说明
This commit is contained in:
parent
947a861411
commit
86f92f7f3f
@ -98,13 +98,14 @@ class MinerUParser(BaseDocumentProcessor):
|
|||||||
file_path: 文件路径
|
file_path: 文件路径
|
||||||
params: 处理参数
|
params: 处理参数
|
||||||
- lang_list: 语言列表 (默认: ["ch"])
|
- lang_list: 语言列表 (默认: ["ch"])
|
||||||
- backend: 后端类型 (默认: "pipeline", 支持 "vlm-*" 系列)
|
- backend: 后端类型 (默认: "hybrid-auto-engine")
|
||||||
- parse_method: 解析方法 (默认: "auto")
|
- parse_method: 解析方法 (默认: "auto")
|
||||||
- start_page_id: 起始页码 (默认: 0)
|
- start_page_id: 起始页码 (默认: 0)
|
||||||
- end_page_id: 结束页码 (默认: 99999)
|
- end_page_id: 结束页码 (默认: 99999)
|
||||||
- formula_enable: 启用公式解析 (默认: True)
|
- formula_enable: 启用公式解析 (默认: True)
|
||||||
- table_enable: 启用表格解析 (默认: True)
|
- table_enable: 启用表格解析 (默认: True)
|
||||||
- server_url: VLM 服务器地址 (vlm-http-client 时需要)
|
- image_analysis: 启用图像/图表解析 (默认: True)
|
||||||
|
- server_url: OpenAI 兼容服务地址 (*-http-client 后端时可选)
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
str: 提取的 Markdown 文本
|
str: 提取的 Markdown 文本
|
||||||
@ -121,23 +122,23 @@ class MinerUParser(BaseDocumentProcessor):
|
|||||||
# 解析参数
|
# 解析参数
|
||||||
params = params or {}
|
params = params or {}
|
||||||
|
|
||||||
# 构建请求数据 - 只保留核心参数
|
|
||||||
data = {
|
data = {
|
||||||
"lang_list": params.get("lang_list", ["ch"]),
|
"lang_list": params.get("lang_list", ["ch"]),
|
||||||
"backend": params.get("backend", "vlm-http-client"),
|
"backend": params.get("backend", "hybrid-auto-engine"),
|
||||||
"parse_method": params.get("parse_method", "auto"),
|
"parse_method": params.get("parse_method", "auto"),
|
||||||
# 固定返回 markdown 格式
|
"formula_enable": params.get("formula_enable", True),
|
||||||
|
"table_enable": params.get("table_enable", True),
|
||||||
|
"image_analysis": params.get("image_analysis", True),
|
||||||
|
"start_page_id": params.get("start_page_id", 0),
|
||||||
|
"end_page_id": params.get("end_page_id", 99999),
|
||||||
"return_md": True,
|
"return_md": True,
|
||||||
# 添加图片解析支持
|
|
||||||
"response_format_zip": True,
|
"response_format_zip": True,
|
||||||
"return_images": True,
|
"return_images": True,
|
||||||
}
|
}
|
||||||
|
|
||||||
# vlm-http-client 后端需要 server_url
|
server_url = params.get("server_url")
|
||||||
if data["backend"] == "vlm-http-client":
|
if server_url:
|
||||||
mineru_vl_server = os.environ.get("MINERU_VL_SERVER")
|
data["server_url"] = server_url
|
||||||
assert mineru_vl_server, "MINERU_VL_SERVER 环境变量未配置"
|
|
||||||
data["server_url"] = mineru_vl_server
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
start_time = time.time()
|
start_time = time.time()
|
||||||
@ -155,7 +156,7 @@ class MinerUParser(BaseDocumentProcessor):
|
|||||||
self.parse_endpoint,
|
self.parse_endpoint,
|
||||||
files=files,
|
files=files,
|
||||||
data=data,
|
data=data,
|
||||||
timeout=os.environ.get("MINERU_TIMEOUT", 1800), # 30分钟超时
|
timeout=int(os.environ.get("MINERU_TIMEOUT", 1800)), # 30分钟超时
|
||||||
)
|
)
|
||||||
|
|
||||||
# 检查响应状态
|
# 检查响应状态
|
||||||
|
|||||||
@ -3,7 +3,7 @@ x-api-worker-env: &api-worker-env
|
|||||||
LITE_MODE: ${LITE_MODE:-}
|
LITE_MODE: ${LITE_MODE:-}
|
||||||
RUNNING_IN_DOCKER: "true"
|
RUNNING_IN_DOCKER: "true"
|
||||||
YUXI_ENV: production
|
YUXI_ENV: production
|
||||||
MODEL_DIR_IN_DOCKER: /models
|
MODEL_DIR_IN_DOCKER: /app/models
|
||||||
PYTHONPATH: /app/package
|
PYTHONPATH: /app/package
|
||||||
POSTGRES_URL: ${POSTGRES_URL:-postgresql+asyncpg://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@postgres:5432/${POSTGRES_DB:-yuxi_know}}
|
POSTGRES_URL: ${POSTGRES_URL:-postgresql+asyncpg://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@postgres:5432/${POSTGRES_DB:-yuxi_know}}
|
||||||
REDIS_URL: ${REDIS_URL:-redis://redis:6379/0}
|
REDIS_URL: ${REDIS_URL:-redis://redis:6379/0}
|
||||||
@ -15,7 +15,6 @@ x-api-worker-env: &api-worker-env
|
|||||||
MILVUS_URI: ${MILVUS_URI:-http://milvus:19530}
|
MILVUS_URI: ${MILVUS_URI:-http://milvus:19530}
|
||||||
MILVUS_DB_NAME: ${MILVUS_DB_NAME:-default}
|
MILVUS_DB_NAME: ${MILVUS_DB_NAME:-default}
|
||||||
MILVUS_TOKEN: ${MILVUS_TOKEN:-}
|
MILVUS_TOKEN: ${MILVUS_TOKEN:-}
|
||||||
MINERU_VL_SERVER: ${MINERU_VL_SERVER:-http://mineru-vllm-server:30000}
|
|
||||||
MINERU_API_URI: ${MINERU_API_URI:-http://mineru-api:30001}
|
MINERU_API_URI: ${MINERU_API_URI:-http://mineru-api:30001}
|
||||||
PADDLEX_URI: ${PADDLEX_URI:-http://paddlex:8080}
|
PADDLEX_URI: ${PADDLEX_URI:-http://paddlex:8080}
|
||||||
MINIO_URI: ${MINIO_URI:-http://minio:9000}
|
MINIO_URI: ${MINIO_URI:-http://minio:9000}
|
||||||
@ -283,39 +282,6 @@ services:
|
|||||||
- app-network
|
- app-network
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
|
||||||
mineru-vllm-server:
|
|
||||||
build:
|
|
||||||
context: .
|
|
||||||
dockerfile: docker/mineru.Dockerfile
|
|
||||||
image: mineru-vllm:latest
|
|
||||||
container_name: mineru-vllm-server
|
|
||||||
profiles:
|
|
||||||
- all
|
|
||||||
env_file:
|
|
||||||
- .env.prod
|
|
||||||
environment:
|
|
||||||
MINERU_MODEL_SOURCE: local
|
|
||||||
entrypoint: mineru-vllm-server
|
|
||||||
command:
|
|
||||||
--host 0.0.0.0
|
|
||||||
--port 30000
|
|
||||||
ulimits:
|
|
||||||
memlock: -1
|
|
||||||
stack: 67108864
|
|
||||||
ipc: host
|
|
||||||
healthcheck:
|
|
||||||
test: ["CMD-SHELL", "curl -f http://localhost:30000/health || exit 1"]
|
|
||||||
deploy:
|
|
||||||
resources:
|
|
||||||
reservations:
|
|
||||||
devices:
|
|
||||||
- driver: nvidia
|
|
||||||
device_ids: ["0"]
|
|
||||||
capabilities: [gpu]
|
|
||||||
networks:
|
|
||||||
- app-network
|
|
||||||
restart: unless-stopped
|
|
||||||
|
|
||||||
mineru-api:
|
mineru-api:
|
||||||
build:
|
build:
|
||||||
context: .
|
context: .
|
||||||
@ -334,13 +300,25 @@ services:
|
|||||||
command:
|
command:
|
||||||
--host 0.0.0.0
|
--host 0.0.0.0
|
||||||
--port 30001
|
--port 30001
|
||||||
|
# --allow-public-http-client # Disabled by default; when binding to 0.0.0.0 or ::, this re-enables *-http-client backends and server_url. Enable only if you accept the SSRF risk.
|
||||||
|
# parameters for vllm-engine
|
||||||
|
# --gpu-memory-utilization 0.5 # If encountering VRAM shortage, reduce the KV cache size by this parameter; if VRAM issues persist, try lowering it further to `0.4` or below.
|
||||||
ulimits:
|
ulimits:
|
||||||
memlock: -1
|
memlock: -1
|
||||||
stack: 67108864
|
stack: 67108864
|
||||||
ipc: host
|
ipc: host
|
||||||
restart: unless-stopped
|
healthcheck:
|
||||||
|
test: ["CMD-SHELL", "curl -f http://localhost:30001/health || exit 1"]
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
reservations:
|
||||||
|
devices:
|
||||||
|
- driver: nvidia
|
||||||
|
device_ids: ["0"] # Modify for multiple GPUs: ["0", "1"]
|
||||||
|
capabilities: [gpu]
|
||||||
networks:
|
networks:
|
||||||
- app-network
|
- app-network
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
paddlex:
|
paddlex:
|
||||||
build:
|
build:
|
||||||
|
|||||||
@ -15,7 +15,6 @@ x-api-worker-env: &api-worker-env
|
|||||||
MILVUS_TOKEN: ${MILVUS_TOKEN:-}
|
MILVUS_TOKEN: ${MILVUS_TOKEN:-}
|
||||||
MINIO_URI: ${MINIO_URI:-http://minio:9000}
|
MINIO_URI: ${MINIO_URI:-http://minio:9000}
|
||||||
# 文档解析
|
# 文档解析
|
||||||
MINERU_VL_SERVER: ${MINERU_VL_SERVER:-http://mineru-vllm-server:30000}
|
|
||||||
MINERU_API_URI: ${MINERU_API_URI:-http://mineru-api:30001}
|
MINERU_API_URI: ${MINERU_API_URI:-http://mineru-api:30001}
|
||||||
PADDLEX_URI: ${PADDLEX_URI:-http://paddlex:8080}
|
PADDLEX_URI: ${PADDLEX_URI:-http://paddlex:8080}
|
||||||
# Sandbox related
|
# Sandbox related
|
||||||
@ -340,44 +339,6 @@ services:
|
|||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
|
||||||
# lastest version: wget https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/docker/compose.yaml
|
# lastest version: wget https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/docker/compose.yaml
|
||||||
mineru-vllm-server:
|
|
||||||
build:
|
|
||||||
context: .
|
|
||||||
dockerfile: docker/mineru.Dockerfile
|
|
||||||
image: mineru-vllm:latest
|
|
||||||
container_name: mineru-vllm-server
|
|
||||||
profiles:
|
|
||||||
- all
|
|
||||||
env_file:
|
|
||||||
- .env
|
|
||||||
ports:
|
|
||||||
- 30000:30000
|
|
||||||
environment:
|
|
||||||
MINERU_MODEL_SOURCE: local
|
|
||||||
entrypoint: mineru-vllm-server
|
|
||||||
command:
|
|
||||||
--host 0.0.0.0
|
|
||||||
--port 30000
|
|
||||||
# parameters for vllm-engine
|
|
||||||
# --data-parallel-size 2 # If using multiple GPUs, increase throughput using vllm's multi-GPU parallel mode
|
|
||||||
# --gpu-memory-utilization 0.5 # If running on a single GPU and encountering VRAM shortage, reduce the KV cache size by this parameter, if VRAM issues persist, try lowering it further to `0.4` or below.
|
|
||||||
ulimits:
|
|
||||||
memlock: -1
|
|
||||||
stack: 67108864
|
|
||||||
ipc: host
|
|
||||||
healthcheck:
|
|
||||||
test: ["CMD-SHELL", "curl -f http://localhost:30000/health || exit 1"]
|
|
||||||
deploy:
|
|
||||||
resources:
|
|
||||||
reservations:
|
|
||||||
devices:
|
|
||||||
- driver: nvidia
|
|
||||||
device_ids: ["0"]
|
|
||||||
capabilities: [gpu]
|
|
||||||
networks:
|
|
||||||
- app-network
|
|
||||||
restart: unless-stopped
|
|
||||||
|
|
||||||
mineru-api:
|
mineru-api:
|
||||||
build:
|
build:
|
||||||
context: .
|
context: .
|
||||||
@ -392,17 +353,30 @@ services:
|
|||||||
- 30001:30001
|
- 30001:30001
|
||||||
environment:
|
environment:
|
||||||
MINERU_MODEL_SOURCE: local
|
MINERU_MODEL_SOURCE: local
|
||||||
|
VLLM_USE_V1: "1"
|
||||||
entrypoint: mineru-api
|
entrypoint: mineru-api
|
||||||
command:
|
command:
|
||||||
--host 0.0.0.0
|
--host 0.0.0.0
|
||||||
--port 30001
|
--port 30001
|
||||||
|
# --allow-public-http-client # Disabled by default; when binding to 0.0.0.0 or ::, this re-enables *-http-client backends and server_url. Enable only if you accept the SSRF risk.
|
||||||
|
# parameters for vllm-engine
|
||||||
|
# --gpu-memory-utilization 0.5 # If encountering VRAM shortage, reduce the KV cache size by this parameter; if VRAM issues persist, try lowering it further to `0.4` or below.
|
||||||
ulimits:
|
ulimits:
|
||||||
memlock: -1
|
memlock: -1
|
||||||
stack: 67108864
|
stack: 67108864
|
||||||
ipc: host
|
ipc: host
|
||||||
restart: unless-stopped
|
healthcheck:
|
||||||
|
test: ["CMD-SHELL", "curl -f http://localhost:30001/health || exit 1"]
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
reservations:
|
||||||
|
devices:
|
||||||
|
- driver: nvidia
|
||||||
|
device_ids: ["0"] # Modify for multiple GPUs: ["0", "1"]
|
||||||
|
capabilities: [gpu]
|
||||||
networks:
|
networks:
|
||||||
- app-network
|
- app-network
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
paddlex:
|
paddlex:
|
||||||
build:
|
build:
|
||||||
|
|||||||
@ -1,15 +1,8 @@
|
|||||||
# Use DaoCloud mirrored vllm image for China region for gpu with Ampere architecture and above (Compute Capability>=8.0)
|
# https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/docker/china/Dockerfile
|
||||||
|
# Use DaoCloud mirrored vllm image for China region for gpu with Volta、Turing、Ampere、Ada Lovelace、Hopper、Blackwell architecture (7.0 <= Compute Capability <= 12.0)
|
||||||
# Compute Capability version query (https://developer.nvidia.com/cuda-gpus)
|
# Compute Capability version query (https://developer.nvidia.com/cuda-gpus)
|
||||||
FROM docker.m.daocloud.io/vllm/vllm-openai:v0.10.1.1
|
# support x86_64 architecture and ARM(AArch64) architecture
|
||||||
|
FROM docker.m.daocloud.io/vllm/vllm-openai:v0.11.2
|
||||||
# Use the official vllm image
|
|
||||||
# FROM vllm/vllm-openai:v0.10.1.1
|
|
||||||
|
|
||||||
# Use DaoCloud mirrored vllm image for China region for gpu with Turing architecture and below (Compute Capability<8.0)
|
|
||||||
# FROM docker.m.daocloud.io/vllm/vllm-openai:v0.10.2
|
|
||||||
|
|
||||||
# Use the official vllm image
|
|
||||||
# FROM vllm/vllm-openai:v0.10.2
|
|
||||||
|
|
||||||
# Install libgl for opencv support & Noto fonts for Chinese characters
|
# Install libgl for opencv support & Noto fonts for Chinese characters
|
||||||
RUN apt-get update && \
|
RUN apt-get update && \
|
||||||
@ -23,7 +16,7 @@ RUN apt-get update && \
|
|||||||
rm -rf /var/lib/apt/lists/*
|
rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
# Install mineru latest
|
# Install mineru latest
|
||||||
RUN python3 -m pip install -U 'mineru[core]' -i https://mirrors.aliyun.com/pypi/simple --break-system-packages && \
|
RUN python3 -m pip install -U 'mineru[core]>=3.0.0' -i https://mirrors.aliyun.com/pypi/simple --break-system-packages && \
|
||||||
python3 -m pip cache purge
|
python3 -m pip cache purge
|
||||||
|
|
||||||
# Download models and update the configuration file
|
# Download models and update the configuration file
|
||||||
|
|||||||
@ -76,35 +76,39 @@ docker compose up -d api
|
|||||||
|
|
||||||
### MinerU(高精度)
|
### MinerU(高精度)
|
||||||
|
|
||||||
```env
|
首先从官网下载最新的 docker-compose 文件:
|
||||||
# .env 配置
|
|
||||||
MINERU_VL_SERVER=http://localhost:30000
|
|
||||||
MINERU_API_URI=http://localhost:30001
|
|
||||||
|
|
||||||
# 启动服务(需要 GPU)
|
```bash
|
||||||
docker compose up mineru-vllm-server mineru-api -d
|
wget https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/docker/compose.yaml -O docker/mineru.compose.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
启动服务(需要 GPU)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose -f docker/mineru.compose.yml --profile openai-server up -d
|
||||||
```
|
```
|
||||||
|
|
||||||
### MinerU Official(云服务)
|
### MinerU Official(云服务)
|
||||||
|
|
||||||
|
从 [MinerU 官网](https://mineru.net) 获取 API 密钥,在 .env 配置环境变量
|
||||||
|
|
||||||
```env
|
```env
|
||||||
# .env 配置
|
|
||||||
MINERU_API_KEY=your-api-key-here
|
MINERU_API_KEY=your-api-key-here
|
||||||
```
|
```
|
||||||
|
|
||||||
从 [MinerU 官网](https://mineru.net) 获取 API 密钥。
|
|
||||||
|
|
||||||
### PP-Structure-V3(结构化)
|
### PP-Structure-V3(结构化)
|
||||||
|
|
||||||
|
启动服务(需要 GPU)
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# 启动服务(需要 GPU)
|
|
||||||
docker compose up paddlex -d
|
docker compose up paddlex -d
|
||||||
```
|
```
|
||||||
|
|
||||||
### DeepSeek OCR(简单云服务)
|
### DeepSeek OCR(简单云服务)
|
||||||
|
|
||||||
|
在 .env 配置(使用已有的 SiliconFlow 密钥)
|
||||||
|
|
||||||
```env
|
```env
|
||||||
# .env 配置(使用已有的 SiliconFlow 密钥)
|
|
||||||
SILICONFLOW_API_KEY=your-api-key-here
|
SILICONFLOW_API_KEY=your-api-key-here
|
||||||
```
|
```
|
||||||
|
|
||||||
@ -114,7 +118,7 @@ SILICONFLOW_API_KEY=your-api-key-here
|
|||||||
|
|
||||||
在 `.env` 中设置服务器 IP:
|
在 `.env` 中设置服务器 IP:
|
||||||
|
|
||||||
```env
|
```
|
||||||
HOST_IP=your_server_ip
|
HOST_IP=your_server_ip
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|||||||
@ -66,7 +66,7 @@
|
|||||||
</div>
|
</div>
|
||||||
<div class="col-item" v-if="uploadMode !== 'url'">
|
<div class="col-item" v-if="uploadMode !== 'url'">
|
||||||
<div class="setting-label">
|
<div class="setting-label">
|
||||||
OCR 引擎
|
OCR 引擎(仅应用于 PDF/图片文件)
|
||||||
<a-tooltip title="检查服务状态">
|
<a-tooltip title="检查服务状态">
|
||||||
<ReloadOutlined
|
<ReloadOutlined
|
||||||
class="action-icon refresh-icon"
|
class="action-icon refresh-icon"
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user