diff --git a/backend/package/yuxi/plugins/parser/mineru.py b/backend/package/yuxi/plugins/parser/mineru.py index 15389cea..6648f72d 100644 --- a/backend/package/yuxi/plugins/parser/mineru.py +++ b/backend/package/yuxi/plugins/parser/mineru.py @@ -98,13 +98,14 @@ class MinerUParser(BaseDocumentProcessor): file_path: 文件路径 params: 处理参数 - lang_list: 语言列表 (默认: ["ch"]) - - backend: 后端类型 (默认: "pipeline", 支持 "vlm-*" 系列) + - backend: 后端类型 (默认: "hybrid-auto-engine") - parse_method: 解析方法 (默认: "auto") - start_page_id: 起始页码 (默认: 0) - end_page_id: 结束页码 (默认: 99999) - formula_enable: 启用公式解析 (默认: True) - table_enable: 启用表格解析 (默认: True) - - server_url: VLM 服务器地址 (vlm-http-client 时需要) + - image_analysis: 启用图像/图表解析 (默认: True) + - server_url: OpenAI 兼容服务地址 (*-http-client 后端时可选) Returns: str: 提取的 Markdown 文本 @@ -121,23 +122,23 @@ class MinerUParser(BaseDocumentProcessor): # 解析参数 params = params or {} - # 构建请求数据 - 只保留核心参数 data = { "lang_list": params.get("lang_list", ["ch"]), - "backend": params.get("backend", "vlm-http-client"), + "backend": params.get("backend", "hybrid-auto-engine"), "parse_method": params.get("parse_method", "auto"), - # 固定返回 markdown 格式 + "formula_enable": params.get("formula_enable", True), + "table_enable": params.get("table_enable", True), + "image_analysis": params.get("image_analysis", True), + "start_page_id": params.get("start_page_id", 0), + "end_page_id": params.get("end_page_id", 99999), "return_md": True, - # 添加图片解析支持 "response_format_zip": True, "return_images": True, } - # vlm-http-client 后端需要 server_url - if data["backend"] == "vlm-http-client": - mineru_vl_server = os.environ.get("MINERU_VL_SERVER") - assert mineru_vl_server, "MINERU_VL_SERVER 环境变量未配置" - data["server_url"] = mineru_vl_server + server_url = params.get("server_url") + if server_url: + data["server_url"] = server_url try: start_time = time.time() @@ -155,7 +156,7 @@ class MinerUParser(BaseDocumentProcessor): self.parse_endpoint, files=files, data=data, - timeout=os.environ.get("MINERU_TIMEOUT", 1800), # 30分钟超时 + timeout=int(os.environ.get("MINERU_TIMEOUT", 1800)), # 30分钟超时 ) # 检查响应状态 diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 5e7f4552..ae629854 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -3,7 +3,7 @@ x-api-worker-env: &api-worker-env LITE_MODE: ${LITE_MODE:-} RUNNING_IN_DOCKER: "true" YUXI_ENV: production - MODEL_DIR_IN_DOCKER: /models + MODEL_DIR_IN_DOCKER: /app/models PYTHONPATH: /app/package POSTGRES_URL: ${POSTGRES_URL:-postgresql+asyncpg://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@postgres:5432/${POSTGRES_DB:-yuxi_know}} REDIS_URL: ${REDIS_URL:-redis://redis:6379/0} @@ -15,7 +15,6 @@ x-api-worker-env: &api-worker-env MILVUS_URI: ${MILVUS_URI:-http://milvus:19530} MILVUS_DB_NAME: ${MILVUS_DB_NAME:-default} MILVUS_TOKEN: ${MILVUS_TOKEN:-} - MINERU_VL_SERVER: ${MINERU_VL_SERVER:-http://mineru-vllm-server:30000} MINERU_API_URI: ${MINERU_API_URI:-http://mineru-api:30001} PADDLEX_URI: ${PADDLEX_URI:-http://paddlex:8080} MINIO_URI: ${MINIO_URI:-http://minio:9000} @@ -283,39 +282,6 @@ services: - app-network restart: unless-stopped - mineru-vllm-server: - build: - context: . - dockerfile: docker/mineru.Dockerfile - image: mineru-vllm:latest - container_name: mineru-vllm-server - profiles: - - all - env_file: - - .env.prod - environment: - MINERU_MODEL_SOURCE: local - entrypoint: mineru-vllm-server - command: - --host 0.0.0.0 - --port 30000 - ulimits: - memlock: -1 - stack: 67108864 - ipc: host - healthcheck: - test: ["CMD-SHELL", "curl -f http://localhost:30000/health || exit 1"] - deploy: - resources: - reservations: - devices: - - driver: nvidia - device_ids: ["0"] - capabilities: [gpu] - networks: - - app-network - restart: unless-stopped - mineru-api: build: context: . @@ -334,13 +300,25 @@ services: command: --host 0.0.0.0 --port 30001 + # --allow-public-http-client # Disabled by default; when binding to 0.0.0.0 or ::, this re-enables *-http-client backends and server_url. Enable only if you accept the SSRF risk. + # parameters for vllm-engine + # --gpu-memory-utilization 0.5 # If encountering VRAM shortage, reduce the KV cache size by this parameter; if VRAM issues persist, try lowering it further to `0.4` or below. ulimits: memlock: -1 stack: 67108864 ipc: host - restart: unless-stopped + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:30001/health || exit 1"] + deploy: + resources: + reservations: + devices: + - driver: nvidia + device_ids: ["0"] # Modify for multiple GPUs: ["0", "1"] + capabilities: [gpu] networks: - app-network + restart: unless-stopped paddlex: build: diff --git a/docker-compose.yml b/docker-compose.yml index ac16dad8..1aa9a814 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,6 @@ x-api-worker-env: &api-worker-env MILVUS_TOKEN: ${MILVUS_TOKEN:-} MINIO_URI: ${MINIO_URI:-http://minio:9000} # 文档解析 - MINERU_VL_SERVER: ${MINERU_VL_SERVER:-http://mineru-vllm-server:30000} MINERU_API_URI: ${MINERU_API_URI:-http://mineru-api:30001} PADDLEX_URI: ${PADDLEX_URI:-http://paddlex:8080} # Sandbox related @@ -340,44 +339,6 @@ services: restart: unless-stopped # lastest version: wget https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/docker/compose.yaml - mineru-vllm-server: - build: - context: . - dockerfile: docker/mineru.Dockerfile - image: mineru-vllm:latest - container_name: mineru-vllm-server - profiles: - - all - env_file: - - .env - ports: - - 30000:30000 - environment: - MINERU_MODEL_SOURCE: local - entrypoint: mineru-vllm-server - command: - --host 0.0.0.0 - --port 30000 - # parameters for vllm-engine - # --data-parallel-size 2 # If using multiple GPUs, increase throughput using vllm's multi-GPU parallel mode - # --gpu-memory-utilization 0.5 # If running on a single GPU and encountering VRAM shortage, reduce the KV cache size by this parameter, if VRAM issues persist, try lowering it further to `0.4` or below. - ulimits: - memlock: -1 - stack: 67108864 - ipc: host - healthcheck: - test: ["CMD-SHELL", "curl -f http://localhost:30000/health || exit 1"] - deploy: - resources: - reservations: - devices: - - driver: nvidia - device_ids: ["0"] - capabilities: [gpu] - networks: - - app-network - restart: unless-stopped - mineru-api: build: context: . @@ -392,17 +353,30 @@ services: - 30001:30001 environment: MINERU_MODEL_SOURCE: local + VLLM_USE_V1: "1" entrypoint: mineru-api command: --host 0.0.0.0 --port 30001 + # --allow-public-http-client # Disabled by default; when binding to 0.0.0.0 or ::, this re-enables *-http-client backends and server_url. Enable only if you accept the SSRF risk. + # parameters for vllm-engine + # --gpu-memory-utilization 0.5 # If encountering VRAM shortage, reduce the KV cache size by this parameter; if VRAM issues persist, try lowering it further to `0.4` or below. ulimits: memlock: -1 stack: 67108864 ipc: host - restart: unless-stopped + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:30001/health || exit 1"] + deploy: + resources: + reservations: + devices: + - driver: nvidia + device_ids: ["0"] # Modify for multiple GPUs: ["0", "1"] + capabilities: [gpu] networks: - app-network + restart: unless-stopped paddlex: build: diff --git a/docker/mineru.Dockerfile b/docker/mineru.Dockerfile index c1a9d51a..21b5f84a 100644 --- a/docker/mineru.Dockerfile +++ b/docker/mineru.Dockerfile @@ -1,15 +1,8 @@ -# Use DaoCloud mirrored vllm image for China region for gpu with Ampere architecture and above (Compute Capability>=8.0) +# https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/docker/china/Dockerfile +# Use DaoCloud mirrored vllm image for China region for gpu with Volta、Turing、Ampere、Ada Lovelace、Hopper、Blackwell architecture (7.0 <= Compute Capability <= 12.0) # Compute Capability version query (https://developer.nvidia.com/cuda-gpus) -FROM docker.m.daocloud.io/vllm/vllm-openai:v0.10.1.1 - -# Use the official vllm image -# FROM vllm/vllm-openai:v0.10.1.1 - -# Use DaoCloud mirrored vllm image for China region for gpu with Turing architecture and below (Compute Capability<8.0) -# FROM docker.m.daocloud.io/vllm/vllm-openai:v0.10.2 - -# Use the official vllm image -# FROM vllm/vllm-openai:v0.10.2 +# support x86_64 architecture and ARM(AArch64) architecture +FROM docker.m.daocloud.io/vllm/vllm-openai:v0.11.2 # Install libgl for opencv support & Noto fonts for Chinese characters RUN apt-get update && \ @@ -23,7 +16,7 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/* # Install mineru latest -RUN python3 -m pip install -U 'mineru[core]' -i https://mirrors.aliyun.com/pypi/simple --break-system-packages && \ +RUN python3 -m pip install -U 'mineru[core]>=3.0.0' -i https://mirrors.aliyun.com/pypi/simple --break-system-packages && \ python3 -m pip cache purge # Download models and update the configuration file diff --git a/docs/advanced/document-processing.md b/docs/advanced/document-processing.md index 12a5a8fa..5467c1ab 100644 --- a/docs/advanced/document-processing.md +++ b/docs/advanced/document-processing.md @@ -76,35 +76,39 @@ docker compose up -d api ### MinerU（高精度） -```env -# .env 配置 -MINERU_VL_SERVER=http://localhost:30000 -MINERU_API_URI=http://localhost:30001 +首先从官网下载最新的 docker-compose 文件： -# 启动服务（需要 GPU） -docker compose up mineru-vllm-server mineru-api -d +```bash +wget https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/docker/compose.yaml -O docker/mineru.compose.yml +``` + +启动服务（需要 GPU） + +```bash +docker compose -f docker/mineru.compose.yml --profile openai-server up -d ``` ### MinerU Official（云服务） +从 [MinerU 官网](https://mineru.net) 获取 API 密钥，在 .env 配置环境变量 + ```env -# .env 配置 MINERU_API_KEY=your-api-key-here ``` -从 [MinerU 官网](https://mineru.net) 获取 API 密钥。 - ### PP-Structure-V3（结构化） +启动服务（需要 GPU） + ```bash -# 启动服务（需要 GPU） docker compose up paddlex -d ``` ### DeepSeek OCR（简单云服务） +在 .env 配置（使用已有的 SiliconFlow 密钥） + ```env -# .env 配置（使用已有的 SiliconFlow 密钥） SILICONFLOW_API_KEY=your-api-key-here ``` @@ -114,7 +118,7 @@ SILICONFLOW_API_KEY=your-api-key-here 在 `.env` 中设置服务器 IP： -```env +``` HOST_IP=your_server_ip ``` diff --git a/web/src/components/FileUploadModal.vue b/web/src/components/FileUploadModal.vue index 55f66e9e..6e3bfeac 100644 --- a/web/src/components/FileUploadModal.vue +++ b/web/src/components/FileUploadModal.vue @@ -66,7 +66,7 @@
- OCR 引擎 + OCR 引擎（仅应用于 PDF/图片文件）