From b403d8dd90a6687448bfdf943e77271195a2148b Mon Sep 17 00:00:00 2001 From: Wenjie Zhang Date: Sun, 27 Jul 2025 12:04:44 +0800 Subject: [PATCH] =?UTF-8?q?chore(dependencies):=20=E6=9B=B4=E6=96=B0mineru?= =?UTF-8?q?=E4=BE=9D=E8=B5=96=E4=B8=BAmineru[core]=EF=BC=8C=E7=AE=80?= =?UTF-8?q?=E5=8C=96=E6=96=87=E6=A1=A3=E8=A7=A3=E6=9E=90=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker/mineru.Dockerfile | 20 ++++++++++++++--- pyproject.toml | 2 +- src/plugins/mineru.py | 46 ++++++++++++++++++---------------------- 3 files changed, 39 insertions(+), 29 deletions(-) diff --git a/docker/mineru.Dockerfile b/docker/mineru.Dockerfile index 8e335882..e369e2d0 100644 --- a/docker/mineru.Dockerfile +++ b/docker/mineru.Dockerfile @@ -1,10 +1,24 @@ # Lastest version: wget https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/docker/china/Dockerfile # Use the official sglang image -FROM lmsysorg/sglang:v0.4.7-cu124 +FROM lmsysorg/sglang:v0.4.9.post3-cu126 +# For blackwell GPU, use the following line instead: +# FROM lmsysorg/sglang:v0.4.9.post3-cu128-b200 -# install mineru latest -RUN python3 -m pip install -U 'mineru[core]' -i https://mirrors.aliyun.com/pypi/simple --break-system-packages +# Install libgl for opencv support & Noto fonts for Chinese characters +RUN apt-get update && \ + apt-get install -y \ + fonts-noto-core \ + fonts-noto-cjk \ + fontconfig \ + libgl1 && \ + fc-cache -fv && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# Install mineru latest +RUN python3 -m pip install -U 'mineru[core]' -i https://mirrors.aliyun.com/pypi/simple --break-system-packages && \ + python3 -m pip cache purge # Download models and update the configuration file RUN /bin/bash -c "mineru-models-download -s modelscope -m all" diff --git a/pyproject.toml b/pyproject.toml index f16ff27b..9dd008cd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,7 @@ dependencies = [ "tqdm>=4.66.4", "rich>=13.7.1", "typer>=0.16.0", - "mineru>=2.1.6", + "mineru[core]>=2.1.6", ] [tool.ruff] line-length = 210 # 代码最大行宽 diff --git a/src/plugins/mineru.py b/src/plugins/mineru.py index 724cc350..67d3bbc7 100644 --- a/src/plugins/mineru.py +++ b/src/plugins/mineru.py @@ -206,33 +206,29 @@ def parse_doc( Adapted only for the case where the backend is set to "pipeline". server_url: When the backend is `sglang-client`, you need to specify the server_url, for example:`http://127.0.0.1:30000` """ - try: - file_name_list = [] - pdf_bytes_list = [] - lang_list = [] - for path in tqdm(path_list, desc="Parsing documents"): - file_name = str(Path(path).stem) - pdf_bytes = read_fn(path) - file_name_list.append(file_name) - pdf_bytes_list.append(pdf_bytes) - lang_list.append(lang) + file_name_list = [] + pdf_bytes_list = [] + lang_list = [] + for path in tqdm(path_list, desc="Parsing documents"): + file_name = str(Path(path).stem) + pdf_bytes = read_fn(path) + file_name_list.append(file_name) + pdf_bytes_list.append(pdf_bytes) + lang_list.append(lang) - result = do_parse( - output_dir=output_dir, - pdf_file_names=file_name_list, - pdf_bytes_list=pdf_bytes_list, - p_lang_list=lang_list, - backend=backend, - parse_method=method, - server_url=server_url, - start_page_id=start_page_id, - end_page_id=end_page_id - ) - return result if result else [""] + result = do_parse( + output_dir=output_dir, + pdf_file_names=file_name_list, + pdf_bytes_list=pdf_bytes_list, + p_lang_list=lang_list, + backend=backend, + parse_method=method, + server_url=server_url, + start_page_id=start_page_id, + end_page_id=end_page_id + ) + return result if result else [""] - except Exception as e: - logger.exception(e) - return [""] if __name__ == "__main__":