chore(dependencies): 更新mineru依赖为mineru[core],简化文档解析逻辑

This commit is contained in:
Wenjie Zhang 2025-07-27 12:04:44 +08:00
parent cf3cb5ef33
commit b403d8dd90
3 changed files with 39 additions and 29 deletions

View File

@ -1,10 +1,24 @@
# Lastest version: wget https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/docker/china/Dockerfile
# Use the official sglang image
FROM lmsysorg/sglang:v0.4.7-cu124
FROM lmsysorg/sglang:v0.4.9.post3-cu126
# For blackwell GPU, use the following line instead:
# FROM lmsysorg/sglang:v0.4.9.post3-cu128-b200
# install mineru latest
RUN python3 -m pip install -U 'mineru[core]' -i https://mirrors.aliyun.com/pypi/simple --break-system-packages
# Install libgl for opencv support & Noto fonts for Chinese characters
RUN apt-get update && \
apt-get install -y \
fonts-noto-core \
fonts-noto-cjk \
fontconfig \
libgl1 && \
fc-cache -fv && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
# Install mineru latest
RUN python3 -m pip install -U 'mineru[core]' -i https://mirrors.aliyun.com/pypi/simple --break-system-packages && \
python3 -m pip cache purge
# Download models and update the configuration file
RUN /bin/bash -c "mineru-models-download -s modelscope -m all"

View File

@ -51,7 +51,7 @@ dependencies = [
"tqdm>=4.66.4",
"rich>=13.7.1",
"typer>=0.16.0",
"mineru>=2.1.6",
"mineru[core]>=2.1.6",
]
[tool.ruff]
line-length = 210 # 代码最大行宽

View File

@ -206,33 +206,29 @@ def parse_doc(
Adapted only for the case where the backend is set to "pipeline".
server_url: When the backend is `sglang-client`, you need to specify the server_url, for example:`http://127.0.0.1:30000`
"""
try:
file_name_list = []
pdf_bytes_list = []
lang_list = []
for path in tqdm(path_list, desc="Parsing documents"):
file_name = str(Path(path).stem)
pdf_bytes = read_fn(path)
file_name_list.append(file_name)
pdf_bytes_list.append(pdf_bytes)
lang_list.append(lang)
file_name_list = []
pdf_bytes_list = []
lang_list = []
for path in tqdm(path_list, desc="Parsing documents"):
file_name = str(Path(path).stem)
pdf_bytes = read_fn(path)
file_name_list.append(file_name)
pdf_bytes_list.append(pdf_bytes)
lang_list.append(lang)
result = do_parse(
output_dir=output_dir,
pdf_file_names=file_name_list,
pdf_bytes_list=pdf_bytes_list,
p_lang_list=lang_list,
backend=backend,
parse_method=method,
server_url=server_url,
start_page_id=start_page_id,
end_page_id=end_page_id
)
return result if result else [""]
result = do_parse(
output_dir=output_dir,
pdf_file_names=file_name_list,
pdf_bytes_list=pdf_bytes_list,
p_lang_list=lang_list,
backend=backend,
parse_method=method,
server_url=server_url,
start_page_id=start_page_id,
end_page_id=end_page_id
)
return result if result else [""]
except Exception as e:
logger.exception(e)
return [""]
if __name__ == "__main__":