chore(dependencies): 更新mineru依赖为mineru[core],简化文档解析逻辑
This commit is contained in:
parent
cf3cb5ef33
commit
b403d8dd90
@ -1,10 +1,24 @@
|
||||
# Lastest version: wget https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/docker/china/Dockerfile
|
||||
|
||||
# Use the official sglang image
|
||||
FROM lmsysorg/sglang:v0.4.7-cu124
|
||||
FROM lmsysorg/sglang:v0.4.9.post3-cu126
|
||||
# For blackwell GPU, use the following line instead:
|
||||
# FROM lmsysorg/sglang:v0.4.9.post3-cu128-b200
|
||||
|
||||
# install mineru latest
|
||||
RUN python3 -m pip install -U 'mineru[core]' -i https://mirrors.aliyun.com/pypi/simple --break-system-packages
|
||||
# Install libgl for opencv support & Noto fonts for Chinese characters
|
||||
RUN apt-get update && \
|
||||
apt-get install -y \
|
||||
fonts-noto-core \
|
||||
fonts-noto-cjk \
|
||||
fontconfig \
|
||||
libgl1 && \
|
||||
fc-cache -fv && \
|
||||
apt-get clean && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install mineru latest
|
||||
RUN python3 -m pip install -U 'mineru[core]' -i https://mirrors.aliyun.com/pypi/simple --break-system-packages && \
|
||||
python3 -m pip cache purge
|
||||
|
||||
# Download models and update the configuration file
|
||||
RUN /bin/bash -c "mineru-models-download -s modelscope -m all"
|
||||
|
||||
@ -51,7 +51,7 @@ dependencies = [
|
||||
"tqdm>=4.66.4",
|
||||
"rich>=13.7.1",
|
||||
"typer>=0.16.0",
|
||||
"mineru>=2.1.6",
|
||||
"mineru[core]>=2.1.6",
|
||||
]
|
||||
[tool.ruff]
|
||||
line-length = 210 # 代码最大行宽
|
||||
|
||||
@ -206,33 +206,29 @@ def parse_doc(
|
||||
Adapted only for the case where the backend is set to "pipeline".
|
||||
server_url: When the backend is `sglang-client`, you need to specify the server_url, for example:`http://127.0.0.1:30000`
|
||||
"""
|
||||
try:
|
||||
file_name_list = []
|
||||
pdf_bytes_list = []
|
||||
lang_list = []
|
||||
for path in tqdm(path_list, desc="Parsing documents"):
|
||||
file_name = str(Path(path).stem)
|
||||
pdf_bytes = read_fn(path)
|
||||
file_name_list.append(file_name)
|
||||
pdf_bytes_list.append(pdf_bytes)
|
||||
lang_list.append(lang)
|
||||
file_name_list = []
|
||||
pdf_bytes_list = []
|
||||
lang_list = []
|
||||
for path in tqdm(path_list, desc="Parsing documents"):
|
||||
file_name = str(Path(path).stem)
|
||||
pdf_bytes = read_fn(path)
|
||||
file_name_list.append(file_name)
|
||||
pdf_bytes_list.append(pdf_bytes)
|
||||
lang_list.append(lang)
|
||||
|
||||
result = do_parse(
|
||||
output_dir=output_dir,
|
||||
pdf_file_names=file_name_list,
|
||||
pdf_bytes_list=pdf_bytes_list,
|
||||
p_lang_list=lang_list,
|
||||
backend=backend,
|
||||
parse_method=method,
|
||||
server_url=server_url,
|
||||
start_page_id=start_page_id,
|
||||
end_page_id=end_page_id
|
||||
)
|
||||
return result if result else [""]
|
||||
result = do_parse(
|
||||
output_dir=output_dir,
|
||||
pdf_file_names=file_name_list,
|
||||
pdf_bytes_list=pdf_bytes_list,
|
||||
p_lang_list=lang_list,
|
||||
backend=backend,
|
||||
parse_method=method,
|
||||
server_url=server_url,
|
||||
start_page_id=start_page_id,
|
||||
end_page_id=end_page_id
|
||||
)
|
||||
return result if result else [""]
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(e)
|
||||
return [""]
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user