kris/ForcePilot
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

update vllm script

Browse Source
This commit is contained in:
Wenjie Zhang 2024-09-13 15:24:48 +08:00
parent 13c6037bdb
commit 8c3d9e0f0a
1 changed files with 14 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
scripts/run_vllm.sh → vllm/run.sh
View File

@ -1,13 +1,17 @@
CUDA_VISIBLE_DEVICES=0 python -m vllm.entrypoints.openai.api_server \ MODEL=Meta-Llama-3-8B-Instruct
--model="/home/zwj/workspace/models/chatglm3-6b" \
--tensor-parallel-size 1 \ if [ "$1" = "llama" ]; then
--trust-remote-code \ CUDA_VISIBLE_DEVICES=0,1 python -m vllm.entrypoints.openai.api_server \
--device auto \ --model="/hdd/zwj/models/meta-llama/$MODEL" \
--gpu-memory-utilization 0.98 \ --tensor-parallel-size 2 \
--dtype half \ --trust-remote-code \
--served-model-name "vllm" \ --device auto \
--host 0.0.0.0 \ --gpu-memory-utilization 0.98 \
--port 8080 --dtype half \
--served-model-name "$1" \
--host 0.0.0.0 \
--port 8080
fi
# https://docs.vllm.ai/en/latest/serving/openai_compatible_server.html#named-arguments # https://docs.vllm.ai/en/latest/serving/openai_compatible_server.html#named-arguments
# model 模型路径，以文件夹结尾 # model 模型路径，以文件夹结尾