From 8c3d9e0f0a585822fe31d6ce512bd1b17974a473 Mon Sep 17 00:00:00 2001 From: Wenjie Zhang Date: Fri, 13 Sep 2024 15:24:48 +0800 Subject: [PATCH] update vllm script --- scripts/run_vllm.sh => vllm/run.sh | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) rename scripts/run_vllm.sh => vllm/run.sh (76%) diff --git a/scripts/run_vllm.sh b/vllm/run.sh similarity index 76% rename from scripts/run_vllm.sh rename to vllm/run.sh index 72ea353e..97a52fc6 100644 --- a/scripts/run_vllm.sh +++ b/vllm/run.sh @@ -1,13 +1,17 @@ -CUDA_VISIBLE_DEVICES=0 python -m vllm.entrypoints.openai.api_server \ - --model="/home/zwj/workspace/models/chatglm3-6b" \ - --tensor-parallel-size 1 \ - --trust-remote-code \ - --device auto \ - --gpu-memory-utilization 0.98 \ - --dtype half \ - --served-model-name "vllm" \ - --host 0.0.0.0 \ - --port 8080 +MODEL=Meta-Llama-3-8B-Instruct + +if [ "$1" = "llama" ]; then + CUDA_VISIBLE_DEVICES=0,1 python -m vllm.entrypoints.openai.api_server \ + --model="/hdd/zwj/models/meta-llama/$MODEL" \ + --tensor-parallel-size 2 \ + --trust-remote-code \ + --device auto \ + --gpu-memory-utilization 0.98 \ + --dtype half \ + --served-model-name "$1" \ + --host 0.0.0.0 \ + --port 8080 +fi # https://docs.vllm.ai/en/latest/serving/openai_compatible_server.html#named-arguments # model 模型路径，以文件夹结尾