update vllm

vllm/run.sh

@@ -1,9 +1,14 @@
 MODEL=Meta-Llama-3-8B-Instruct
 
+if [ -z "$1" ]; then
+    echo "Error: No argument provided. Please specify a model name."
+    exit 1
+fi
+
 if [ "$1" = "llama" ]; then
-    CUDA_VISIBLE_DEVICES=0,1 python -m vllm.entrypoints.openai.api_server \
+    CUDA_VISIBLE_DEVICES=0 python -m vllm.entrypoints.openai.api_server \
         --model="/hdd/zwj/models/meta-llama/$MODEL" \
-        --tensor-parallel-size 2 \
+        --tensor-parallel-size 1 \
         --trust-remote-code \
         --device auto \
         --gpu-memory-utilization 0.98 \

vllm/test_vllm.py

@@ -0,0 +1,18 @@
+from openai import OpenAI
+# Set OpenAI's API key and API base to use vLLM's API server.
+openai_api_key = "EMPTY"
+openai_api_base = "http://localhost:8080/v1"
+
+client = OpenAI(
+    api_key=openai_api_key,
+    base_url=openai_api_base,
+)
+
+chat_response = client.chat.completions.create(
+    model="llama",
+    messages=[
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user", "content": "Tell me a joke."},
+    ]
+)
+print("Chat response:", chat_response)