2024-09-24 11:58:13 +08:00
|
|
|
from openai import OpenAI
|
|
|
|
|
# Set OpenAI's API key and API base to use vLLM's API server.
|
|
|
|
|
openai_api_key = "EMPTY"
|
|
|
|
|
openai_api_base = "http://localhost:8080/v1"
|
|
|
|
|
|
|
|
|
|
client = OpenAI(
|
|
|
|
|
api_key=openai_api_key,
|
|
|
|
|
base_url=openai_api_base,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
chat_response = client.chat.completions.create(
|
|
|
|
|
model="llama",
|
|
|
|
|
messages=[
|
|
|
|
|
{"role": "system", "content": "You are a helpful assistant."},
|
|
|
|
|
{"role": "user", "content": "Tell me a joke."},
|
|
|
|
|
]
|
|
|
|
|
)
|
2025-05-24 11:29:45 +08:00
|
|
|
print("Chat response:", chat_response)
|
