Getting Started

            

            
docker run --rm --runtime=nvidia --gpus all ubuntu nvidia-smi
        

            

            
+-----------------------------------------------------------------------------------------+
| NVIDIA-SMI 550.54.14              Driver Version: 550.54.14      CUDA Version: 12.4     |
|-----------------------------------------+------------------------+----------------------+
| GPU  Name                 Persistence-M | Bus-Id          Disp.A | Volatile Uncorr. ECC |
| Fan  Temp   Perf          Pwr:Usage/Cap |           Memory-Usage | GPU-Util  Compute M. |
|                                         |                        |               MIG M. |
|=========================================+========================+======================|
|   0  NVIDIA H100 80GB HBM3          On  |   00000000:1B:00.0 Off |                    0 |
| N/A   36C    P0            112W /  700W |   78489MiB /  81559MiB |      0%      Default |
|                                         |                        |             Disabled |
+-----------------------------------------+------------------------+----------------------+

+-----------------------------------------------------------------------------------------+
| Processes:                                                                              |
|  GPU   GI   CI        PID   Type   Process name                              GPU Memory |
|        ID   ID                                                               Usage      |
|=========================================================================================|
|  No running processes found                                                             |
+-----------------------------------------------------------------------------------------+
        

            

            
export NGC_API_KEY=<value>
        

            

            
# If using bash
echo "export NGC_API_KEY=<value>" >> ~/.bashrc

# If using zsh
echo "export NGC_API_KEY=<value>" >> ~/.zshrc
        

            

            
echo "$NGC_API_KEY" | docker login nvcr.io --username '$oauthtoken' --password-stdin
        

            

            
# Choose a container name for bookkeeping
export CONTAINER_NAME=meta-llama3-8b-instruct

# Choose a LLM NIM Image from NGC
export IMG_NAME="nvcr.io/nvidian/nim-llm-dev/${CONTAINER_NAME}:24.05.rc15"

# Choose a path on your system to cache the downloaded models
export LOCAL_NIM_CACHE=~/.cache/nim
mkdir -p "$LOCAL_NIM_CACHE"

# Start the LLM NIM
docker run -it --rm --name=$CONTAINER_NAME \
  --runtime=nvidia \
  --gpus all \
  --shm-size=16GB \
  -e NGC_API_KEY \
  -v "$LOCAL_NIM_CACHE:/opt/nim/.cache" \
  -u $(id -u) \
  -p 8000:8000 \
  $IMG_NAME
        

            

            
INFO: Application startup complete. 
INFO: Uvicorn running on http://0.0.0.0:8000 (Press CTRL+C to quit)
        

            

            
curl -X GET 'http://0.0.0.0:8000/v1/models'
        

            

            
{ 
   "object": "list", 
   "data": [ 
     { 
       "id": "meta-llama3-8b-instruct", 
       "object": "model", 
       "created": 1715659875, 
       "owned_by": "vllm", 
       "root": "meta-llama3-8b-instruct", 
       "parent": null, 
       "permission": [ 
         { 
           "id": "modelperm-e39aaffe7015444eba964fa7736ae653", 
           "object": "model_permission", 
           "created": 1715659875, 
           "allow_create_engine": false, 
           "allow_sampling": true, 
           "allow_logprobs": true, 
           "allow_search_indices": false, 
           "allow_view": true, 
           "allow_fine_tuning": false, 
           "organization": "*", 
           "group": null, 
           "is_blocking": false 
         } 
       ] 
     } 
   ] 
 }
        

            

            
curl -X 'POST' \
    'http://0.0.0.0:8000/v1/completions' \
    -H 'accept: application/json' \
    -H 'Content-Type: application/json' \
    -d '{
"model": "meta/llama3-8b-instruct",
"prompt": "Once upon a time",
"max_tokens": 64
}'
        

            

            
from openai import OpenAI
client = OpenAI(base_url="http://0.0.0.0:8000/v1", api_key="not-used")
prompt = "Once upon a time"
response = client.completions.create(
    model="meta/llama3-8b-instruct",
    prompt=prompt,
    max_tokens=16,
    stream=False
)
completion = response.choices[0].text
print(completion)

# Prints:
# , there was a young man named Jack who lived in a small village at the
        

            

            
curl -X 'POST' \
'http://0.0.0.0:8000/v1/chat/completions' \
    -H 'accept: application/json' \
    -H 'Content-Type: application/json' \
    -d '{
"model": "meta/llama3-8b-instruct",
"messages": [
{
"role":"user",
"content":"Hello! How are you?"
},
{
"role":"assistant",
"content":"Hi! I am quite well, how can I help you today?"
},
{
"role":"user",
"content":"Can you write me a song?"
}
],
"max_tokens": 32
}'
        

            

            
from openai import OpenAI
client = OpenAI(base_url="http://0.0.0.0:8000/v1", api_key="not-used")
messages = [
    {"role": "user", "content": "Hello! How are you?"},
    {"role": "assistant", "content": "Hi! I am quite well, how can I help you today?"},
    {"role": "user", "content": "Write a short limerick about the wonders of GPU computing."}
]
chat_response = client.chat.completions.create(
    model="meta/llama3-8b-instruct",
    messages=messages,
    max_tokens=32,
    stream=False
)
assistant_message = chat_response.choices[0].message
print(assistant_message)

# Prints:
# ChatCompletionMessage(content='There once was a GPU so fine,\nProcessed data in parallel so divine,\nIt crunched with great zest,\nAnd computational quest,\nUnleashing speed, a true wonder sublime!', role='assistant', function_call=None, tool_calls=None)
        

            

            
# In the previous sections, the environment variable CONTAINER_NAME was
# defined using `export CONTAINER_NAME=meta-llama3-8b-instruct`
docker stop $CONTAINER_NAME