Create a NVIDIA NIM service. [Experimental]

POST

http://localhost:8080/api/v2/workloads/nim-services

POST

/api/v2/workloads/nim-services

curl -X POST http://localhost:8080/api/v2/workloads/nim-services \
     -H "Authorization: Bearer <token>" \
     -H "Content-Type: application/json" \
     -d '{
  "metadata": {
    "name": "my-workload-name",
    "projectId": "1"
  },
  "spec": {}
}'

{
  "metadata": {
    "name": "my-workload-name",
    "projectId": "1",
    "id": "string",
    "gvk": {
      "version": "v1",
      "kind": "Deployment",
      "group": "apps"
    },
    "projectName": "project-a",
    "clusterId": "71f69d83-ba66-4822-adf5-55ce55efd210",
    "tenantId": 1001,
    "departmentId": "2",
    "departmentName": "default",
    "createdAt": "2024-01-15T10:30:00Z",
    "createdBy": "user@run.ai",
    "updatedAt": "2024-01-15T10:35:00Z",
    "updatedBy": "user@run.ai",
    "priority": "string",
    "category": "string",
    "preemptibility": "preemptible",
    "configuration": {
      "mnnvl": "string",
      "userCredentials": {
        "sources": [
          {
            "ref": "my-hf-token-abc1234567"
          }
        ]
      }
    },
    "deletedAt": "2024-01-15T10:35:00Z",
    "deletedBy": "user@run.ai"
  },
  "desiredPhase": "Running",
  "spec": {
    "annotations": [
      {
        "name": "billing",
        "value": "my-billing-unit",
        "exclude": false
      }
    ],
    "autoscaling": {
      "maxReplicas": 1,
      "metric": "http_requests_total",
      "metricThreshold": 1,
      "minReplicas": 1,
      "scaleWindowSeconds": 1
    },
    "category": "string",
    "compute": {
      "cpuCoreLimit": 2,
      "cpuCoreRequest": 0.5,
      "cpuMemoryLimit": "30M",
      "cpuMemoryRequest": "20M",
      "gpuDevicesRequest": 1,
      "gpuMemoryLimit": "10M",
      "gpuMemoryRequest": "10M",
      "gpuPortionLimit": 0.5,
      "gpuPortionRequest": 0.5,
      "gpuRequestType": "portion"
    },
    "environmentVariables": [
      {
        "name": "HOME",
        "value": "/home/my-folder",
        "secret": {
          "name": "postgress_secret",
          "key": "POSTGRES_PASSWORD"
        },
        "configMap": {
          "name": "my-config-map",
          "key": "MY_POSTGRES_SCHEMA"
        },
        "podFieldRef": {
          "path": "metadata.name"
        },
        "userCredential": {
          "name": "my_postgres_user_and_password",
          "key": "POSTGRES_PASSWORD"
        },
        "exclude": false,
        "description": "Home directory of the user."
      }
    ],
    "image": "python:3.8",
    "imagePullPolicy": "Always",
    "imagePullSecrets": [
      {
        "name": "string",
        "userCredential": true,
        "exclude": false
      }
    ],
    "labels": [
      {
        "name": "stage",
        "value": "initial-research",
        "exclude": false
      }
    ],
    "modelStore": {
      "nimCache": {
        "name": "nim-cache-a",
        "profile": "tensorrt_llm-b200-fp8-tp2-pp1-latency-2901:10de-2"
      },
      "pvc": {
        "existingPvc": false,
        "claimName": "my-claim",
        "readOnly": false,
        "claimInfo": {
          "size": "1G",
          "storageClass": "my-storage-class",
          "accessModes": {
            "readWriteOnce": true,
            "readOnlyMany": false,
            "readWriteMany": false
          },
          "volumeMode": "Filesystem",
          "addedAttrValues": [
            {
              "key": "dnsname",
              "value": "my.dns.com"
            }
          ]
        }
      }
    },
    "multiNode": {
      "workers": 3
    },
    "ngcAuthSecret": "string",
    "nodePools": [
      "my-node-pool-a",
      "my-node-pool-b"
    ],
    "preemptibility": "preemptible",
    "priorityClass": "string",
    "probes": {
      "readiness": {
        "initialDelaySeconds": 1,
        "periodSeconds": 1,
        "timeoutSeconds": 1,
        "successThreshold": 1,
        "failureThreshold": 1,
        "handler": {
          "httpGet": {
            "path": "/",
            "port": 1,
            "host": "example.com",
            "scheme": "HTTP"
          }
        }
      }
    },
    "replicas": 2,
    "security": {
      "runAsGid": 30,
      "runAsUid": 500
    },
    "servingPort": {
      "serviceType": "ClusterIP",
      "port": 8000,
      "grpcPort": 8001,
      "metricsPort": 8002,
      "exposeExternally": true,
      "exposedUrl": "string",
      "exposedProtocol": "http",
      "authorizationType": "public",
      "authorizedUsers": [
        "user.a@example.com",
        "user.b@example.com"
      ],
      "authorizedGroups": [
        "group-a",
        "group-b"
      ]
    },
    "tolerations": [
      {
        "name": "string",
        "operator": "Equal",
        "key": "string",
        "value": "string",
        "effect": "NoSchedule",
        "seconds": 1,
        "exclude": false
      }
    ]
  }
}

Create a NVIDIA NIM service

Authentication

AuthorizationBearer

Bearer authentication

Request

This endpoint expects an object.

metadataobjectRequired

specobjectRequired

Response

Workload creation accepted

metadataobject

desiredPhaseenum

The desired phase of the workload.

Allowed values:

specobject

Errors

400

Bad Request Error

401

Unauthorized Error

403

Forbidden Error

409

Conflict Error

500

Internal Server Error

503

Service Unavailable Error