Kubernetes Deployment

1
apiVersion: batch/v1
2
kind: Job
3
metadata:
4
  name: nel-eval-gsm8k
5
spec:
6
  template:
7
    spec:
8
      containers:
9
        - name: eval
10
          image: nemo-evaluator:latest
11
          args: ["eval", "run", "--bench", "gsm8k", "--repeats", "4",
12
                 "--output-dir", "/data/results", "--no-progress"]
13
          env:
14
            - name: NEMO_API_KEY
15
              valueFrom:
16
                secretKeyRef:
17
                  name: nemo-api
18
                  key: api_key
19
          volumeMounts:
20
            - name: data
21
              mountPath: /data
22
      volumes:
23
        - name: data
24
          persistentVolumeClaim:
25
            claimName: eval-data
26
      restartPolicy: Never

1
apiVersion: batch/v1
2
kind: Job
3
metadata:
4
  name: nel-eval-sharded
5
spec:
6
  completions: 8
7
  parallelism: 8
8
  completionMode: Indexed
9
  template:
10
    spec:
11
      containers:
12
        - name: eval
13
          image: nemo-evaluator:latest
14
          args: ["eval", "run", "--bench", "gsm8k", "--repeats", "8",
15
                 "--output-dir", "/data/shard_$(NEL_SHARD_IDX)", "--no-progress"]
16
          env:
17
            - name: NEL_SHARD_IDX
18
              valueFrom:
19
                fieldRef:
20
                  fieldPath: metadata.annotations['batch.kubernetes.io/job-completion-index']
21
            - name: NEL_TOTAL_SHARDS
22
              value: "8"

1
readinessProbe:
2
  httpGet:
3
    path: /health
4
    port: 9090
5
  initialDelaySeconds: 5
6
  periodSeconds: 10
7
livenessProbe:
8
  httpGet:
9
    path: /health
10
    port: 9090
11
  initialDelaySeconds: 15
12
  periodSeconds: 30

1
resource_servers:
2
  nemo_evaluator:
3
    endpoint: http://nel-serve.default.svc:9090