Appendix#

MIG - NeMo Retriever (NV-Ingest) MIG custom Helm values file#

Here is an example of custom Helm values files for Embedding and Extraction NIM microservices nvingest-mig-values.yaml:

#NV-Ingest WORKERS and CPU resources
envVars:
  MAX_INGEST_PROCESS_WORKERS: 32
  NV_INGEST_MAX_UTIL: 32

# Define endpoints for Embedding and Extraction NIM microservices
  EMBEDDING_NIM_ENDPOINT: "http://llama-nemotron-embed-1b-v2:8000/v1"
  EMBEDDING_NIM_MODEL_NAME: "nvidia/llama-nemotron-embed-1b-v2"

  YOLOX_PAGE_IMAGE_FORMAT: JPEG
  YOLOX_GRPC_ENDPOINT: nemotron-page-elements-v3:8001
  YOLOX_HTTP_ENDPOINT: http://nemotron-page-elements-v3:8000/v1/infer
  YOLOX_INFER_PROTOCOL: grpc

  YOLOX_GRAPHIC_ELEMENTS_GRPC_ENDPOINT: nemotron-graphic-elements-v1:8001
  YOLOX_GRAPHIC_ELEMENTS_HTTP_ENDPOINT: http://nemotron-graphic-elements-v1:8000/v1/infer
  YOLOX_GRAPHIC_ELEMENTS_INFER_PROTOCOL: grpc

  YOLOX_TABLE_STRUCTURE_GRPC_ENDPOINT: nemotron-table-structure-v1:8001
  YOLOX_TABLE_STRUCTURE_HTTP_ENDPOINT: http://nemotron-table-structure-v1:8000/v1/infer
  YOLOX_TABLE_STRUCTURE_INFER_PROTOCOL: grpc

  OCR_GRPC_ENDPOINT: nemotron-ocr-v1:8001
  OCR_HTTP_ENDPOINT: http://nemotron-ocr-v1:8000/v1/infer
  OCR_INFER_PROTOCOL: grpc
  OCR_MODEL_NAME: pipeline

  MINIO_BUCKET: NV-Ingest # embeddings bucket for Vector DB bulk ingestion
  MINIO_ACCESS_KEY: minioadmin
  MINIO_SECRET_KEY: minioadmin
  MINIO_INTERNAL_ADDRESS: milvus-minio:9000
  MINIO_PUBLIC_ADDRESS: http://10.x.x.x:9000 #MINIO Load Balancer IP
  MILVUS_ENDPOINT: "http://milvus:19530" #Milvus endpoint

  OTEL_EXPORTER_OTLP_ENDPOINT: "nvingest-opentelemtry-collector:4317"

# Define image, tag and repo for NV-Ingest server
image:
  pullPolicy: IfNotPresent
  repository: nvcr.io/nvidia/nemo-microservices/NV-Ingest
  tag: 26.3.0
imagePullSecrets:
  - name: ngc-api
  - name: ngc-secret

replicaCount: 1

service:
  type: LoadBalancer #Enable LoadBalancer Service for NV-Ingest server

# NV-Ingest CPU/Memory (RAM) resources for worker threads
resources:
  limits:
    memory: 48Gi
    cpu: "48000m"
  requests:
    memory: 24Gi
    cpu: "24000m" # 24 vCPUs allocated for workers

# ephemeral disk size to store temporary data
tmpDirSize: 64Gi # default 50Gi

# Milvus (Distributed) in GPU-Index and CPU-Search (hybrid) is deployed independently
milvusDeployed: false

## enable OTEL collection
otelEnabled: true
# deploy OTEL from this helm chart
otelDeployed: true

#Redis image,tag and resources
redis:
  image:
    repository: redis
    tag: "8.2.3"
  auth:
    enabled: false
  replica:
    replicaCount: 1
    persistence:
      enabled: true
      size: "50Gi"
      storageClass: longhorn  # enable fast block storage, storageClass longhorn
    resources:
      requests:
        memory: "12Gi" # default 6Gi, increased for large batch sizes
      limits:
        memory: "24Gi"
  master:
    persistence:
      enabled: true
      size: "50Gi"
      storageClass: longhorn  # enable fast block storage, storageClass longhorn
    resources:
      requests:
        memory: "12Gi"
      limits:
        memory: "24Gi"
    configmap: |-
      protected-mode no

# MIG-optimized resource configuration for NeMo Retriever (NV-Ingest)
# This file overrides GPU resource requirements to use MIG profiles
# Configure NIM Operator components to use MIG
nimOperator:
  embedqa:
    resources:
      limits:
        nvidia.com/gpu: 0
        nvidia.com/mig-2g.48gb: 1
        #nvidia.com/mig-4g.71gb: 1 # enable H200 NVL MIG profile
        cpu: "2000m" # allocate 2 vCPU
        memory: 8Gi
      requests:
        nvidia.com/gpu: 0
        nvidia.com/mig-2g.48gb: 1
        #nvidia.com/mig-4g.71gb: 1 # enable H200 NVL MIG profile
        cpu: "1000m" # allocate 1 vCPU
        memory: 8Gi
    expose:
      service:
        type: LoadBalancer #Enable LoadBalancer service for external access
        port: 8000
        metricsPort: 8002
    metrics:
      enabled: true
      serviceMonitor:
        additionalLabels:
          release: kube-prometheus-stack
    env:
      - name: NIM_ENABLE_OTEL
        value: "1"
      - name: OTEL_SERVICE_NAME
        value: "nim-embed"
      - name: OTEL_TRACES_EXPORTER
        value: otlp
      - name: OTEL_EXPORTER_OTLP_ENDPOINT
        value: http://rag-opentelemetry-collector:4318
      - name: NIM_TRITON_PERFORMANCE_MODE
        value: "throughput"
  ocr:
    resources:
      limits:
        nvidia.com/gpu: 0
        nvidia.com/mig-2g.48gb: 1 # enable RTX PRO6K MIG profile
        #nvidia.com/mig-3g.71gb: 1 # enable H200 NVL MIG profile
      requests:
        nvidia.com/gpu: 0
        nvidia.com/mig-2g.48gb: 1 # enable RTX PRO6K MIG profile
        #nvidia.com/mig-3g.71gb: 1 # enable H200 NVL MIG profile
        cpu: "3000m" # allocate 3 vCPU
        memory: 4Gi
    expose:
      service:
        type: ClusterIP
        port: 8000
        grpcPort: 8001
    metrics:
      enabled: true
      serviceMonitor:
        additionalLabels:
          release: kube-prometheus-stack
    env:
      - name: NIM_ENABLE_OTEL
        value: "1"
      - name: OTEL_SERVICE_NAME
        value: "nemotron-ocr"
      - name: OTEL_TRACES_EXPORTER
        value: otlp
      - name: OTEL_EXPORTER_OTLP_ENDPOINT
        value: http://rag-opentelemetry-collector:4318
      # Triton OpenTelemetry Settings
      - name: TRITON_OTEL_URL
        value: http://rag-opentelemetry-collector:4318/v1/traces
      - name: OMP_NUM_THREADS
        value: "8"
      - name: NIM_HTTP_API_PORT
        value: "8000"
      - name: NIM_TRITON_LOG_VERBOSE
        value: "1"
      - name: NIM_TRITON_MAX_BATCH_SIZE
        value: "32"
  page-elements:
    resources:
      limits:
        nvidia.com/gpu: 0
        nvidia.com/mig-1g.24gb: 1 # enable RTX PRO6K MIG profile
        #nvidia.com/mig-2g.35gb: 1 # enable H200 NVL MIG profile
      requests:
        nvidia.com/gpu: 0
        nvidia.com/mig-1g.24gb: 1 # enable RTX PRO6K MIG profile
        #nvidia.com/mig-2g.35gb: 1 # enable H200 NVL MIG profile
        cpu: "1000m" # allocate 1 vCPU
        memory: 2Gi
    expose:
      service:
        type: ClusterIP
        port: 8000
        grpcPort: 8001
    metrics:
      enabled: true
      serviceMonitor:
        additionalLabels:
          release: kube-prometheus-stack
    env:
      - name: NIM_ENABLE_OTEL
        value: "1"
      - name: OTEL_SERVICE_NAME
        value: "nemotron-page-elements"
      - name: OTEL_TRACES_EXPORTER
        value: otlp
      - name: OTEL_EXPORTER_OTLP_ENDPOINT
        value: http://rag-opentelemetry-collector:4318
      # Triton OpenTelemetry Settings
      - name: TRITON_OTEL_URL
        value: http://rag-opentelemetry-collector:4318/v1/traces
      - name: OMP_NUM_THREADS
        value: "2"
      - name: NIM_HTTP_API_PORT
        value: "8000"
      - name: NIM_TRITON_LOG_VERBOSE
        value: "1"
      - name: NIM_TRITON_MAX_BATCH_SIZE
        value: "32"
      - name: NIM_TRITON_CPU_THREADS_PRE_PROCESSOR
        value: "2"
      - name: NIM_TRITON_CPU_THREADS_POST_PROCESSOR
        value: "1"
   table-structure:
    resources:
      limits:
        nvidia.com/gpu: 0
        nvidia.com/mig-1g.24gb: 1 # enable RTX PRO6K MIG profile
        #nvidia.com/mig-2g.35gb: 1 # enable H200 NVL MIG profile
      requests:
        nvidia.com/gpu: 0
        nvidia.com/mig-1g.24gb: 1 # enable RTX PRO6K MIG profile
        #nvidia.com/mig-2g.35gb: 1 # enable H200 NVL MIG profile
        cpu: "1000m" # request 1 vCPU
        memory: 3.5Gi
    expose:
      service:
        type: ClusterIP
        port: 8000
        grpcPort: 8001
    metrics:
      enabled: true
      serviceMonitor:
        additionalLabels:
          release: kube-prometheus-stack
    env:
      - name: NIM_ENABLE_OTEL
        value: "1"
      - name: OTEL_SERVICE_NAME
        value: "nemotron-table-structure"
      - name: OTEL_TRACES_EXPORTER
        value: otlp
      - name: OTEL_EXPORTER_OTLP_ENDPOINT
        value: http://rag-opentelemetry-collector:4318
      # Triton OpenTelemetry Settings
      - name: TRITON_OTEL_URL
        value: http://rag-opentelemetry-collector:4318/v1/traces
      - name: OMP_NUM_THREADS
        value: "1"
      - name: NIM_HTTP_API_PORT
        value: "8000"
      - name: NIM_TRITON_LOG_VERBOSE
        value: "1"
      - name: NIM_TRITON_MAX_BATCH_SIZE
        value: "32"
      - name: NIM_TRITON_RATE_LIMIT
        value: "3"
      - name: NIM_TRITON_CUDA_MEMORY_POOL_MB
        value: "2048"
   graphic-elements:
    resources:
      limits:
        nvidia.com/gpu: 0
        nvidia.com/mig-1g.24gb: 1 # enable RTX PRO6K MIG profile
        #nvidia.com/mig-1g.18gb: 1 # enable H200 NVL MIG profile
      requests:
        nvidia.com/gpu: 0
        nvidia.com/mig-1g.24gb: 1 # enable RTX PRO6K MIG profile
        #nvidia.com/mig-1g.18gb: 1 # enable H200 NVL MIG profile
        cpu: "500m" # request 0.5 vCPU
        memory: 3Gi
    expose:
      service:
        type: ClusterIP
        port: 8000
        grpcPort: 8001
    metrics:
      enabled: true
      serviceMonitor:
        additionalLabels:
          release: kube-prometheus-stack
    env:
      - name: NIM_ENABLE_OTEL
        value: "1"
      - name: OTEL_SERVICE_NAME
        value: "nemotron-graphic-elements"
      - name: OTEL_TRACES_EXPORTER
        value: otlp
      - name: OTEL_EXPORTER_OTLP_ENDPOINT
        value: http://rag-opentelemetry-collector:4318
      # Triton OpenTelemetry Settings
      - name: TRITON_OTEL_URL
        value: http://rag-opentelemetry-collector:4318/v1/traces
      - name: OMP_NUM_THREADS
        value: "1"
      - name: NIM_HTTP_API_PORT
        value: "8000"
      - name: NIM_TRITON_LOG_VERBOSE
        value: "1"
      - name: NIM_TRITON_MAX_BATCH_SIZE
        value: "32"
      - name: NIM_TRITON_RATE_LIMIT
        value: "3"
      - name: NIM_TRITON_CUDA_MEMORY_POOL_MB
        value: "2048"

Run:ai 1 - NeMo Retriever (NV-Ingest) custom Helm values file#

Here is an example of custom Helm values files for deploying NV-Ingest and Redis but disable deploying Embedding and Extraction NIM microservices with NIM service. Set the NIM endpoints to match the NIM deployment manifest => nvingest-runai-values.yaml:

#NV-Ingest WORKERS and CPU resources
envVars:
  MAX_INGEST_PROCESS_WORKERS: 32
  NV_INGEST_MAX_UTIL: 32

# Define endpoints for Embedding and Extraction NIM microservices
  EMBEDDING_NIM_ENDPOINT: "http://nemotron-embed:8000/v1"
  EMBEDDING_NIM_MODEL_NAME: "nvidia/llama-nemotron-embed-1b-v2"

  YOLOX_PAGE_IMAGE_FORMAT: JPEG
  YOLOX_GRPC_ENDPOINT: nemotron-page:8001
  YOLOX_HTTP_ENDPOINT: http://nemotron-page:8000/v1/infer
  YOLOX_INFER_PROTOCOL: grpc

  YOLOX_GRAPHIC_ELEMENTS_GRPC_ENDPOINT: nemotron-graphic:8001
  YOLOX_GRAPHIC_ELEMENTS_HTTP_ENDPOINT: http://nemotron-graphic:8000/v1/infer
  YOLOX_GRAPHIC_ELEMENTS_INFER_PROTOCOL: grpc

  YOLOX_TABLE_STRUCTURE_GRPC_ENDPOINT: nemotron-table:8001
  YOLOX_TABLE_STRUCTURE_HTTP_ENDPOINT: http://nemotron-table:8000/v1/infer
  YOLOX_TABLE_STRUCTURE_INFER_PROTOCOL: grpc

  OCR_GRPC_ENDPOINT: nemotron-ocr:8001
  OCR_HTTP_ENDPOINT: http://nemotron-ocr:8000/v1/infer
  OCR_INFER_PROTOCOL: grpc
  OCR_MODEL_NAME: pipeline

  MINIO_BUCKET: NV-Ingest # embeddings bucket for Vector DB bulk ingestion
  MINIO_ACCESS_KEY: minioadmin
  MINIO_SECRET_KEY: minioadmin
  MINIO_INTERNAL_ADDRESS: milvus-minio:9000
  MINIO_PUBLIC_ADDRESS: http://10.x.x.x:9000 #MINIO Load Balancer IP
  MILVUS_ENDPOINT: "http://milvus:19530" #Milvus endpoint

  OTEL_EXPORTER_OTLP_ENDPOINT: "nvingest-opentelemtry-collector:4317"

# Define image, tag and repo for NV-Ingest server
image:
  pullPolicy: IfNotPresent
  repository: nvcr.io/nvidia/nemo-microservices/NV-Ingest
  tag: 26.3.0
imagePullSecrets:
  - name: ngc-api
  - name: ngc-secret

replicaCount: 1

service:
  type: LoadBalancer #Enable LoadBalancer Service for NV-Ingest server

# NV-Ingest CPU/Memory (RAM) resources for worker threads
resources:
  limits:
    memory: 48Gi
    cpu: "48000m"
  requests:
    memory: 24Gi
    cpu: "24000m" # 24 vCPUs allocated for workers

# ephemeral disk size to store temporary data
tmpDirSize: 64Gi # default 50Gi

# Milvus (Distributed) in GPU-Index and CPU-Search (hybrid) is deployed independently
milvusDeployed: false

## enable OTEL collection
otelEnabled: true
# deploy OTEL from this helm chart
otelDeployed: true

#Redis image,tag and resources
redis:
  image:
    repository: redis
    tag: "8.2.3"
  auth:
    enabled: false
  replica:
    replicaCount: 1
    persistence:
      enabled: true
      size: "50Gi"
      storageClass: longhorn  # enable fast block storage, storageClass longhorn
    resources:
      requests:
        memory: "12Gi" # default 6Gi, increased for large batch sizes
      limits:
        memory: "24Gi"
  master:
    persistence:
      enabled: true
      size: "50Gi"
      storageClass: longhorn  # enable fast block storage, storageClass longhorn
    resources:
      requests:
        memory: "12Gi"
      limits:
        memory: "24Gi"
    configmap: |-
      protected-mode no

# NIM Operator resource configuration for NeMo Retriever (NV-Ingest)
# This file overrides GPU resource requirements to use Run:ai
# NIM Operator components require K8s scheduler, cannot use Run:ai scheduler, disable NIM microservices
nimOperator:
  embedqa:
    enabled: false
    resources:
      limits:
        nvidia.com/gpu: 0.5
      requests:
        nvidia.com/gpu: 0.5
        cpu: "4000m" # allocate 4 vCPU
        memory: 8Gi
  ocr:
    enabled: false
    resources:
      limits:
        nvidia.com/gpu: 0.4
      requests:
        nvidia.com/gpu: 0.4
        cpu: "3000m" # allocate 3 vCPU
        memory: 4Gi
  page-elements:
    enabled: false
    resources:
      limits:
        nvidia.com/gpu: 0.3
      requests:
        nvidia.com/gpu: 0.3
        cpu: "1000m" # allocate 1 vCPU
        memory: 2Gi
   table-structure:
    enabled: false
    resources:
      limits:
        nvidia.com/gpu: 0.2
      requests:
        nvidia.com/gpu: 0.2
        cpu: "1000m" # request 1 vCPU
        memory: 3.5Gi
   graphic-elements:
    enabled: false
    resources:
      limits:
        nvidia.com/gpu: 0.1
      requests:
        nvidia.com/gpu: 0.1
        cpu: "500m" # request 0.5 vCPU
        memory: 3Gi

Run:ai 2 - NeMo Retriever (NV-Ingest) NIM manifest deployment#

Here is a sample custom Kubernetes manifest file for Embedding NIM: nemotron-embed-1b-v2 and Extraction NIM: nemotron-ocr-v1 shown below => nvingest-runai-NIM-microservices-manifest.yaml. Other extraction NIM microservices can be added using these examples.

apiVersion: apps/v1
kind: Deployment
metadata:
  name: nemotron-embed
  namespace: runai-rag
  labels:
    app: nemotron-embed
    app.kubernetes.io/name: nemotron-embed
spec:
  replicas: 1
  selector:
    matchLabels:
      app: nemotron-embed
  template:
    metadata:
      labels:
        app: nemotron-embed
        app.kubernetes.io/name: nemotron-embed
        priorityClassName: very-high
        project: rag
      annotations:
        runai-nodepools: rag-ingest
        gpu-fraction: "0.5"
        gpu-fraction-num-devices: "1"
    spec:
      schedulerName: runai-scheduler
      priorityClassName: very-high
      imagePullSecrets:
      - name: ngc-secret
      containers:
        - name: nemotron-embed
          image: nvcr.io/nim/nvidia/llama-nemotron-embed-1b-v2:1.13.0
          imagePullPolicy: IfNotPresent
          ports:
            - containerPort: 8000
              name: api
              protocol: TCP
            - containerPort: 8001
              name: grpc
              protocol: TCP
            - containerPort: 8002
              name: metrics
              protocol: TCP
          env:
            - name: NGC_API_KEY
              valueFrom:
                secretKeyRef:
                  name: ngc-api
                  key: NGC_API_KEY
            - name: NIM_SERVER_PORT
              value: "8000"
            - name: NIM_HTTP_API_PORT
              value: "8000"
            - name: NIM_JSONL_LOGGING
              value: "1"
            - name: NIM_TRITON_LOG_VERBOSE
              value: "1"
            - name: OUTLINES_CACHE_DIR
              value: /tmp/outlines
            - name: OMP_NUM_THREADS
              value: "1"
            - name: NIM_TRITON_PERFORMANCE_MODE
              value: throughput
            - name: NIM_TRITON_METRICS_PORT
              value: "8002"
            - name: NIM_LOG_LEVEL
              value: INFO
            - name: NIM_GRPC_API_PORT
              value: "8001"
            - name: NIM_TRITON_GRPC_PORT
              value: "8001"
            - name: NIM_ENABLE_OTEL
              value: "true"
            - name: NIM_OTEL_SERVICE_NAME
              value: embed
            - name: NIM_OTEL_TRACES_EXPORTER
              value: otlp
            - name: NIM_OTEL_METRICS_EXPORTER
              value: console
            - name: NIM_OTEL_EXPORTER_OTLP_ENDPOINT
              value: http://nvingest-opentelemetry-collector:4318
            - name: TRITON_OTEL_URL
              value: http://nvingest-opentelemetry-collector:4318/v1/traces
            - name: TRITON_OTEL_RATE
              value: "1"
            - name: RUNAI_JOB_NAME
              value: nemotron-embed
            - name: RUNAI_PROJECT
              value: rag
---
apiVersion: v1
kind: Service
metadata:
  name: nemotron-embed
  namespace: runai-rag
  labels:
    app: nemotron-embed
    app.kubernetes.io/name: nemotron-embed
spec:
  type: LoadBalancer #ClusterIP
  selector:
    app: nemotron-embed
  ports:
    - name: api
      port: 8000
    - name: grpc
      port: 8001
    - name: metrics
      port: 8002
---
# Service Monitoring for nemotron-embed
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: nemotron-embed
  namespace: runai-rag
  labels:
    app: nemotron-embed
    release: kube-prometheus-stack
spec:
  selector:
    matchLabels:
      app: nemotron-embed
  endpoints:
  - port: api
    path: /v1/metrics
    interval: 30s
  - port: metrics
    path: /metrics
    interval: 30s
  namespaceSelector:
    matchNames:
    - runai-rag
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: nemotron-ocr
  namespace: runai-rag
  labels:
    app: nemotron-ocr
    app.kubernetes.io/name: nemotron-ocr
spec:
  replicas: 1
  selector:
    matchLabels:
      app: nemotron-ocr
  template:
    metadata:
      labels:
        app: nemotron-ocr
        app.kubernetes.io/name: nemotron-ocr
        priorityClassName: very-high
        project: rag
      annotations:
        runai-nodepools: rag-ingest
        gpu-fraction: "0.4"
        gpu-fraction-num-devices: "1"
    spec:
      schedulerName: runai-scheduler
      priorityClassName: very-high
      imagePullSecrets:
      - name: ngc-secret
      containers:
        - name: nemotron-ocr
          image: nvcr.io/nim/nvidia/nemoretriever-ocr-v1:1.3.0
          imagePullPolicy: IfNotPresent
          ports:
            - containerPort: 8000
              name: api
            - containerPort: 8001
              name: grpc
          env:
            - name: NGC_API_KEY
              valueFrom:
                secretKeyRef:
                  name: ngc-api
                  key: NGC_API_KEY
            - name: NIM_JSONL_LOGGING
              value: "1"
            - name: NIM_LOG_LEVEL
              value: INFO
            - name: NIM_SERVER_PORT
              value: "8000"
            - name: NIM_HTTP_API_PORT
              value: "8000"
            - name: NIM_GRPC_API_PORT
              value: "8001"
            - name: NIM_TRITON_GRPC_PORT
              value: "8001"
            - name: OUTLINES_CACHE_DIR
              value: /tmp/outlines
            - name: OMP_NUM_THREADS
              value: "8"
            - name: NIM_TRITON_MAX_BATCH_SIZE
              value: "32"
            - name: NIM_TRITON_LOG_VERBOSE
              value: "1"
            - name: RUNAI_JOB_NAME
              value: nemotron-ocr
            - name: RUNAI_PROJECT
              value: rag
            - name: NIM_ENABLE_OTEL
              value: "true"
            - name: NIM_OTEL_SERVICE_NAME
              value: ocr
            - name: NIM_OTEL_TRACES_EXPORTER
              value: otlp
            - name: NIM_OTEL_METRICS_EXPORTER
              value: console
            - name: NIM_OTEL_EXPORTER_OTLP_ENDPOINT
              value: http://nvingest-opentelemetry-collector:4318
            - name: TRITON_OTEL_URL
              value: http://nvingest-opentelemetry-collector:4318/v1/traces
            - name: TRITON_OTEL_RATE
              value: "1"
---
apiVersion: v1
kind: Service
metadata:
  name: nemotron-ocr
  namespace: runai-rag
  labels:
    app: nemotron-ocr
    app.kubernetes.io/name: nemotron-ocr
spec:
  type: ClusterIP
  selector:
    app: nemotron-ocr
  ports:
    - name: api
      port: 8000
    - name: grpc
      port: 8001
---
# Service Monitoring for nemotron-ocr
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: nemotron-ocr
  namespace: runai-rag
  labels:
    app: nemotron-ocr
    release: kube-prometheus-stack
spec:
  selector:
    matchLabels:
      app: nemotron-ocr
  endpoints:
  - port: api
    path: /v1/metrics
    interval: 30s
  namespaceSelector:
    matchNames:
    - runai-rag

Milvus distributed Helm custom values file#

Included below is an example of custom Helm values files for Milvus distributed deployment milvus-distributed-values.yaml:

## Expand the name of the chart
nameOverride: ""
## Default fully qualified app name
fullnameOverride: "milvus"
## Enable Milvus Distributed mode
cluster:
  enabled: true
image:
  all:
    repository: milvusdb/milvus
    tag: v2.6.5-gpu #Enable GPU accelerated (cuVS) image
    pullPolicy: IfNotPresent

## Enable tracing
extraConfigFiles:
  user.yaml: |+
    trace:
      exporter: otlp
      sampleFraction: 1
      otlp:
        endpoint: rag-opentelemetry-collector:4317
        secure: false
service:
  type: LoadBalancer #Enable LoadBalancer IP for external access
  port: 19530
  portName: milvus

metrics:
  enabled: true
  serviceMonitor:
    # Set this to `true` to create ServiceMonitor for Prometheus operator
    enabled: true
    interval: "30s"
    scrapeTimeout: "10s"
    additionalLabels:
      release: kube-prometheus-stack # prometheus stack release label

# Streaming node
streaming:
  enabled: true
  woodpecker:
    embedded: true
    storage:
      type: minio

streamingNode:
  replicas: 1

# ATTU - Frontend Management Console
attu:
  enabled: true
  name: attu
  ingress:
    enabled: true
    hosts:
    - vdb.era285.nvidia.com
    ingressClassName: nginx

#===
# Etcd
#===
etcd:
  enabled: true
  name: etcd
  replicaCount: 3
  pdb:
    create: false
  image:
    repository: "milvusdb/etcd"
    tag: "3.5.23-r2"
    pullPolicy: IfNotPresent
  autoCompactionMode: revision
  autoCompactionRetention: "1000"
  persistence:
    accessMode: ReadWriteOnce
    enabled: true
    size: 10Gi
    storageClass: longhorn # enable fast block storage

#===
# Minio
#===
## Configuration values for the minio dependency
minio:
  enabled: true
  accessKey: minioadmin
  secretKey: minioadmin
  mode: distributed
  name: minio
  bucketName: "NV-Ingest"
  zones: 2 # enable for larger scale deployment in ERA cluster with 4+nodes
  image:
    tag: "RELEASE.2025-09-07T16-13-09Z"
    pullPolicy: IfNotPresent
  resources:
    requests:
      cpu: 4
      memory: 8Gi
    limits:
       cpu: 8
       memory: 16Gi
  service:
    type: LoadBalancer # Enable LoadBalancer IP for external access
    port: 9000
  persistence:
    enabled: true
    storageClass: nfs-client
    accessMode: ReadWriteOnce
    size: 500Gi

#====
# Data Node, is consolidated with index node
#====
# Data node performs index build and ingestion
# Allocate GPU MIG profile or Run:ai GPU fraction and CPU / memory resources
# Scale up/out with more data and scale down when not ingesting data
dataNode:
  enabled: true
  replicas: 1 # scale out replicas with >10M vectors
  resources:
    limits:
      cpu: "20"
      memory: 40Gi
      nvidia.com/gpu: "0" # disable full GPU allocation
      nvidia.com/mig-1g.24gb: 1 # enable RTX PRO6K MIG profile
      #nvidia.com/mig-1g.18gb: 1 # enable H200 NVL MIG profile
      #nvidia.com/gpu: "0.25" # enable Run:ai GPU fraction
    requests:
      # baseline 4vCPU / 8Gi for up to ~1M vectors,
      # scale up to 16vCPU / 32Gi for large enterprise data >10M vectors
      cpu: "16"
      memory: 32Gi
      nvidia.com/gpu: "0" # disable full GPU allocation
      nvidia.com/mig-1g.24gb: 1 # enable RTX PRO6K MIG profile
      #nvidia.com/mig-1g.18gb: 1 # enable H200 NVL MIG profile
      #nvidia.com/gpu: "0.25" # enable Run:ai GPU fraction
#====
# Query Node
#====
# Query node performs search requests
# Uses CPU and memory resources
# Scale up with more data and scale out with more search requests
queryNode:
  enabled: true
  # replicas to -1 to enable autoscaling with HPA
  replicas: -1
  resources:
    limits:
      cpu: "20"
      memory: 50Gi
    requests:
      # baseline 4vCPU / 10Gi for up to ~1M vectors,
      # scale up to 8vCPU / 40Gi for large enterprise data >10M vectors
      cpu: "16"
      memory: 40Gi
  disk:
    enabled: true  # Enable index node build disk vector index
    size:
      enabled: false  # Enable local storage size limit
  hpa:
    enabled: true
    minReplicas: 1
    maxReplicas: 8
    cpuUtilization: 80 # autoscale at 80% CPU utilization
    memoryUtilization: 80 # autoscale at 80% Memory utilization

NIM Cache for Retrieval NIM#

NIM Cache examples for Retrieval NIM microservices: LLM NIM and Reranking NIM are shown below => nimcache-retrieval.yaml:

# RAG pipeline NIM Cache for LLM NIM
apiVersion: apps.nvidia.com/v1alpha1
kind: NIMCache
metadata:
  name: nimcache-nemo49b-vllm-nvfp4-tp1
  namespace: rag
spec:
  source:
    ngc:
      modelPuller: nvcr.io/nim/nvidia/llama-3.3-nemotron-super-49b-v1.5:1.15.5
      pullSecret: ngc-secret
      authSecret: ngc-api
      model:
        engine: vllm #tensorrt_llm
        lora: false
        gpus:
        - product: RTX6000_BLACKWELL_SV # H200_NVL
        precision: nvfp4 # fp8
        qosProfile: throughput  # latency
        tensorParallelism: "1" # "2"
        #profiles:
        #- 528fb89432d285acafa908147aa3d05a7e5018830312ca95bd9d2661fb38889f
  storage:
    pvc:
      create: true
      storageClass: "nfs-client"
      size: "80Gi"
      volumeAccessMode: ReadWriteMany
  resources: {}
---
# RAG pipeline NIM Cache for Reranking NIM
apiVersion: apps.nvidia.com/v1alpha1
kind: NIMCache
metadata:
  name: nimcache-nemotron-rerank
  namespace: rag
spec:
  source:
    ngc:
      modelPuller: nvcr.io/nim/nvidia/llama-nemotron-rerank-1b-v2:1.11.0
      pullSecret: ngc-secret
      authSecret: ngc-api
      model:
        profiles:
        - all
  storage:
    pvc:
      create: true
      storageClass: "nfs-client"
      size: "50Gi"
      volumeAccessMode: ReadWriteMany
  resources: {}

NIM Service for Retrieval NIM microservices#

NIM Service examples for LLM NIM and Reranking NIM as part of the RAG Retrieval NIM microservices => NIMService-retrieval.yaml:

apiVersion: apps.nvidia.com/v1alpha1
kind: NIMService
metadata:
  name: nim-llm-nemotron49b
  namespace: rag
spec:
  image:
    repository: nvcr.io/nim/nvidia/llama-3.3-nemotron-super-49b-v1.5
    tag: 1.15.5
    pullPolicy: IfNotPresent
    pullSecrets:
      - ngc-secret
  authSecret: ngc-api
  storage:
    nimCache:
      name: nimcache-nemo49b-vllm-nvfp4-tp1
      profile: 'e9cc0c5ea49283a493a0b18a05a97eb9b15a82a0d6acbb967e35609ddeb767fa'
  replicas: 1
  resources:
    limits:
      nvidia.com/gpu: 1
    requests:
      nvidia.com/gpu: 1
  expose:
    service:
      type: LoadBalancer #ClusterIP
      port: 8000
  metrics:
    enabled: true
    serviceMonitor:
      additionalLabels:
        release: kube-prometheus-stack
  env:
    - name: NIM_ENABLE_OTEL
      value: "1"
    - name: OTEL_SERVICE_NAME
      value: "nim-llm-nemotron49b"
    - name: OTEL_TRACES_EXPORTER
      value: otlp
    - name: OTEL_EXPORTER_OTLP_ENDPOINT
      value: http://rag-opentelemetry-collector:4318
      # For RTX PRO6k, reduce MAX CONTEXT to 16256 tokens
      # For H200 NVL, increase MAX CONTEXT to 32768 tokens
    - name: NIM_MAX_MODEL_LEN
      value: "16256" # 32768 for H200 NVL
---
apiVersion: apps.nvidia.com/v1alpha1
kind: NIMService
metadata:
  name: nemotron-rerank
  namespace: rag
spec:
  image:
    repository: nvcr.io/nim/nvidia/llama-nemotron-rerank-1b-v2
    tag: 1.11.0
    pullPolicy: IfNotPresent
    pullSecrets:
      - ngc-secret
  authSecret: ngc-api
  storage:
    nimCache:
      name: nimcache-nemotron-reranker
  replicas: 1
  resources:
    limits:
      nvidia.com/gpu: 1
    requests:
      nvidia.com/gpu: 1
  expose:
    service:
      type: ClusterIP
      port: 8000
      metricsPort: 8002
  metrics:
    enabled: true
    serviceMonitor:
      additionalLabels:
        release: kube-prometheus-stack
  env:
    - name: NIM_ENABLE_OTEL
      value: "1"
    - name: OTEL_SERVICE_NAME
      value: "nim-rerank"
    - name: OTEL_TRACES_EXPORTER
      value: otlp
    - name: OTEL_EXPORTER_OTLP_ENDPOINT
      value: http://rag-opentelemetry-collector:4318

RAG Blueprint Helm custom values#

Included below is an example of custom Helm values files for RAG blueprint deployment rag-blueprint-values.yaml:

# -- Global chart configuration
nameOverride: ""
fullnameOverride: "rag-server"
# subsection: rag-server
# RAG LangChain Orchestrator Service

# -- Image pull secret for all images used by this chart
imagePullSecret:
  name: "ngc-secret"
  create: false

# -- Secret containing API keys for NVIDIA NGC model registry
ngcApiSecret:
  name: "ngc-api"
  create: false

# -- RAG server container image and replicas
image:
  repository: nvcr.io/nvidia/blueprint/rag-server
  tag: "2.5.0"
  pullPolicy: IfNotPresent

# -- RAG server service configuration
service:
  type: LoadBalancer # Enable LoadBalancer service for external access
  port: 8081

# -- RAG server worker and container resources for High Performance and Scale
server:
  workers: 64
resources:
  limits:
    cpu: "20"
    memory: "80Gi"
  requests:
    cpu: "16"
    memory: "64Gi"
# -- RAG server - scale out replicas with load and LLM scale
replicaCount: 1

# -- Kubernetes scheduling
nodeSelector: {}
affinity: {}
tolerations: []

# -- Environment variables for rag-server
envVars:
  ##===Vector DB specific configurations===
  # URL on which vectorstore is hosted
  APP_VECTORSTORE_URL: "http://milvus:19530"
  # Milvus on Run:ai in runai-rag namespace needs FQDN DNS name
  #APP_VECTORSTORE_URL: "http://milvus.runai-rag.svc.clsuter.local:19530"
  # Type of vector db used to store embedding supported type "milvus" or "elasticsearch"
  APP_VECTORSTORE_NAME: "milvus"
  # cuVS enabled GPU accelerated Index (GPU_CAGRA)
  APP_VECTORSTORE_INDEXTYPE: "GPU_CAGRA"
  # Type of vector db search to be used
  APP_VECTORSTORE_SEARCHTYPE: "dense"
  # Type of ranker to use for vector store in case of Hybrid Search
  APP_VECTORSTORE_RANKER_TYPE: "rrf" # Can be "rrf" or "weighted"
  # Weight for dense vector search in case of "weighted" Hybrid Search
  APP_VECTORSTORE_DENSE_WEIGHT: "0.5"
  # Weight for sparse vector search in case of "weighted" Hybrid Search
  APP_VECTORSTORE_SPARSE_WEIGHT: "0.5"
  # Disable GPU search for milvus vector DB specific to rag-server
  APP_VECTORSTORE_ENABLEGPUSEARCH: "False"
  # For Milvus CPU search set the EF parameter for each search query
  # ef: Parameter controls query time/accuracy trade-off. Higher ef leads to more accurate but slower search. EF needs to be >= Vector DB topK
  APP_VECTORSTORE_EF: "20"

  # default vectorstore collection name to store embeddings
  COLLECTION_NAME: "multimodal_data"
  APP_RETRIEVER_SCORETHRESHOLD: "0.25"
  # Top K from vector DB, which goes as input to reranker model - not applicable if ENABLE_RERANKER is set to False
  VECTOR_DB_TOPK: "100"
  # Number of document chunks to insert in LLM prompt
  APP_RETRIEVER_TOPK: "10"

# Ingestion envVars used in Ingestor-server if enabled
  ##===MINIO config to store multimodal embeddings for VDB bulk upload===
  MINIO_ENDPOINT: "milvus-minio:9000"
  # Milvus MinIO on Run:ai in runai-rag namespace needs FQDN DNS name
  #MINIO_ENDPOINT: "milvus-minio.runai-rag.svc.cluster.local:9000"
  MINIO_ACCESSKEY: "minioadmin"
  MINIO_SECRETKEY: "minioadmin"

  ##===Redis configurations for summary status tracking===
  REDIS_HOST: "nvingest-redis-master"
  # NV-Ingest Redis on Run:ai in runai-rag namespace needs FQDN DNS name
  #REDIS_HOST: "nvingest-redis-master.runai-rag.svc.cluster.local"
  REDIS_PORT: "6379"
  REDIS_DB: "0"

# NIM endpoint configurations
  ##===LLM Model specific configurations===
  APP_LLM_MODELNAME: "nvidia/llama-3.3-nemotron-super-49b-v1.5"
  # URL at which LLM model is hosted, NIM Service name used to deploy LLM
  APP_LLM_SERVERURL: "nim-llm-nemotron49b:8000"
  # LLM model parameters
  LLM_MAX_TOKENS: "32768" # "16256"
  LLM_TEMPERATURE: "0"
  LLM_TOP_P: "1.0"

  ##===Query Rewriter Model specific configurations===
  APP_QUERYREWRITER_MODELNAME: "nvidia/llama-3.3-nemotron-super-49b-v1.5"
  # URL on which query rewriter model is hosted.
  APP_QUERYREWRITER_SERVERURL: "nim-llm-nemotron49b:8000"

  ##===Filter Expression Generator Model specific configurations===
  APP_FILTEREXPRESSIONGENERATOR_MODELNAME: "nvidia/llama-3.3-nemotron-super-49b-v1.5"
  APP_FILTEREXPRESSIONGENERATOR_SERVERURL: "nim-llm-nemotron49b:8000"
  # enable filter expression generator for natural language to filter expression conversion
  ENABLE_FILTER_GENERATOR: "False"

  ##===Embedding Model specific configurations===
  # URL on which embedding model is hosted. NV-Ingest Embedding NIM service name
  APP_EMBEDDINGS_SERVERURL: "llama-nemotron-embed-1b-v2:8000/v1"
  # Run:ai Embedding NIM deployment in runai-rag namespace needs FQDN DNS name
  #APP_EMBEDDINGS_SERVERURL: "nemotron-embed.runai-rag.svc.cluster.local:8000/v1"
  APP_EMBEDDINGS_MODELNAME: "nvidia/llama-nemotron-embed-1b-v2"
  APP_EMBEDDINGS_DIMENSIONS: "2048"

  ##===Reranking Model specific configurations===
  # URL on which ranking model is hosted. NIM service name used to deploy
  APP_RANKING_SERVERURL: "nemotron-rerank:8000"
  APP_RANKING_MODELNAME: "nvidia/llama-nemotron-rerank-1b-v2"
  ENABLE_RERANKER: "True"
  # Default score threshold for filtering documents by reranker relevance (0.0 to 1.0)
  RERANKER_SCORE_THRESHOLD: "0.0"

 # === RAG Retrieval pipeline performance optimizations
  # Disable citations in the response for benchmarking, enable for RAG accuracy and evaluation
  ENABLE_CITATIONS: "False"
  # Disable guardrails for performance benchmarking, adds additional latency
  ENABLE_GUARDRAILS: "False"
  # Log level for server, supported level NOTSET, DEBUG, INFO, WARN, ERROR, CRITICAL
  LOGLEVEL: "INFO"
  # Disable multi-turn conversation
  ENABLE_MULTITURN: "False"
  # Disable query rewriting for multiturn conversation
  ENABLE_QUERYREWRITER: "False"
  # Disable conversation history with current query for retrieval (when query rewriter is disabled)
  MULTITURN_RETRIEVER_SIMPLE: "False"
  # number of last n chat messages to consider from the provided conversation history (0 = no history)
  CONVERSATION_HISTORY: "0"

# === Enable Metrics and Tracing with Jaeger or Zipkin===
  APP_TRACING_ENABLED: "True"
  # HTTP endpoint
  APP_TRACING_OTLPHTTPENDPOINT: "http://rag-opentelemetry-collector:4318/v1/traces"
  # GRPC endpoint
  APP_TRACING_OTLPGRPCENDPOINT: "grpc://rag-opentelemetry-collector:4317"

  # === Disable Reflection ===
  ENABLE_REFLECTION: "false"

  # Enable source metadata in document content during generation
  ENABLE_SOURCE_METADATA: "true"

  # Filter content within <think></think> tags in model responses
  FILTER_THINK_TOKENS: "true"

  # For performance benchmarking, disable Reasoning with reasoning models like Nemotron Super-49B
  LLM_ENABLE_THINKING: "false"
  LLM_REASONING_BUDGET: "0"
  LLM_LOW_EFFORT: "false"

# -- Ingestor Server
# subsection: ingestor-server used for basic ingestion validation
# Not used for ingestion performance benchmarking
# Ingestor API Service - frontend API service to NV-Ingest

ingestor-server:
  enabled: true
  appName: ingestor-server

  replicaCount: 1

  imagePullSecret:
    create: false
    name: "ngc-secret"

  image:
    repository: nvcr.io/nvidia/blueprint/ingestor-server
    tag: "2.5.0"

  # -- Service config for ingestor-server
  service:
    type: NodePort #ClusterIP, use NodePort for initial ingestion validation
    port: 8082

  server:
    workers: 1
  resources:
    limits:
      memory: "25Gi"
    requests:
      memory: "25Gi"

  envVars:

    # === Vector Store Configurations ===
    APP_VECTORSTORE_URL: "http://milvus:19530"
    # Milvus on Run:ai in runai-rag namespace needs FQDN DNS name
    #APP_VECTORSTORE_URL: "http://milvus.runai-rag.svc.clsuter.local:19530"
    APP_VECTORSTORE_NAME: "milvus" # supported values: "milvus" or "elasticsearch"
    APP_VECTORSTORE_SEARCHTYPE: "dense"
    # Type of ranker to use for vector store in case of Hybrid Search
    APP_VECTORSTORE_RANKER_TYPE: "rrf" # Can be "rrf" or "weighted"
    # Weight for dense vector search in case of "weighted" Hybrid Search
    APP_VECTORSTORE_DENSE_WEIGHT: "0.5"
    # Weight for sparse vector search in case of "weighted" Hybrid Search
    APP_VECTORSTORE_SPARSE_WEIGHT: "0.5"
    APP_VECTORSTORE_ENABLEGPUINDEX: "True"
    APP_VECTORSTORE_ENABLEGPUSEARCH: "False"
    COLLECTION_NAME: "multimodal_data"

    # === MinIO Configurations ===
    ##===MINIO config to store multimodal embeddings for VDB bulk upload===
    MINIO_ENDPOINT: "milvus-minio:9000"
    # Milvus MinIO on Run:ai in runai-rag namespace needs FQDN DNS name
    #MINIO_ENDPOINT: "milvus-minio.runai-rag.svc.cluster.local:9000"
    MINIO_ACCESSKEY: "minioadmin"
    MINIO_SECRETKEY: "minioadmin"

    # === Embeddings Configurations ===
    # NV-Ingest Embedding NIM service name in rag namespace
    APP_EMBEDDINGS_SERVERURL: "llama-nemotron-embed-1b-v2:8000/v1"
    # Run:ai Embedding NIM deployment in runai-rag namespace needs FQDN DNS name
    #APP_EMBEDDINGS_SERVERURL: "nemotron-embed.runai-rag.svc.cluster.local:8000/v1"
    APP_EMBEDDINGS_MODELNAME: "nvidia/llama-nemotron-embed-1b-v2"
    APP_EMBEDDINGS_DIMENSIONS: "2048"

    # === NV-Ingest Configurations ===
    # NV-Ingest service name in rag namespace
    APP_NVINGEST_MESSAGECLIENTHOSTNAME: "nvingest-NV-Ingest"
    # NV-Ingest on Run:ai in runai-rag namespace needs FQDN DNS name
    #APP_NVINGEST_MESSAGECLIENTHOSTNAME: "nvingest-NV-Ingest.runai-rag.svc.cluster.local"
    APP_NVINGEST_MESSAGECLIENTPORT: "7670"

    # === NV-Ingest extraction configurations ===
    APP_NVINGEST_PDFEXTRACTMETHOD: "None"
    APP_NVINGEST_EXTRACTTABLESMETHOD: "yolox"
    APP_NVINGEST_EXTRACTTEXT: "True"
    APP_NVINGEST_EXTRACTINFOGRAPHICS: "False"
    APP_NVINGEST_EXTRACTTABLES: "True"
    APP_NVINGEST_EXTRACTCHARTS: "True"
    APP_NVINGEST_EXTRACTIMAGES: "False"
    APP_NVINGEST_EXTRACTPAGEASIMAGE: "False"
    APP_NVINGEST_STRUCTURED_ELEMENTS_MODALITY: ""
    APP_NVINGEST_IMAGE_ELEMENTS_MODALITY: ""
    APP_NVINGEST_TEXTDEPTH: "page"
    # === NV-Ingest save to disk configurations ===
    APP_NVINGEST_SAVETODISK: "False"
    NVINGEST_MINIO_BUCKET: "NV-Ingest"

    # === NV-Ingest performance configurations ===
    APP_NVINGEST_ENABLE_PDF_SPLIT_PROCESSING: "False"
    APP_NVINGEST_PAGES_PER_CHUNK: "16"

    # === General ===
    # Summary Model Configurations
    SUMMARY_LLM: "nvidia/llama-3.3-nemotron-super-49b-v1.5"
    SUMMARY_LLM_SERVERURL: "nim-llm-nemotron49b:8000"
    SUMMARY_LLM_MAX_CHUNK_LENGTH: "9000"
    SUMMARY_CHUNK_OVERLAP: "400"
    SUMMARY_LLM_TEMPERATURE: "0.0"
    SUMMARY_LLM_TOP_P: "1.0"
    SUMMARY_MAX_PARALLELIZATION: "20"

    # === General ===
    ENABLE_CITATIONS: "True" # Enable citations for RAG accuracy and evaluation
    LOGLEVEL: "INFO"

    # === NV-Ingest splitting configurations ===
    APP_NVINGEST_CHUNKSIZE: "256"  # 512 default; Size of chunks for splitting
    APP_NVINGEST_CHUNKOVERLAP: "50"  # 150 default; Overlap size for chunks
    APP_NVINGEST_ENABLEPDFSPLITTER: "True"  # Enable PDF splitter
    APP_NVINGEST_SEGMENTAUDIO: "False"  # Enable audio segmentation for NV Ingest

    # === Redis configurations ===
    REDIS_HOST: "nvingest-redis-master"
    # NV-Ingest Redis on Run:ai in runai-rag namespace needs FQDN DNS name
    #REDIS_HOST: "nvingest-redis-master.runai-rag.svc.cluster.local"
    REDIS_PORT: "6379"
    REDIS_DB: "0"
    ENABLE_REDIS_BACKEND: "False"

    # === Bulk upload to MinIO ===
    ENABLE_MINIO_BULK_UPLOAD: "True"
    TEMP_DIR: "/tmp-data"
    INGESTOR_SERVER_DATA_DIR: "/data/"

    # === NV-Ingest Batch Mode Configurations ===
    NV_INGEST_FILES_PER_BATCH: "30" #16 default
    NV_INGEST_CONCURRENT_BATCHES: "10" #4 default
    ENABLE_NV_INGEST_DYNAMIC_BATCHING: "True"

    # === Tracing ===
    APP_TRACING_ENABLED: "False"
    # HTTP endpoint
    APP_TRACING_OTLPHTTPENDPOINT: "http://rag-opentelemetry-collector:4318/v1/traces"
    # GRPC endpoint
    APP_TRACING_OTLPGRPCENDPOINT: "grpc://rag-opentelemetry-collector:4317"

# -- Frontend
# subsection: frontend
# rag frontend - web console for initial validation and demo purposes only
frontend:
  enabled: true
  appName: "rag-frontend"

  replicaCount: 1

  image:
    repository: nvcr.io/nvidia/blueprint/rag-frontend
    pullPolicy: IfNotPresent
    tag: "2.5.0"

  imagePullSecret:
    name: "ngc-secret"

  service:
    type: NodePort
    port: 3000

# -- Observability
# subsection: serviceMonitor
# Enable metrics for the rag-server to be sent to Prometheus
serviceMonitor:
  enabled: true

# subsection: opentelemetry-collector
# enable OpenTelemetry OTEL collector
opentelemetry-collector:
  enabled: true
  mode: deployment
  image:
    repository: docker.io/otel/opentelemetry-collector-contrib
    tag: "0.140.0"
  command:
    name: otelcol-contrib
  config:
    receivers:
      otlp:
        protocols:
          grpc:
            endpoint: '${env:MY_POD_IP}:4317'
          http:
            cors:
              allowed_origins:
                - "*"
    exporters:
      zipkin:
        endpoint: "http://rag-zipkin:9411/api/v2/spans"
      debug:
        verbosity: detailed
      prometheus:
        endpoint: ${env:MY_POD_IP}:8889
    extensions:
      health_check: {}
      zpages:
        endpoint: 0.0.0.0:55679
    processors:
      batch: {}
      tail_sampling:
        policies:
          - name: drop_noisy_traces_url
            type: string_attribute
            string_attribute:
              key: http.target
              values:
                - \/health
              enabled_regex_matching: true
              invert_match: true
      transform:
        trace_statements:
          - context: span
            statements:
              - set(status.code, 1) where attributes["http.path"] == "/health"
              - replace_match(attributes["http.route"], "/v1", attributes["http.target"]) where attributes["http.target"] != nil
              - replace_pattern(name, "/v1", attributes["http.route"]) where attributes["http.route"] != nil
              - set(name, Concat([name, attributes["http.url"]], " ")) where name == "POST"
    service:
      extensions: [zpages, health_check]
      pipelines:
        traces:
          receivers: [otlp]
          exporters: [debug, zipkin]
          processors: [tail_sampling, transform]
        metrics:
          exporters:
            - debug
            - prometheus
          processors:
            - memory_limiter
            - batch
          receivers:
            - otlp
            - prometheus
        logs:
          receivers: [otlp]
          exporters: [debug]
          processors: [batch]
  ports:
    metrics:
      enabled: true
      containerPort: 8889
      servicePort: 8889
      protocol: TCP

# subsection: zipkin
# Jaeger web UI is already deployed in the Enterprise RA cluster
zipkin:
  enabled: false


# subsection: kube-prometheus-stack
# kube-prometheus and grafana are already deployed in the Enterprise RA cluster
kube-prometheus-stack:
  enabled: false

# -- NIM microservices (dependencies) configuration
# Disable all NIM microservices, NIM microservices are deployed using independent
# NIM service, NeMo Retriever (NV-Ingest) or Run:ai deployments
# subsection: nim-llm
# NIM LLM
nimOperator:
  draResources:
    enabled: false
    name: "rag-claim"
    type: "TimeSlicing"
  nim-llm:
    enabled: false
# subsection: nvidia-nim-llama-32-nv-embedqa-1b-v2
# NIM Text Embedding
  nvidia-nim-llama-32-nv-embedqa-1b-v2:
    enabled: false
# subsection: text-reranking-nim
# NIM Text Reranking
  nvidia-nim-llama-32-nv-rerankqa-1b-v2:
    enabled: false
# subsection: nim-vlm
# NIM Vision-Language (VLM)
  nim-vlm:
    enabled: false

# -- NV-Ingest dependency configuration
# subsection: NV-Ingest
# NV-Ingest Service is deployed separately
NV-Ingest:
  enabled: false