Installing with Helm

            

            
cd ~/universe-helm-charts/universe-vault
        

            

            
cat << 'EOF' | tee values-external.yaml
vault:
enabled: true
injector:
enabled: true
externalVaultAddr: http://<VAULT_SERVER>:<VAULT_PORT>
nodeSelector:
node-role.kubernetes.io/control-plane: ""
tolerations:
- effect: NoSchedule
operator: "Exists"
key: node-role.kubernetes.io/master
- effect: NoSchedule
operator: "Exists"
key: node-role.kubernetes.io/control-plane
server:
enabled: false
EOF
        

            

            
helm install -n vault --create-namespace -f values-external.yaml vault .
        

            

            
cd ~/universe-helm-charts/universe-infra-control-plane
        

            

            
cat << 'EOF' | tee values-secure.yaml
global:
image:
registry: nvcr.io/nvstaging/doca/
imagePullSecrets:
- name: nvcrio-cred
nodeSelector:
node-role.kubernetes.io/control-plane: ""
tolerations:
- effect: NoSchedule
operator: "Exists"
key: node-role.kubernetes.io/master
- effect: NoSchedule
operator: "Exists"
key: node-role.kubernetes.io/control-plane
# ip for ironic host
ironicHostIP: "<ironic ip address>"
provisioningStorage:
# hostpath is used by bootp and ironic
hostpath: <host path of local pv>
# hostname is used by bootp and ironic
hostname: "<host name of local pv>"

universe-infra-admin-controller:
enabled: true
tenantConfig:
create: true
tenants:
- id: tenant1
hostnames:
- host-a
- host-b
dpuInventory:
create: true
dpus:
- id: dpu1-host-a
host: host-a
- id: dpu1-host-b
host: host-b

universe-infra-resource-manager:
enabled: true

universe-infra-provisioning-manager:
enabled: true

universe-infra-provisioning-controller:
enabled: true
capiConfig:
create: true
clusterName: <name of the cluster>
controlPlaneEndPoint: <the ip address, host ip or vip of control plane endpoint>
controlPlaneEndPointPort: <the port of control plane endpoint>
controller:
args:
ntpServer: <ntp server ip address>
imageRegistry: <image registry ip address>

universe-infra-provisioning-executor:
enabled: true
universe-infra-provisioning-mariadb:
pv:
name: mariadb-pv
hostpath: <host path of local pv>
hostname: <host name of local pv>
universe-infra-provisioning-bootp:
bootp:
# -- dnsmasq configuration, refer https://linux.die.net/man/8/dnsmasq
dnsmasq:
# args is a list of dnsmasq command parameters. You can set any parameters supported by dnsmasq.
# --k, --interface, --dhcp-range and --dhcp-boot are required.
# --k: do not go into the background at startup.
# --interface: Listen only on the specified interface(s).
# --dhcp-range: addresses will be given out from the range <start-addr> to <end-addr>. If the lease time is given, then leases will be given for that length of time.
# --dhcp-boot: dnsmasq is providing a TFTP service. the filename is required here to enable network booting.
# --dhcp-option: specify different or extra options to DHCP clients.
args:
- --k
- --interface=<interface name>
- --dhcp-range=<dhcp ip range> # e.g. "172.16.105.200,172.16.105.240,12h"
- --dhcp-boot=<dhcp boot images> # e.g. "efi/grubaa64-BlueField-3.9.2.12271.2.7.4.efi"
- --dhcp-option=3
- --dhcp-option=6,<dns server> # multiple DNSs are comma separated. e.g. "10.211.0.124,10.211.0.121,10.7.77.135"
- --dhcp-option=option:classless-static-route,<route rules> # multiple route rules are comma separated. e.g. "0.0.0.0/0,172.16.105.1"

universe-infra-workload-manager:
enabled: true

universe-infra-workload-rule-manager:
enabled: true

universe-infra-workload-controller:
enabled: true

universe-infra-api-gateway:
enabled: true
vaultApproleSecret:
create: true
roleID: 3134f7ed-f66b-1347-83e0-54e1e003cd10 # example roleID
secretID: b7ac107d-d7be-cd38-4ad0-d41b4dddf5a0 # example secretID
vaultAnnotations:
addAnnotations: true
envoy:
config:
listener:
serverTLS:
enabled: true
peerValidation:
enabled: true
EOF

# The kubeconfig and CA files are used to authenticate provisioned DPU to join current cluster; if the cluster
# was installed by kubeadm, the default path are "/root/.kube/config", "/etc/kubernetes/pki/ca.crt" and
# "/etc/kubernetes/pki/ca.key" as follow.
#
# NOTES: It is important to check in the kubeconfig that the "server" URL is not using localhost (127.0.0.1),
# but the control plane API IP/VIP address

helm_set_file_params="universe-infra-provisioning-controller.capiConfig.kubeconfig=/root/.kube/config,"
helm_set_file_params+="universe-infra-provisioning-controller.capiConfig.tlsCrt=/etc/kubernetes/pki/ca.crt,"
helm_set_file_params+="universe-infra-provisioning-controller.capiConfig.tlsKey=/etc/kubernetes/pki/ca.key"

helm install -n universe --create-namespace -f values-secure.yaml icp . --set-file ${helm_set_file_params}
        

            

            
cd ~/universe-helm-charts/universe-tenant-control-plane
        

            

            
cat << 'EOF' | tee values-secure.yaml
global:
image:
registry: nvcr.io/nvstaging/doca/
imagePullSecrets:
- name: nvcrio-cred
nodeSelector:
node-role.kubernetes.io/control-plane: ""
tolerations:
- effect: NoSchedule
operator: "Exists"
key: node-role.kubernetes.io/master
- effect: NoSchedule
operator: "Exists"
key: node-role.kubernetes.io/control-plane
sidecars:
proxy:
config:
listener:
inject_headers:
tenant-id: tenant1
upstream:
address: 10.133.133.1 # iCP master ip address
port: 30001
clientTLS:
enabled: true
peerValidation:
enabled: true
vaultAnnotations:
addAnnotations: true
vaultApproleSecret:
create: true
roleID: 24ad0b7c-ad63-bc9d-f45d-bdd0bf75d7a2 # vault roleID, will be shared by all plugins
secretID: 290f4c90-5e10-80e2-a068-937c31ac512b # vault secretID, will be shared by all plugins

universe-k8s-tenant-resource-plugin:
enabled: true

universe-k8s-tenant-workload-plugin:
enabled: true

universe-k8s-tenant-workload-rule-plugin:
enabled: true
EOF

helm install -n universe --create-namespace -f values-secure.yaml tcp .
        

            

            
cat << 'EOF' | tee tenant-pod1.yaml
apiVersion: resource.universe.nvidia.com/v1alpha1
kind: UVSPod
metadata:
name: tenant-pod1
namespace: universe
spec:
object:
apiVersion: v1
kind: Pod
metadata:
name: tenant-pod1
spec:
containers:
- name: nginx
image: nginx:1.14.2
EOF

kubectl apply -f tenant-pod1.yaml
        

            

            
kubectl get uvspods.resource.universe.nvidia.com -n universe tenant-pod1 -o jsonpath='{.status.syncResult}{"\n"}'
{"result":"success"}
        

            

            
cat << 'EOF' | tee workloadrule1.yaml
apiVersion: workload.universe.nvidia.com/v1alpha1
kind: WorkloadRule
metadata:
name: rule1
namespace: universe
spec:
resourceType: v1/Pod
workloadTerms:
- matchExpressions:
- key: metadata.resourceNamespace
operator: In
values:
- default
- key: metadata.resourceName
operator: In
values:
- test-pod
workloadInfoInject:
- workloadKey: state.nodeName
asAnnotation:
name: tenant-node-name
dpuSelectionPolicy: Any
template:
apiVersion: v1
kind: Pod
spec:
containers:
- name: nginx
image: nginx:1.14.2
volumeMounts:
- name: workload-info
mountPath: /workload-info
# standard k8s way to mount annotation as a volume
volumes:
- name: workload-info
downwardAPI:
items:
- path: node-name
fieldRef:
fieldPath: metadata.annotations['tenant-node-name']
EOF

kubectl apply -f workloadrule1.yaml
        

            

            
cat << 'EOF' | tee test-pod.yaml
apiVersion: v1
kind: Pod
metadata:
name: test-pod
spec:
containers:
- name: nginx
image: nginx:1.14.2
EOF

kubectl apply -f test-pod.yaml
        

            

            
kubectl get uvspods -n universe \
-o jsonpath='{range .items[?(@.spec.object.metadata.annotations.workloadrule\.workload\.universe\.nvidia\.com/name=="rule1")]}{ .metadata.namespace }/{ .metadata.name}{"\n"}{end}'

universe/rule1-57283160-a077-446d-882b-4f1373b0d02e