Appendix

nic-cluster-policy.yaml

apiVersion: mellanox.com/v1alpha1
kind: NicClusterPolicy
metadata:
  name: nic-cluster-policy
spec:
  secondaryNetwork:
    cniPlugins:
      image: plugins
      repository: ghcr.io/k8snetworkplumbingwg
      version: v1.6.2-update.1
      imagePullSecrets: []
    multus:
      image: multus-cni
      repository: ghcr.io/k8snetworkplumbingwg
      version: v4.1.0
      imagePullSecrets: []
      containerResources:
        - name: "kube-multus"
          limits: {memory: "100Mi"}
          requests: {memory: "100Mi"}
  nvIpam:
    image: nvidia-k8s-ipam
    repository: ghcr.io/mellanox
    version: v0.3.7
    imagePullSecrets: []
    enableWebhook: false

combined-ippools.yaml

apiVersion: nv-ipam.nvidia.com/v1alpha1
kind: IPPool
metadata:
  name: sriovibnet-pool-a-su-1
  namespace: network-operator
spec:
  subnet: 192.176.0.0/16
  perNodeBlockSize: 32
  gateway: 192.176.0.1
  nodeSelector:
    nodeSelectorTerms:
    - matchExpressions:
      - key: node-role.kubernetes.io/worker
        operator: Exists
---
apiVersion: nv-ipam.nvidia.com/v1alpha1
kind: IPPool
metadata:
  name: sriovibnet-pool-b-su-1
  namespace: network-operator
spec:
  subnet: 192.169.0.0/16
  perNodeBlockSize: 32
  gateway: 192.169.0.1
  nodeSelector:
    nodeSelectorTerms:
    - matchExpressions:
      - key: node-role.kubernetes.io/worker
        operator: Exists
---
apiVersion: nv-ipam.nvidia.com/v1alpha1
kind: IPPool
metadata:
  name: sriovibnet-pool-c-su-1
  namespace: network-operator
spec:
  subnet: 192.170.0.0/16
  perNodeBlockSize: 32
  gateway: 192.170.0.1
  nodeSelector:
    nodeSelectorTerms:
    - matchExpressions:
      - key: node-role.kubernetes.io/worker
        operator: Exists
---
apiVersion: nv-ipam.nvidia.com/v1alpha1
kind: IPPool
metadata:
  name: sriovibnet-pool-d-su-1
  namespace: network-operator
spec:
  subnet: 192.171.0.0/16
  perNodeBlockSize: 32
  gateway: 192.171.0.1
  nodeSelector:
    nodeSelectorTerms:
    - matchExpressions:
      - key: node-role.kubernetes.io/worker
        operator: Exists
---
apiVersion: nv-ipam.nvidia.com/v1alpha1
kind: IPPool
metadata:
  name: sriovibnet-pool-e-su-1
  namespace: network-operator
spec:
  subnet: 192.172.0.0/16
  perNodeBlockSize: 32
  gateway: 192.172.0.1
  nodeSelector:
    nodeSelectorTerms:
    - matchExpressions:
      - key: node-role.kubernetes.io/worker
        operator: Exists
---
apiVersion: nv-ipam.nvidia.com/v1alpha1
kind: IPPool
metadata:
  name: sriovibnet-pool-f-su-1
  namespace: network-operator
spec:
  subnet: 192.173.0.0/16
  perNodeBlockSize: 32
  gateway: 192.173.0.1
  nodeSelector:
    nodeSelectorTerms:
    - matchExpressions:
      - key: node-role.kubernetes.io/worker
        operator: Exists
---
apiVersion: nv-ipam.nvidia.com/v1alpha1
kind: IPPool
metadata:
  name: sriovibnet-pool-g-su-1
  namespace: network-operator
spec:
  subnet: 192.174.0.0/16
  perNodeBlockSize: 32
  gateway: 192.174.0.1
  nodeSelector:
    nodeSelectorTerms:
    - matchExpressions:
      - key: node-role.kubernetes.io/worker
        operator: Exists
---
apiVersion: nv-ipam.nvidia.com/v1alpha1
kind: IPPool
metadata:
  name: sriovibnet-pool-h-su-1
  namespace: network-operator
spec:
  subnet: 192.175.0.0/16
  perNodeBlockSize: 32
  gateway: 192.175.0.1
  nodeSelector:
    nodeSelectorTerms:
    - matchExpressions:
      - key: node-role.kubernetes.io/worker
        operator: Exists

combined-sriovibnet.yaml (SR/IOV Network Node Policy + SR/IOV IB Network)

apiVersion: sriovnetwork.openshift.io/v1
kind: SriovNetworkNodePolicy
metadata:
  name: sriovibnet-rdma-node-policy-a-su-1
  namespace: network-operator
spec:
  deviceType: netdevice
  nicSelector:
    vendor: "15b3"
    pfNames: [ "ibp24s0" ]
  numVfs: 8
  linkType: IB
  priority: 90
  isRdma: true
  resourceName: sriovib_resource_a
  nodeSelector:
    node-role.kubernetes.io/worker: ""
    feature.node.kubernetes.io/pci-15b3.present: "true"
---
apiVersion: sriovnetwork.openshift.io/v1
kind: SriovNetworkNodePolicy
metadata:
  name: sriovibnet-rdma-node-policy-b-su-1
  namespace: network-operator
spec:
  deviceType: netdevice
  nicSelector:
    vendor: "15b3"
    pfNames: [ "ibp64s0" ]
  numVfs: 8
  linkType: IB
  priority: 90
  isRdma: true
  resourceName: sriovib_resource_b
  nodeSelector:
    node-role.kubernetes.io/worker: ""
    feature.node.kubernetes.io/pci-15b3.present: "true"
---
apiVersion: sriovnetwork.openshift.io/v1
kind: SriovNetworkNodePolicy
metadata:
  name: sriovibnet-rdma-node-policy-c-su-1
  namespace: network-operator
spec:
  deviceType: netdevice
  nicSelector:
    vendor: "15b3"
    pfNames: [ "ibp79s0" ]
  numVfs: 8
  linkType: IB
  priority: 90
  isRdma: true
  resourceName: sriovib_resource_c
  nodeSelector:
    node-role.kubernetes.io/worker: ""
    feature.node.kubernetes.io/pci-15b3.present: "true"
---
apiVersion: sriovnetwork.openshift.io/v1
kind: SriovNetworkNodePolicy
metadata:
  name: sriovibnet-rdma-node-policy-d-su-1
  namespace: network-operator
spec:
  deviceType: netdevice
  nicSelector:
    vendor: "15b3"
    pfNames: [ "ibp94s0" ]
  numVfs: 8
  linkType: IB
  priority: 90
  isRdma: true
  resourceName: sriovib_resource_d
  nodeSelector:
    node-role.kubernetes.io/worker: ""
    feature.node.kubernetes.io/pci-15b3.present: "true"
---
apiVersion: sriovnetwork.openshift.io/v1
kind: SriovNetworkNodePolicy
metadata:
  name: sriovibnet-rdma-node-policy-e-su-1
  namespace: network-operator
spec:
  deviceType: netdevice
  nicSelector:
    vendor: "15b3"
    pfNames: [ "ibp154s0" ]
  numVfs: 8
  linkType: IB
  priority: 90
  isRdma: true
  resourceName: sriovib_resource_e
  nodeSelector:
    node-role.kubernetes.io/worker: ""
    feature.node.kubernetes.io/pci-15b3.present: "true"
---
apiVersion: sriovnetwork.openshift.io/v1
kind: SriovNetworkNodePolicy
metadata:
  name: sriovibnet-rdma-node-policy-f-su-1
  namespace: network-operator
spec:
  deviceType: netdevice
  nicSelector:
    vendor: "15b3"
    pfNames: [ "ibp192s0" ]
  numVfs: 8
  linkType: IB
  priority: 90
  isRdma: true
  resourceName: sriovib_resource_f
  nodeSelector:
    node-role.kubernetes.io/worker: ""
    feature.node.kubernetes.io/pci-15b3.present: "true"
---
apiVersion: sriovnetwork.openshift.io/v1
kind: SriovNetworkNodePolicy
metadata:
  name: sriovibnet-rdma-node-policy-g-su-1
  namespace: network-operator
spec:
  deviceType: netdevice
  nicSelector:
    vendor: "15b3"
    pfNames: [ "ibp206s0" ]
  numVfs: 8
  linkType: IB
  priority: 90
  isRdma: true
  resourceName: sriovib_resource_g
  nodeSelector:
    node-role.kubernetes.io/worker: ""
    feature.node.kubernetes.io/pci-15b3.present: "true"
---
apiVersion: sriovnetwork.openshift.io/v1
kind: SriovNetworkNodePolicy
metadata:
  name: sriovibnet-rdma-node-policy-h-su-1
  namespace: network-operator
spec:
  deviceType: netdevice
  nicSelector:
    vendor: "15b3"
    pfNames: [ "ibp220s0" ]
  numVfs: 8
  linkType: IB
  priority: 90
  isRdma: true
  resourceName: sriovib_resource_h
  nodeSelector:
    node-role.kubernetes.io/worker: ""
    feature.node.kubernetes.io/pci-15b3.present: "true"
---
apiVersion: sriovnetwork.openshift.io/v1
kind: SriovIBNetwork
metadata:
  name: "sriovibnet-rdma-default-a-su-1"
  namespace: network-operator
spec:
  linkState: enable
  networkNamespace: "default"
  resourceName: "sriovib_resource_a"
  ipam: |
   {
      "type": "nv-ipam",
      "datastore": "kubernetes",
      "kubernetes": {
         "kubeconfig": "/etc/cni/net.d/nv-ipam.d/nv-ipam.kubeconfig"
       },
      "log_file": "/var/log/SriovIBNetwork_nv-ipam.log",
      "log_level": "debug",
      "poolName": "sriovibnet-pool-a-su-1",
      "poolType": "IPPool"
    }
---
apiVersion: sriovnetwork.openshift.io/v1
kind: SriovIBNetwork
metadata:
  name: "sriovibnet-rdma-default-b-su-1"
  namespace: network-operator
spec:
  linkState: enable
  networkNamespace: "default"
  resourceName: "sriovib_resource_b"
  ipam: |
   {
      "type": "nv-ipam",
      "datastore": "kubernetes",
      "kubernetes": {
         "kubeconfig": "/etc/cni/net.d/nv-ipam.d/nv-ipam.kubeconfig"
       },
      "log_file": "/var/log/SriovIBNetwork_nv-ipam.log",
      "log_level": "debug",
      "poolName": "sriovibnet-pool-b-su-1",
      "poolType": "IPPool"
    }
---
apiVersion: sriovnetwork.openshift.io/v1
kind: SriovIBNetwork
metadata:
  name: "sriovibnet-rdma-default-c-su-1"
  namespace: network-operator
spec:
  linkState: enable
  networkNamespace: "default"
  resourceName: "sriovib_resource_c"
  ipam: |
   {
      "type": "nv-ipam",
      "datastore": "kubernetes",
      "kubernetes": {
         "kubeconfig": "/etc/cni/net.d/nv-ipam.d/nv-ipam.kubeconfig"
       },
      "log_file": "/var/log/SriovIBNetwork_nv-ipam.log",
      "log_level": "debug",
      "poolName": "sriovibnet-pool-c-su-1",
      "poolType": "IPPool"
    }
---
apiVersion: sriovnetwork.openshift.io/v1
kind: SriovIBNetwork
metadata:
  name: "sriovibnet-rdma-default-d-su-1"
  namespace: network-operator
spec:
  linkState: enable
  networkNamespace: "default"
  resourceName: "sriovib_resource_d"
  ipam: |
   {
      "type": "nv-ipam",
      "datastore": "kubernetes",
      "kubernetes": {
         "kubeconfig": "/etc/cni/net.d/nv-ipam.d/nv-ipam.kubeconfig"
       },
      "log_file": "/var/log/SriovIBNetwork_nv-ipam.log",
      "log_level": "debug",
      "poolName": "sriovibnet-pool-d-su-1",
      "poolType": "IPPool"
    }
---
apiVersion: sriovnetwork.openshift.io/v1
kind: SriovIBNetwork
metadata:
  name: "sriovibnet-rdma-default-e-su-1"
  namespace: network-operator
spec:
  linkState: enable
  networkNamespace: "default"
  resourceName: "sriovib_resource_e"
  ipam: |
   {
      "type": "nv-ipam",
      "datastore": "kubernetes",
      "kubernetes": {
         "kubeconfig": "/etc/cni/net.d/nv-ipam.d/nv-ipam.kubeconfig"
       },
      "log_file": "/var/log/SriovIBNetwork_nv-ipam.log",
      "log_level": "debug",
      "poolName": "sriovibnet-pool-e-su-1",
      "poolType": "IPPool"
    }
---
apiVersion: sriovnetwork.openshift.io/v1
kind: SriovIBNetwork
metadata:
  name: "sriovibnet-rdma-default-f-su-1"
  namespace: network-operator
spec:
  linkState: enable
  networkNamespace: "default"
  resourceName: "sriovib_resource_f"
  ipam: |
   {
      "type": "nv-ipam",
      "datastore": "kubernetes",
      "kubernetes": {
         "kubeconfig": "/etc/cni/net.d/nv-ipam.d/nv-ipam.kubeconfig"
       },
      "log_file": "/var/log/SriovIBNetwork_nv-ipam.log",
      "log_level": "debug",
      "poolName": "sriovibnet-pool-f-su-1",
      "poolType": "IPPool"
    }
---
apiVersion: sriovnetwork.openshift.io/v1
kind: SriovIBNetwork
metadata:
  name: "sriovibnet-rdma-default-g-su-1"
  namespace: network-operator
spec:
  linkState: enable
  networkNamespace: "default"
  resourceName: "sriovib_resource_g"
  ipam: |
   {
      "type": "nv-ipam",
      "datastore": "kubernetes",
      "kubernetes": {
         "kubeconfig": "/etc/cni/net.d/nv-ipam.d/nv-ipam.kubeconfig"
       },
      "log_file": "/var/log/SriovIBNetwork_nv-ipam.log",
      "log_level": "debug",
      "poolName": "sriovibnet-pool-g-su-1",
      "poolType": "IPPool"
    }
---
apiVersion: sriovnetwork.openshift.io/v1
kind: SriovIBNetwork
metadata:
  name: "sriovibnet-rdma-default-h-su-1"
  namespace: network-operator
spec:
  linkState: enable
  networkNamespace: "default"
  resourceName: "sriovib_resource_h"
  ipam: |
   {
      "type": "nv-ipam",
      "datastore": "kubernetes",
      "kubernetes": {
         "kubeconfig": "/etc/cni/net.d/nv-ipam.d/nv-ipam.kubeconfig"
       },
      "log_file": "/var/log/SriovIBNetwork_nv-ipam.log",
      "log_level": "debug",
      "poolName": "sriovibnet-pool-h-su-1",
      "poolType": "IPPool"
    }

sriov-node-pool-config.yaml

apiVersion: sriovnetwork.openshift.io/v1
 kind: SriovNetworkPoolConfig
 metadata:
   name: node-pool-unavailable-config
   namespace: network-operator
 spec:
   maxUnavailable: "100%"
   nodeSelector:
     matchExpressions:
       - key: node-role.kubernetes.io/worker
         operator: Exists