Appendix: Configuration Examples#
Section 1: OOB (example)#
Section: A1: IPMI Configuration (OOB)#
BASE Configuration
nv set system hostname OOB-01
nv set interface swp1-48
nv set interface swp1-48 bridge domain br_default
nv set interface swp1-48 bridge domain br_default access 10
nv set interface swp1-48 bridge domain br_default stp admin-edge on
nv set interface swp1-48 bridge domain br_default stp auto-edge on
nv set interface swp1-48 bridge domain br_default stp bpdu-guard on
nv set interface swp1-48 link state up
nv set interface swp1-48 type swp
UPLINK from OOB to SPINE
(non-routeable subnet): 10.254.0.0/16
Check which Spine interface is connected to downlink ports
nv set interface swp51-52 link state up
nv set interface swp51 ip address 10.254.253.1/31
nv set interface swp52 ip address 10.254.254.1/31
VRF: OOB
nv set vrf OOB
nv set vrf OOB evpn enable on
nv set vrf OOB evpn vni 10010
nv set vrf OOB router bgp enable on
nv set vrf OOB router bgp router-id 7.241.5.9
nv set vrf OOB loopback ip address 7.241.5.9/32
nv set vrf OOB router bgp autonomous-system 64601
nv set vrf OOB router bgp address-family l2vpn-evpn enable on
nv set vrf OOB router bgp address-family ipv4-unicast redistribute connected enable on
nv set vrf OOB router bgp address-family ipv4-unicast enable on
nv set vrf OOB router bgp address-family ipv4-unicast route-export to-evpn enable on
EVPN
nv set evpn enable on
nv set evpn multihoming enable on
nv set evpn multihoming startup-delay 10
nv set evpn route-advertise
nv set nve vxlan source address <LOOPBACK IP>
SVI
nv set interface vlan10 vlan 10
nv set interface vlan10 ip vrf OOB
nv set interface vlan10 type svi
nv set interface vlan10 ip address <SVI_VLAN_IP>
nv set bridge domain br_default vlan 10
nv set bridge domain br_default vlan 10 vni 10
nv set interface lo ip address <LOOPBACK IP>/32
nv set interface lo type loopback
POLICY
nv set router policy route-map lo_to_bgp rule 10 action permit
nv set router policy route-map lo_to_bgp rule 10 description permit_lo_ip
nv set router policy route-map lo_to_bgp rule 10 match interface lo
nv set router policy route-map lo_to_bgp rule 10 match type ipv4
nv set router policy route-map w_ecmp rule 10 action permit
nv set router policy route-map w_ecmp rule 10 description enable_w_ecmp
nv set router policy route-map w_ecmp rule 10 set ext-community-bw multipaths
BGP
nv set router bgp autonomous-system <4bytes_ASN_OOB_RANGE>
nv set router bgp enable on
nv set router bgp router-id <LOOPBACK IP>
nv set vrf default router bgp address-family ipv4-unicast enable on
nv set vrf default router bgp address-family ipv4-unicast redistribute connected enable on
nv set vrf default router bgp address-family ipv4-unicast redistribute connected route-map lo_to_bgp
nv set vrf default router bgp address-family l2vpn-evpn enable on
nv set vrf default router bgp enable on
nv set vrf default router bgp path-selection multipath aspath-ignore on
UNDERLAY NEIGHBOR
nv set vrf default router bgp neighbor 10.254.253.0 peer-group underlay
nv set vrf default router bgp neighbor 10.254.253.0 type numbered
nv set vrf default router bgp neighbor 10.254.254.0 peer-group underlay
nv set vrf default router bgp neighbor 10.254.254.0 type numbered
OVERLAY VIA SPINE LOOPBACK
nv set vrf default router bgp neighbor <SPINE1_LoopbackIP> peer-group overlay
nv set vrf default router bgp neighbor <SPINE1_LoopbackIP> type numbered
nv set vrf default router bgp neighbor <SPINE2_LoopbackIP> peer-group overlay
nv set vrf default router bgp neighbor <SPINE2_LoopbackIP> type numbered
PEER-GROUP: UNDERLAY
nv set vrf default router bgp peer-group underlay address-family ipv4-unicast policy outbound route-map w_ecmp
nv set vrf default router bgp peer-group underlay bfd detect-multiplier 3
nv set vrf default router bgp peer-group underlay bfd enable on
nv set vrf default router bgp peer-group underlay bfd min-rx-interval 300
nv set vrf default router bgp peer-group underlay bfd min-tx-interval 300
nv set vrf default router bgp peer-group underlay description underlay_switch_interconnect
nv set vrf default router bgp peer-group underlay remote-as external
PEER-GROUP: OVERLAY
nv set vrf default router bgp peer-group overlay address-family ipv4-unicast enable off
nv set vrf default router bgp peer-group overlay address-family l2vpn-evpn enable on
nv set vrf default router bgp peer-group overlay bfd detect-multiplier 3
nv set vrf default router bgp peer-group overlay bfd enable on
nv set vrf default router bgp peer-group overlay bfd min-rx-interval 1000
nv set vrf default router bgp peer-group overlay bfd min-tx-interval 1000
nv set vrf default router bgp peer-group overlay multihop-ttl 2
nv set vrf default router bgp peer-group overlay remote-as external
nv set vrf default router bgp peer-group overlay update-source lo
DHCP-RELAY
nv set vrf OOB evpn vlan 1001
nv set service dhcp-relay OOB interface vlan10
nv set service dhcp-relay OOB interface vlan1001_l3
nv set service dhcp-relay OOB server <DHCP_SERVER_HEADNODE_OOB_IP>
nv set service dhcp-relay OOB source-ip giaddress
Section 2: TOR (example)#
BTOR-0[1-2] Basic (Border Leaf)#
Basic system/service configuration
BTOR-0[x-y] CONTROLNODE BONDING#
BOND[ParentSwitchport#]_[Subport]
LOCAL-ID [1-72]
MAC-ADDRESS: 44:38:39:ff:[01]:[0A-0F]
BTOR-01 SVI#
# Create v[x]0[1-2] with the active VRR as part of the InBand VRF.
nv set interface vlan100 ip address 7.241.16.2/24
nv set interface vlan100 ip vrf INBAND
nv set interface vlan100 ip vrr address 7.241.16.1/24
nv set interface vlan100 ip vrr enable on
nv set interface vlan100 ip vrr state up
nv set interface vlan100 type svi
nv set interface vlan100 vlan 100
nv set bridge domain br_default vlan 100 vni 100
BTOR-02 SVI#
# Create v[x]0[1-2] with the active VRR as part of the InBand VRF.
nv set interface vlan100 ip address 7.241.16.3/24
nv set interface vlan100 ip vrf INBAND
nv set interface vlan100 ip vrr address 7.241.16.1/24
nv set interface vlan100 ip vrr enable on
nv set interface vlan100 ip vrr state up
nv set interface vlan100 type svi
nv set interface vlan100 vlan 100
nv set bridge domain br_default vlan 100 vni 100
BTOR-0[1-2] EVPN/VXLAN/RoCE#
Enable evpn, vxlan and roce
Create vrf: inband, storage, exit with inband and oob leaked route
nv set nve vxlan arp-nd-suppress on
nv set nve vxlan enable on
nv set nve vxlan source address <LOOPBACKUP IP>
nv set evpn multihoming enable on
nv set evpn enable on
nv set evpn route-advertise
nv set vrf STORAGE
nv set vrf INBAND
nv set vrf OOB
nv set vrf OOB evpn enable on
nv set vrf INBAND evpn enable on
nv set vrf STORAGE evpn enable on
nv set vrf INBAND evpn vni 200100
nv set vrf STORAGE evpn vni 400100
nv set vrf OOB evpn vni 100100
ROCE
nv set qos pfc default-global port-buffer 353000
nv set qos pfc default-global switch-priority 3
nv set qos pfc default-global xoff-threshold 65000
nv set qos pfc default-global xon-threshold 65000
nv set qos roce enable on
nv set qos roce mode lossless
nv set qos traffic-pool default-lossy memory-percent 10
nv set qos traffic-pool roce-lossless memory-percent 90
nv set router adaptive-routing enable on
nv set router adaptive-routing profile profile-custom
BTOR-0[1-2] BGP VRF#
An EVPN overlay will be established between VTEP using loopback interfaces.
Establish a BGP session for the underlay.
nv set vrf default router bgp path-selection multipath aspath-ignore on
nv set vrf default evpn enable on
nv set vrf default router bgp address-family ipv4-unicast enable on
nv set vrf default router bgp address-family ipv4-unicast redistribute connected enable on
nv set vrf default router bgp address-family ipv4-unicast redistribute connected route-map lo-to-bgp
nv set vrf default router bgp address-family l2vpn-evpn enable on
nv set vrf default router bgp enable on
nv set vrf default router bgp neighbor 7.241.17.9 peer-group overlay
nv set vrf default router bgp neighbor 7.241.17.9 type numbered
nv set vrf default router bgp neighbor 7.241.17.10 peer-group overlay
nv set vrf default router bgp neighbor 7.241.17.10 type numbered
nv set vrf default router bgp neighbor 10.254.32.128 peer-group underlay
nv set vrf default router bgp neighbor 10.254.32.128 type numbered
nv set vrf default router bgp neighbor 10.254.33.128 peer-group underlay
BTOR-0[1-2] BGP EDGE#
A /31 subnet of the InBand VRF will be established for point-to-point connections.
Create EXIT VRF with route leaked vrf (INBAND)
nv set vrf EXIT
nv set vrf EXIT evpn enable on
nv set vrf EXIT evpn vni 500100
nv set vrf EXIT router bgp enable on
nv set vrf EXIT router bgp autonomous-system <CUSTOMER BGP ASN>
nv set vrf EXIT router bgp enable on
nv set vrf EXIT router bgp router-id 7.241.17.1
nv set vrf EXIT router bgp address-family ipv4-unicast redistribute connected enable on
nv set vrf EXIT router bgp address-family ipv4-unicast route-export to-evpn
nv set vrf EXIT router bgp address-family l2vpn-evpn enable on
nv set vrf EXIT router bgp address-family ipv4-unicast route-import from-vrf enable on
nv set vrf EXIT router bgp address-family ipv4-unicast route-import from-vrf list INBAND
nv set vrf STORAGE evpn enable on
nv set vrf STORAGE evpn vni 400100
nv set vrf STORAGE router bgp address-family ipv4-unicast enable on
nv set vrf STORAGE router bgp address-family ipv4-unicast redistribute connected enable on
nv set vrf STORAGE router bgp address-family ipv4-unicast redistribute connected route-map RM_DGX_STORAGE
nv set vrf STORAGE router bgp address-family ipv4-unicast route-export to-evpn enable on
nv set vrf STORAGE router bgp address-family l2vpn-evpn enable on
nv set vrf STORAGE router bgp autonomous-system 4260037001
nv set vrf STORAGE router bgp enable on
nv set vrf STORAGE router bgp router-id 7.241.17.1
BTOR-0[1-2] POLICY#
The Underlay and Overlay policy will exclusively allow InBand and loopback subnets while discarding other traffic.
PEER-GROUP: UNDERLAY
nv set vrf default router bgp peer-group underlay address-family ipv4-unicast policy outbound route-map w_ecmp
nv set vrf default router bgp peer-group underlay bfd detect-multiplier 3
nv set vrf default router bgp peer-group underlay bfd enable on
nv set vrf default router bgp peer-group underlay bfd min-rx-interval 300
nv set vrf default router bgp peer-group underlay bfd min-tx-interval 300
nv set vrf default router bgp peer-group underlay description underlay_switch_interconnect
nv set vrf default router bgp peer-group underlay remote-as external
PEER-GROUP: OVERLAY
nv set vrf default router bgp peer-group overlay address-family ipv4-unicast enable off
nv set vrf default router bgp peer-group overlay address-family l2vpn-evpn enable on
nv set vrf default router bgp peer-group overlay bfd detect-multiplier 3
nv set vrf default router bgp peer-group overlay bfd enable on
nv set vrf default router bgp peer-group overlay bfd min-rx-interval 1000
nv set vrf default router bgp peer-group overlay bfd min-tx-interval 1000
nv set vrf default router bgp peer-group overlay multihop-ttl 2
nv set vrf default router bgp peer-group overlay remote-as external
nv set vrf default router bgp peer-group overlay update-source lo
BTOR-0[1-2] to SPINEs#
A policy will be implemented to prevent non-routable traffic from being advertised and to discard any that is received.
The handoff between customer edges will only receive the default route and will exclusively advertise summarized routes for the InBand root prefix.
BTOR-0[1-2] DHCP-Relay#
A DHCP instance will be established within the INBAND VRF
DHCP-relay
nv set vrf INBAND evpn vlan 2001
nv set service dhcp-relay INBAND interface vlan100
nv set service dhcp-relay INBAND interface vlan2001_l3
nv set service dhcp-relay INBAND server 7.241.16.8
nv set service dhcp-relay INBAND server 7.241.16.9
nv set service dhcp-relay INBAND server 7.241.16.10
nv set service dhcp-relay INBAND source-ip giaddress
BTOR-0[1-2] STORAGE VRF#
nv set vrf STORAGE evpn enable on
nv set vrf STORAGE evpn vni 400100
nv set vrf STORAGE loopback ip address 7.241.17.1/32
nv set vrf STORAGE router bgp address-family ipv4-unicast aggregate-route 100.127.0.0/26 summary-only on
nv set vrf STORAGE router bgp address-family ipv4-unicast enable on
nv set vrf STORAGE router bgp address-family ipv4-unicast redistribute connected enable on
nv set vrf STORAGE router bgp address-family ipv4-unicast redistribute connected route-map dgx_storage_subnets
nv set vrf STORAGE router bgp address-family ipv4-unicast route-export to-evpn enable on
nv set vrf STORAGE router bgp address-family l2vpn-evpn enable on
nv set vrf STORAGE router bgp autonomous-system 4260037000
nv set vrf STORAGE router bgp enable on
nv set vrf STORAGE router bgp router-id 7.241.17.1
BTOR-0[1-2] POLICY 2#
nv set router policy prefix-list PREFIX_DGX_STORAGE rule 10 action permit
nv set router policy prefix-list PREFIX_DGX_STORAGE rule 10 match 100.127.0.0/16 max-prefix-len 31
nv set router policy prefix-list PREFIX_DGX_STORAGE type ipv4
nv set router policy route-map RM_DGX_STORAGE rule 10 action permit
nv set router policy route-map RM_DGX_STORAGE rule 10 match ip-prefix-list PREFIX_DGX_STORAGE
nv set router policy route-map RM_DGX_STORAGE rule 10 match type ipv4
nv set router policy prefix-list PREFIX_INBAND_SUBNET rule 10 action permit
nv set router policy prefix-list PREFIX_INBAND_SUBNET rule 10 match 7.241.16.0/21
nv set router policy prefix-list PREFIX_INBAND_SUBNET type ipv4
nv set router policy route-map RM_INBAND_SUBNET_LO rule 10 action permit
nv set router policy route-map RM_INBAND_SUBNET_LO rule 10 match ip-prefix-list PREFIX_INBAND_SUBNET
nv set router policy route-map RM_INBAND_SUBNET_LO rule 10 match type ipv4
nv set router policy route-map RM_INBAND_SUBNET_LO rule 20 action permit
nv set router policy route-map RM_INBAND_SUBNET_LO rule 20 description permit_lo_ip
nv set router policy route-map RM_INBAND_SUBNET_LO rule 20 match interface lo
nv set router policy route-map RM_INBAND_SUBNET_LO rule 20 match type ipv4
nv set router policy route-map RM_W_ECMP rule 10 action permit
nv set router policy route-map RM_W_ECMP rule 10 description enable_RM_W_ECMP
nv set router policy route-map RM_W_ECMP rule 10 set ext-community-bw multipaths
nv set vrf default router bgp address-family ipv4-unicast enable on
nv set vrf default router bgp address-family ipv4-unicast redistribute connected enable on
nv set vrf default router bgp address-family ipv4-unicast redistribute connected route-map RM_INBAND_SUBNET_LO
nv set vrf default router bgp address-family l2vpn-evpn enable on
nv set vrf default router bgp enable on
Inband Policy Inserts Default Route#
nv set router policy prefix-list PREFIX_DEFAULT rule 10 action permit
nv set router policy prefix-list PREFIX_DEFAULT rule 10 match 0.0.0.0/0
nv set router policy prefix-list PREFIX_DEFAULT type ipv4
nv set router policy route-map ONLY_DEFAULT_ROUTE rule 10 action permit
nv set router policy route-map ONLY_DEFAULT_ROUTE rule 10 match ip-prefix-list PREFIX_DEFAULT
nv set router policy route-map ONLY_DEFAULT_ROUTE rule 10 match type ipv4
nv set vrf INBAND router bgp address-family ipv4-unicast route-import from-vrf enable on
nv set vrf INBAND router bgp address-family ipv4-unicast route-import from-vrf list EXIT
nv set vrf INBAND router bgp address-family ipv4-unicast route-import from-vrf route-map ONLY_DEFAULT_ROUTE
Connection Towards Customer#
nv set vrf EXIT
nv set vrf EXIT loopback ip address <LoopbackIP>/32
nv set interface swp53 link breakout <2,4,8>x
nv set interface swp53s0-<#> ip vrf EXIT
nv set interface swp53s0 ip address <IP>/31
nv set vrf EXIT router bgp autonomous-system <CUSTOMER-ASN>
nv set vrf EXIT router bgp enable on
nv set vrf EXIT router bgp address-family ipv4-unicast enable on
nv set vrf EXIT router bgp address-family ipv4-unicast redistribute connected enable on
nv set vrf EXIT router bgp address-family ipv4-unicast route-import from-vrf enable on
nv set vrf EXIT router bgp address-family ipv4-unicast route-import from-vrf list INBAND
nv set vrf EXIT router bgp address-family ipv4-unicast route-import from-vrf list OOB
nv set vrf EXIT router bgp neighbor <PEER-IP> remote-as external
nv set vrf EXIT router bgp neighbor <PEER-IP> type numbered
SEND INBAND ROUTE TO CUSTOMER
nv set vrf INBAND router bgp address-family ipv4-unicast route-export to-evpn enable on
nv set vrf INBAND router bgp address-family ipv4-unicast route-import from-vrf enable on
nv set vrf INBAND router bgp address-family ipv4-unicast route-import from-vrf list EXIT
Aggregate Summary Routes towards Edge#
nv set vrf EXIT router bgp address-family ipv4-unicast aggregate-route <OOB_root_prefix> summary-only
These commands will advertise a single subnet along with the origin ASN, without including any internal pod ASN information in the announcement.
Section 3: GB200 Rack Inventory (Example)#
Column header which factory can populate data in the below format.
PO# |
VENDOR |
RACK_PN |
RACK_SN |
TAG |
|---|---|---|---|---|
[example] |
[example] |
[example] |
[example] |
[example] |
LOCATION |
DEVICE_PN |
DEVICE_SN |
DEVICE_TYPE |
|---|---|---|---|
[example] |
[example] |
[example] |
[example] |
MGMT1_MAC |
MGMT2_MAC |
MGMT3_MAC |
BF3_1_DATA1_MAC |
BF3_1_DATA2_MAC |
BF3_2_DATA1_MAC |
BF3_2_DATA2_MAC |
|---|---|---|---|---|---|---|
[example] |
[example] |
[example] |
[example] |
[example] |
[example] |
[example] |