Page History

For each SBC HA pair, use unique subnet for pkt0 and pkt1.

The interfaces may sometimes display in the incorrect order on the HFE node at the Linux level. However, this is not an issue because the HFE script ensures the entire configuration is set up correctly based on the the Azure NICs, not the local interface names.

--serverice-endpoints is required to allow the HFE to download the HFE script from storage.

az network vnet subnet create --name <NAME>
                              --address-prefixes <CIDR>
                              --resource-group <RESOURCE-GROUP-NAME>
                              --vnet-name <VNET_NAME>
                              --network-security-group <SECURITY GROUP NAME>
							  --service-endpoints Microsoft.Storage

az network vnet subnet create --name pkt0--hfe --address-prefixes 10.2.4.0/24 --resource-group RBBN-SBC-RG --vnet-name RibbonNet --network-security-group pkt0RbbnSbcSG --service-endpoints Microsoft.Storage

az network vnet subnet create --name pkt1--hfe --address-prefixes 10.2.5.0/24 --resource-group RBBN-SBC-RG --vnet-name RibbonNet --network-security-group Pkt1RbbnSbcSG --service-endpoints Microsoft.Storage

Content-Type: multipart/mixed; boundary="//"
MIME-Version: 1.0
--//
Content-Type: text/cloud-config; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment; filename="cloud-config.txt"
#cloud-config
cloud_final_modules:
- [scripts-user, always]
--//
Content-Type: text/x-shellscript; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment; filename="userdata.txt"
#!/bin/bash
HFE_DIR="/opt/HFE"
HFE_LOG_DIR="$HFE_DIR/log"
HFE_FILE="$HFE_DIR/HFE_AZ.sh"
LOG_FILE="$HFE_LOG_DIR/cloud-init-nat.log"
NAT_VAR="$HFE_DIR/natVars.input"
TEMP_MGMT_ROUTE="$HFE_DIR/.tempRoute"
AZ_BLOB_URL="<HFE_SCRIPT_LOCATION>" # URL of uploaded HFE script
timestamp()
{
  date +"%Y-%m-%d %T"
}
if [ ! -d $HFE_LOG_DIR ]; then
  mkdir -p $HFE_LOG_DIR;
fi;

/bin/echo $(timestamp) " ========================= cloud-init configuration for HFE ==========================================" >> $LOG_FILE

#Fix any interfaces
defaultRoute=$(ip route | grep default) # There will only be 1 default route with a metric 100
ip a | grep -E eth.: | grep DOWN | awk -F' ' '{print $2}' | sed 's/://' | while read intf; do
  /bin/echo $(timestamp) "Bringing up $intf" >> $LOG_FILE
  dhclient $intf
  ip route | grep -E "default.*$intf" | while read r; do
    if [[ "$r" != "$defaultRoute" ]];then
      if [ $(echo $r | grep -c metric) -eq 0 ]; then
        /bin/echo $(timestamp) "Deleting new route $r" >> $LOG_FILE
        ip route delete $r
      fi
    fi
  done
done

#Test for internet access
curl --connect-timeout 10 https://management.azure.com
if [ $? -ne 0 ]; then
  for i in $(seq 1 $(ip a | grep -E eth.: | grep -c -v eth0)); do
    MGT_INTF_NAME=eth$i
    cidrIp=$(ip route | grep "$MGT_INTF_NAME proto kernel scope link" | awk -F " " '{print $1}' | awk -F "/" '{print $1}')
    finalOct=$(echo $cidrIp | awk -F "." '{print $4}')
    gwOct=$(( finalOct + 1 ))
    mgtGwIp=$(echo $cidrIp | awk -v var="$gwOct" -F. '{$NF=var}1' OFS=.)
    echo -e "tempMgtGw=$mgtGwIp\ntempMgtIntf=$MGT_INTF_NAME" > $TEMP_MGMT_ROUTE
    /bin/echo $(timestamp) "Adding temporary default route for $MGT_INTF_NAME" >> $LOG_FILE
    ip route add 0.0.0.0/0 via $mgtGwIp dev $MGT_INTF_NAME metric 10
    curl --connect-timeout 10 https://management.azure.com
    if [ $? -eq 0 ]; then
      break
    else
      rm $TEMP_MGMT_ROUTE
      /bin/echo $(timestamp) "Removing temporary default route for $MGT_INTF_NAME" >> $LOG_FILE
      ip route delete 0.0.0.0/0 via $mgtGwIp dev $MGT_INTF_NAME metric 10
    fi
  done < <(ip a |  grep -E eth.:)
fi

curl --connect-timeout 10 "$AZ_BLOB_URL" -H 'x-ms-version : 2019-02-02' -o $HFE_FILE
if [ $? -ne 0 ]; then
  /bin/echo $(timestamp) "Error:Could not copy HFE script from Azure Blob Container." >> $LOG_FILE
else
  /bin/echo $(timestamp) "Copied HFE script from Azure Blob Container." >> $LOG_FILE
fi;
/bin/echo > $NAT_VAR
/bin/echo "ACTIVE_SBC_VM_NAME=\"<ACTIVE_SBC_NAME>\"" >> $NAT_VAR
/bin/echo "STANDBY_SBC_VM_NAME=\"<STANDBY_SBC_NAME>\"" >> $NAT_VAR
/bin/echo "REMOTE_SSH_MACHINE_IP=\"<REMOTE_SSH_MACHINE_IP>\"" >> $NAT_VAR
/bin/echo "SBC_PKT_PORT_NAME=\"<SBC_PKT_PORT_NAME>\"" >> $NAT_VAR
/bin/echo "CUSTOM_ROUTES=\"<CUSTOM_STATIC_ROUTES_CONFIG>\"" >> $NAT_VAR
/bin/echo "ENABLE_PKT_DNS_QUERY=<0/1>" >> $NAT_VAR
/bin/echo $(timestamp) "Copied natVars.input" >> $LOG_FILE
sudo chmod 744 $HFE_FILE
/bin/echo $(timestamp) "Configured using HFE script - $HFE_FILE" >> $LOG_FILE
/bin/echo $(timestamp) " ========================= Done ==========================================" >> $LOG_FILE
nohup $HFE_FILE setup > /dev/null 2>&1 &

az network public-ip create --name <PUBLIC IP NAME> --resource-group <RESOURCE-GROUP-NAME> --allocation-method Static

az network public-ip create --name pkt0-mgmt-ip --resource-group RBBN-SBC-RG --allocation-method Static
 
az network public-ip create --name hfe-pkt0-ip --resource-group RBBN-SBC-RG --allocation-method Static
 
az network public-ip create --name pkt1-mgmt-ip --resource-group RBBN-SBC-RG --allocation-method Static

az network nic create --name <NIC NAME>
                      --resource-group <RESOURCE-GROUP-NAME>
                      --vnet-name <VIRTUAL NETWORK NAME>
                      --subnet <SUBNET NAME>
                      --network-security-group <SECURITY GROUP NAME>

az network nic create --name hfe-pkt0-nic0 --resource-group RBBN-SBC-RG --vnet-name RibbonNet --subnet SubnetMgmt --network-security-group pkt0RbbnSbcSG --public-ip-address hfe-pkt0-ip
az network nic create --name hfe-pkt0-nic1 --resource-group RBBN-SBC-RG --vnet-name RibbonNet --subnet SubnetMgmt --network-security-group mgmtRbbnSbcSG --public-ip-address pkt0-mgmt-ip
az network nic create --name hfe-pkt0-nic2 --resource-group RBBN-SBC-RG --vnet-name RibbonNet --subnet SubnetMgmt --network-security-group pkt0RbbnSbcSG

az network nic create --name hfe-pkt1-nic0 --resource-group RBBN-SBC-RG --vnet-name RibbonNet --subnet SubnetMgmt --network-security-group pkt1RbbnSbcSG
az network nic create --name hfe-pkt1-nic1 --resource-group RBBN-SBC-RG --vnet-name RibbonNet --subnet SubnetMgmt --network-security-group mgmtRbbnSbcSG --public-ip-address pkt0-mgmt-ip
az network nic create --name hfe-pkt1-nic2 --resource-group RBBN-SBC-RG --vnet-name RibbonNet --subnet SubnetMgmt --network-security-group pkt1RbbnSbcSG

az vm create --name <INSTANCE NAME>
             --resource-group <RESOURCE_GROUP_NAME>
             --admin-username <UserName>
             --custom-data <USERDATA FILE>
             --image <IMAGE NAME>
             --location <LOCATION>
             --size <INSTANCE SIZE>
             --ssh-key-values <PUBLIC SSH KEY FILENAME>
             --nics <ETH0 NIC> <ETH1 NIC> <ETH2 NIC>
             --boot-diagnostics-storage <STORAGE ACCOUNT NAME>
             --assign-identity <USER ASSIGNED MANAGED IDENTITY ID>

Consider the following when creating routes in Azure:

• Custom routes are not given complete priority over the standard Azure routing. If there is a more specific Azure route, Azure directs traffic based on that default rule.
  • For example, if you create a custom route to 0.0.0.0/0 via the HFE, but send the traffic to a private IP within the same Virtual network, Azure does NOT route via the HFE eth2. Instead, the traffic flows directly from the SBC to the private IP. Refer to Virtual network traffic routing on the Microsoft Azure website for information on default rules.
  • If a custom route destination matches a default route destination, the SBC uses the custom route.

• Spacevars
  0 company

   recommends to supply a specific IP or CIDR of the endpoints as the destination IP, to prevent any routing issues with the Azure default routing rules.
• Routes are applied to all traffic within a subnet. If multiple routes using the same destination address exist, the SBC can use any route.
  • If multiple SBC setups are using the same endpoint (for example, in an SLB/SBC setup), separate the SBCs into separate subnets and route tables to ensure they route to the correct HFE. Refer to Configure SBC SWe on Azure for SLB, for more information.

az network nic create --name <NIC NAME>
                      --resource-group <RESOURCE GROUP NAME>
                      --vnet-name <VIRTUAL NETWORK NAME>
                      --subnet <SUBNET NAME>
                      --network-security-group <SECURITY GROUP NAME>
                      --accelerated-networking true

az network nic create --name sbc1-pkt0 --resource-group RBBN-SBC-RG --vnet-name RibbonNet --subnet pkt0 --network-security-group pkt0RbbnSbcSG--ip-forwarding
az network nic create --name sbc1-pkt1 --resource-group RBBN-SBC-RG --vnet-name RibbonNet --subnet pkt1 --network-security-group pkt1RbbnSbcSG--ip-forwarding

az network nic create --name sbc2-pkt0 --resource-group RBBN-SBC-RG --vnet-name RibbonNet --subnet pkt0 --network-security-group pkt0RbbnSbcSG--ip-forwarding
az network nic create --name sbc2-pkt1 --resource-group RBBN-SBC-RG --vnet-name RibbonNet --subnet pkt1 --network-security-group pkt1RbbnSbcSG--ip-forwarding

Because the HFE Node receives all the traffic, it is not necessary to create Public IP addresses for these ports, or add them to the NICs.

• Before creating the Secondary IP configuration, create the NICSs for the SBCs.
• You cannot set the IP config name as "ipconfig1", because it is reserved for the primary IP configuration on a NIC.

az network nic ip-config create --name <NAME> --nic-name <PKT0/PKT1 NIC NAME> --resource-group <RESOURCE_GROUP_NAME>

az network nic ip-config create --name sbc1-pkt0-secIp --nic-name sbc1-pkt0 --resource-group RBBN-SBC-RG
az network nic ip-config create --name sbc1-pkt1-secIp --nic-name sbc1-pkt0 --resource-group RBBN-SBC-RG

az network nic ip-config create --name sbc2-pkt0-secIp --nic-name sbc1-pkt0 --resource-group RBBN-SBC-RG
az network nic ip-config create --name sbc2-pkt1-secIp --nic-name sbc1-pkt0 --resource-group RBBN-SBC-RG

{   
  "CEName" : "<SBC CE NAME>",
  "ReverseNatPkt0" : "True",
  "ReverseNatPkt1" : "True",
  "SystemName" : "<SYSTEM NAME>",
  "SbcPersonalityType": "isbc",
  "AdminSshKey" : "<ssh-rsa ...>",
  "ThirdPartyCpuAlloc" : "<0-4>",
  "ThirdPartyMemAlloc" : "<0-4096>",
  "CERole" : "<ACTIVE/STANDBY>",
  "PeerCEHa0IPv4Address" : "<PEER HA IP ADDRESS>",
  "ClusterIp" : "<PEER HA IP ADDRESS>",
  "PeerCEName" : "<PEER SBC CE NAME>",
  "SbcHaMode" : "1to1",
  "PeerInstanceName" : "<PEER INSTANCE NAME>",
  "Pkt0HfeInstanceName" : "<PKT0 HFE NODE INSTANCE NAME>",
  "Pkt1HfeInstanceName" : "<PKT1 HFE NODE INSTANCE NAME>" 
}

• The SBC requires user data in a valid JSON format. If the user-data is not a valid JSON, the instance shuts down immediately.
• You cannot update user data on VMs in the Azure framework.

admin@sbc-10.2.2.12> conf
Entering configuration mode private
[ok][2019-10-04 09:04:15]
 
[edit]
admin@sbc-10.2.2.12% set addressContext default ipInterfaceGroup LIG1 ipInterface LIF1 portName pkt0 ipPublicVarV4 IF2.IPV4 prefixVarV4 IF2.PrefixV4 mode inService state enabled
[ok][2019-10-04 09:04:46]
 
[edit]
admin@sbc-10.2.2.12% commit
Commit complete.
[ok][2019-10-04 09:04:50]
 
[edit]
admin@sbc-10.2.2.12% set addressContext default ipInterfaceGroup LIG2 ipInterface LIF2 portName pkt1 ipPublicVarV4 IF3.IPV4 prefixVarV4 IF3.PrefixV4 mode inService state enabled
[ok][2019-10-04 09:04:58]
 
[edit]
admin@sbc-10.2.2.12% com
Commit complete.
[ok][2019-10-04 09:05:00]
 
[edit]
admin@sbc-10.2.2.12% set addressContext default staticRoute 0.0.0.0 0 <PKT0 SUBNET GATEWAY> LIG1 LIF1 preference 100
[ok][2019-10-04 09:05:11]
 
[edit]
admin@sbc-10.2.2.12% com
Commit complete.
[ok][2019-10-04 09:05:15]
 
[edit]
admin@sbc-10.2.2.12% set addressContext default staticRoute 0.0.0.0 0 <PKT1 SUBNET GATEWAY> LIG2 LIF2 preference 100
[ok][2019-10-04 09:05:22]
 
[edit]
admin@sbc-10.2.2.12% com
Commit complete.
[ok][2019-10-04 09:05:24]
 
[edit]
admin@sbc-10.2.2.12%

admin@sbc-10.2.2.12> show table addressContext default staticRoute
                               IP
                               INTERFACE  IP
DESTINATION                    GROUP      INTERFACE              CE
IP ADDRESS   PREFIX  NEXT HOP  NAME       NAME       PREFERENCE  NAME
-----------------------------------------------------------------------
0.0.0.0      0       10.2.3.1  LIG1       LIF1       100         -
0.0.0.0      0       10.2.4.1  LIG2       LIF2       100         -
[ok][2019-10-04 09:16:47]
admin@sbc-10.2.2.12>
admin@sbc-10.2.2.12> show table addressContext default ipInterfaceGroup
 
                                                                                                                                                               IP      IP           IP
                      CE    PORT  IP               ALT IP   ALT                        DRYUP             BW           VLAN             IP VAR    PREFIX VAR    PUBLIC  VAR  PREFIX  PUBLIC
NAME  IPSEC     NAME  NAME  NAME  ADDRESS  PREFIX  ADDRESS  PREFIX  MODE       ACTION  TIMEOUT  STATE    CONTINGENCY  TAG   BANDWIDTH  V4        V4            VAR V4  V6   VAR V6  VAR V6
-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
LIG1  disabled  LIF1  -     pkt0  -        -       -        -       inService  dryUp   60       enabled  0            -     0          IF2.IPV4  IF2.PrefixV4  -       -    -       -
LIG2  disabled  LIF2  -     pkt1  -        -       -        -       inService  dryUp   60       enabled  0            -     0          IF3.IPV4  IF3.PrefixV4  -       -    -       -
[ok][2019-10-04 09:18:35]

admin@act-10.2.2.127> show table system metaVariable
CE NAME         NAME                  VALUE
-----------------------------------------------------
act-10.2.2.127  IF0.GWV4              10.2.0.1
act-10.2.2.127  IF0.IPV4              10.2.0.9
act-10.2.2.127  IF0.Port              Mgt0
act-10.2.2.127  IF0.RNat              True
act-10.2.2.127  IF1.GWV4              10.2.2.1
act-10.2.2.127  IF1.IPV4              10.2.2.127
act-10.2.2.127  IF1.Port              Ha0
act-10.2.2.127  IF1.RNat              True
act-10.2.2.127  IF2.GWV4              10.2.3.1
act-10.2.2.127  IF2.IPV4              10.2.3.10
act-10.2.2.127  IF2.Port              Pkt0
act-10.2.2.127  IF2.RNat              True
act-10.2.2.127  IF3.GWV4              10.2.4.1
act-10.2.2.127  IF3.IPV4              10.2.4.10
act-10.2.2.127  IF3.Port              Pkt1
act-10.2.2.127  IF3.RNat              True
act-10.2.2.127  IF0.FIPV4             137.117.73.22
act-10.2.2.127  IF0.PrefixV4          24
act-10.2.2.127  IF1.PrefixV4          24
act-10.2.2.127  IF2.PrefixV4          24
act-10.2.2.127  IF3.PrefixV4          24
act-10.2.2.127  HFE_IF2.FIPV4         52.168.34.216
act-10.2.2.127  HFE_IF3.FIPV4         10.2.2.7
act-10.2.2.127  HFE_IF2.IFName        IF_HFE_PKT0
act-10.2.2.127  HFE_IF3.IFName        IF_HFE_PKT1
act-10.2.2.127  secondaryIPList.Pkt0  ['10.2.3.10']
act-10.2.2.127  secondaryIPList.Pkt1  ['10.2.4.10']
sby-10.2.2.227  IF0.GWV4              10.2.0.1
sby-10.2.2.227  IF0.IPV4              10.2.0.14
sby-10.2.2.227  IF0.Port              Mgt0
sby-10.2.2.227  IF0.RNat              True
sby-10.2.2.227  IF1.GWV4              10.2.2.1
sby-10.2.2.227  IF1.IPV4              10.2.2.227
sby-10.2.2.227  IF1.Port              Ha0
sby-10.2.2.227  IF1.RNat              True
sby-10.2.2.227  IF2.GWV4              10.2.3.1
sby-10.2.2.227  IF2.IPV4              10.2.3.10
sby-10.2.2.227  IF2.Port              Pkt0
sby-10.2.2.227  IF2.RNat              True
sby-10.2.2.227  IF3.GWV4              10.2.4.1
sby-10.2.2.227  IF3.IPV4              10.2.4.10
sby-10.2.2.227  IF3.Port              Pkt1
sby-10.2.2.227  IF3.RNat              True
sby-10.2.2.227  IF0.FIPV4             40.76.8.39
sby-10.2.2.227  IF0.PrefixV4          24
sby-10.2.2.227  IF1.PrefixV4          24
sby-10.2.2.227  IF2.PrefixV4          24
sby-10.2.2.227  IF3.PrefixV4          24
sby-10.2.2.227  HFE_IF2.FIPV4         52.168.34.216
sby-10.2.2.227  HFE_IF3.FIPV4         10.2.2.7
sby-10.2.2.227  HFE_IF2.IFName        IF_HFE_PKT0
sby-10.2.2.227  HFE_IF3.IFName        IF_HFE_PKT1
sby-10.2.2.227  secondaryIPList.Pkt0  ['10.2.3.11']
sby-10.2.2.227  secondaryIPList.Pkt1  ['10.2.4.11']
[ok][2019-10-07 11:48:16]
admin@act-10.2.2.127>

echo "CUSTOM_ROUTES=\"<DESTINATION_IP_CIDR>_<INTERFACE_NAME>, <DESTINATION_IP_CIDR>_<INTERFACE_NAME>\"" | sudo tee -a /opt/HFE/natVars.user

For <INTERFACE_NAME>, use the standard eth0, eth1, and so on always even if the Linux distribution does not use this naming convention. The HFE_AZ.sh determines the interface to add the route.