CLOUDSTACK-9175: [VMware DRS] Adding new host to DRS cluster does not participate in load balancing.

Summary: When a new host is added to a cluster, Cloudstack doesn't create all the port groups (created by cloudstack earlier in other hosts) present in the cluster. Since the new host doesn't have all the necessary networking port groups of cloudstack, it is not eligible to participate in DRS load balancing or HA.

Solution: When adding a host to the cluster in Cloudstack, use VMware API to find the list of unique port groups on a previously added host (older host in the cluster) if exists and then create them on the new host.

Author: sureshanaparti

engine/schema/src/main/java/com/cloud/host/dao/HostDao.java

@@ -75,6 +75,8 @@ public interface HostDao extends GenericDao<HostVO, Long>, StateDao<Status, Stat
 
     List<HostVO> findHypervisorHostInCluster(long clusterId);
 
+    HostVO findOldestExistentHypervisorHostInCluster(long clusterId);
+
     List<HostVO> listAllUpAndEnabledNonHAHosts(Type type, Long clusterId, Long podId, long dcId, String haTag);
 
     List<HostVO> findByDataCenterId(Long zoneId);

engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java

@@ -1168,6 +1168,23 @@ public List<HostVO> findHypervisorHostInCluster(long clusterId) {
         return listBy(sc);
     }
 
+    @Override
+    public HostVO findOldestExistentHypervisorHostInCluster(long clusterId) {
+        SearchCriteria<HostVO> sc = TypeClusterStatusSearch.create();
+        sc.setParameters("type", Host.Type.Routing);
+        sc.setParameters("cluster", clusterId);
+        sc.setParameters("status", Status.Up);
+        sc.setParameters("resourceState", ResourceState.Enabled);
+        Filter orderByFilter = new Filter(HostVO.class, "created", true, null, null);
+
+        List<HostVO> hosts = search(sc, orderByFilter, null, false);
+        if (hosts != null && hosts.size() > 0) {
+            return hosts.get(0);
+        }
+
+        return null;
+    }
+
     @Override
     public List<Long> listAllHosts(long zoneId) {
         SearchCriteria<Long> sc = HostIdSearch.create();

plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java

@@ -49,6 +49,7 @@
 import com.cloud.exception.OperationTimedoutException;
 import com.cloud.exception.ResourceInUseException;
 import com.cloud.host.Host;
+import com.cloud.host.HostVO;
 import com.cloud.host.Status;
 import com.cloud.host.dao.HostDao;
 import com.cloud.host.dao.HostDetailsDao;
@@ -100,6 +101,7 @@
 import com.cloud.utils.FileUtil;
 import com.cloud.utils.NumbersUtil;
 import com.cloud.utils.Pair;
+import com.cloud.utils.UriUtils;
 import com.cloud.utils.component.ManagerBase;
 import com.cloud.utils.concurrency.NamedThreadFactory;
 import com.cloud.utils.db.DB;
@@ -145,6 +147,7 @@
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
+import java.net.URLDecoder;
 import java.rmi.RemoteException;
 import java.time.Duration;
 import java.time.Instant;
@@ -446,6 +449,29 @@ private void prepareHost(HostMO hostMo, String privateTrafficLabel) throws Excep
         }
     }
 
+    private HostMO getOldestExistentHostInCluster(Long clusterId, VmwareContext serviceContext) throws Exception {
+        HostVO host = hostDao.findOldestExistentHypervisorHostInCluster(clusterId);
+        if (host == null) {
+            return null;
+        }
+
+        ManagedObjectReference morSrcHost = HypervisorHostHelper.getHypervisorHostMorFromGuid(host.getGuid());
+        if (morSrcHost == null) {
+            Map<String, String> clusterDetails = clusterDetailsDao.findDetails(clusterId);
+            if (clusterDetails.get("url") == null) {
+                return null;
+            }
+
+            URI uriForHost = new URI(UriUtils.encodeURIComponent(clusterDetails.get("url") + "/" + host.getName()));
+            morSrcHost = serviceContext.getHostMorByPath(URLDecoder.decode(uriForHost.getPath(), "UTF-8"));
+            if (morSrcHost == null) {
+                return null;
+            }
+        }
+
+        return new HostMO(serviceContext, morSrcHost);
+    }
+
     @Override
     public List<ManagedObjectReference> addHostToPodCluster(VmwareContext serviceContext, long dcId, Long podId, Long clusterId, String hostInventoryPath)
             throws Exception {
@@ -498,6 +524,11 @@ public List<ManagedObjectReference> addHostToPodCluster(VmwareContext serviceCon
                 // For ESX host, we need to enable host firewall to allow VNC access
                 HostMO hostMo = new HostMO(serviceContext, mor);
                 prepareHost(hostMo, privateTrafficLabel);
+                HostMO olderHostMo = getOldestExistentHostInCluster(clusterId, serviceContext);
+                if (olderHostMo != null) {
+                    hostMo.copyPortGroupsFromHost(olderHostMo);
+                }
+
                 returnedHostList.add(mor);
                 return returnedHostList;
             } else {

vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HostMO.java

@@ -51,6 +51,7 @@
 import com.vmware.vim25.HostNetworkTrafficShapingPolicy;
 import com.vmware.vim25.HostOpaqueNetworkInfo;
 import com.vmware.vim25.HostPortGroup;
+import com.vmware.vim25.HostPortGroupPort;
 import com.vmware.vim25.HostPortGroupSpec;
 import com.vmware.vim25.HostRuntimeInfo;
 import com.vmware.vim25.HostSystemConnectionState;
@@ -130,6 +131,43 @@ public HostPortGroupSpec getHostPortGroupSpec(String portGroupName) throws Excep
         return null;
     }
 
+    public List<HostPortGroupSpec> getHostPortGroupSpecs() throws Exception {
+        HostNetworkInfo hostNetInfo = getHostNetworkInfo();
+        if (hostNetInfo == null) {
+            return null;
+        }
+
+        List<HostPortGroup> portGroups = hostNetInfo.getPortgroup();
+        if (CollectionUtils.isEmpty(portGroups)) {
+            return null;
+        }
+
+        List<HostPortGroupSpec> portGroupSpecs = new ArrayList<HostPortGroupSpec>();
+        for (HostPortGroup portGroup : portGroups) {
+            if (!isVMKernelPort(portGroup)) {
+                portGroupSpecs.add(portGroup.getSpec());
+            }
+        }
+
+        return portGroupSpecs;
+    }
+
+    private boolean isVMKernelPort(HostPortGroup portGroup) {
+        assert (portGroup != null);
+        List<HostPortGroupPort> ports = portGroup.getPort();
+        if (CollectionUtils.isEmpty(ports)) {
+            return false;
+        }
+
+        for (HostPortGroupPort port : ports) {
+            if (port.getType().equalsIgnoreCase("host")) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
     @Override
     public String getHyperHostName() throws Exception {
         return getName();
@@ -1156,6 +1194,41 @@ public String getNetworkName(String netMorVal) throws Exception {
         return networkName;
     }
 
+    public void createPortGroup(HostPortGroupSpec spec) throws Exception {
+        if (spec == null) {
+            return;
+        }
+
+        synchronized (_mor.getValue().intern()) {
+            HostNetworkSystemMO hostNetMo = getHostNetworkSystemMO();
+            if (hostNetMo == null) {
+                return;
+            }
+
+            ManagedObjectReference morNetwork = getNetworkMor(spec.getName());
+            if (morNetwork == null) {
+                hostNetMo.addPortGroup(spec);
+            }
+        }
+    }
+
+    public void copyPortGroupsFromHost(HostMO srcHost) throws Exception {
+        if (srcHost == null) {
+            return;
+        }
+
+        List<HostPortGroupSpec> portGroupSpecs = srcHost.getHostPortGroupSpecs();
+        if (CollectionUtils.isEmpty(portGroupSpecs)) {
+            s_logger.debug("No port groups in the host: " + srcHost.getName());
+            return;
+        }
+
+        for (HostPortGroupSpec spec : portGroupSpecs) {
+            s_logger.debug("Creating port group: " + spec.getName() + " in the host: " + getName());
+            createPortGroup(spec);
+        }
+    }
+
     public void createPortGroup(HostVirtualSwitch vSwitch, String portGroupName, Integer vlanId,
             HostNetworkSecurityPolicy secPolicy, HostNetworkTrafficShapingPolicy shapingPolicy, long timeOutMs)
             throws Exception {

vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java

@@ -2187,6 +2187,29 @@ public static String getOVFParamValue(VirtualMachineConfigSpec config) {
        return paramVal;
    }
 
+    public static ManagedObjectReference getHypervisorHostMorFromGuid(String guid) {
+        if (guid == null) {
+            return null;
+        }
+
+        String[] tokens = guid.split("@");
+        if (tokens == null || tokens.length != 2) {
+            s_logger.error("Invalid content in host guid");
+            return null;
+        }
+
+        String[] hostTokens = tokens[0].split(":");
+        if (hostTokens == null || hostTokens.length != 2) {
+            s_logger.error("Invalid content in host guid");
+            return null;
+        }
+
+        ManagedObjectReference morHyperHost = new ManagedObjectReference();
+        morHyperHost.setType(hostTokens[0]);
+        morHyperHost.setValue(hostTokens[1]);
+
+        return morHyperHost;
+    }
 
     public static String getScsiController(Pair<String, String> controllerInfo, String recommendedController) {
         String rootDiskController = controllerInfo.first();