server: Search zone-wide storage pool when allocation algothrim is firstfitleastconsumed (#4002)

Author: ustcweizhou

engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDao.java

@@ -56,5 +56,5 @@ List<SummedCapacity> findCapacityBy(Integer capacityType, Long zoneId,
 
     float findClusterConsumption(Long clusterId, short capacityType, long computeRequested);
 
-    List<Long> orderHostsByFreeCapacity(Long clusterId, short capacityType);
+    List<Long> orderHostsByFreeCapacity(Long zoneId, Long clusterId, short capacityType);
 }

engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDaoImpl.java

@@ -903,20 +903,28 @@ public Pair<List<Long>, Map<Long, Double>> orderClustersByAggregateCapacity(long
     }
 
     @Override
-    public List<Long> orderHostsByFreeCapacity(Long clusterId, short capacityTypeForOrdering){
+    public List<Long> orderHostsByFreeCapacity(Long zoneId, Long clusterId, short capacityTypeForOrdering){
          TransactionLegacy txn = TransactionLegacy.currentTxn();
          PreparedStatement pstmt = null;
          List<Long> result = new ArrayList<Long>();
          StringBuilder sql = new StringBuilder(ORDER_HOSTS_BY_FREE_CAPACITY_PART1);
-        if(clusterId != null) {
-            sql.append("AND cluster_id = ?");
-        }
-        sql.append(ORDER_HOSTS_BY_FREE_CAPACITY_PART2);
+         if (zoneId != null) {
+             sql.append(" AND data_center_id = ?");
+         }
+         if (clusterId != null) {
+             sql.append(" AND cluster_id = ?");
+         }
+         sql.append(ORDER_HOSTS_BY_FREE_CAPACITY_PART2);
          try {
              pstmt = txn.prepareAutoCloseStatement(sql.toString());
              pstmt.setShort(1, capacityTypeForOrdering);
-             if(clusterId != null) {
-                pstmt.setLong(2, clusterId);
+             int index = 2;
+             if (zoneId != null) {
+                 pstmt.setLong(index, zoneId);
+                 index ++;
+             }
+             if (clusterId != null) {
+                 pstmt.setLong(index, clusterId);
              }
 
              ResultSet rs = pstmt.executeQuery();

engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java

@@ -94,6 +94,7 @@ public List<StoragePool> allocateToPool(DiskProfile dskCh, VirtualMachineProfile
 
     protected List<StoragePool> reorderPoolsByCapacity(DeploymentPlan plan,
         List<StoragePool> pools) {
+        Long zoneId = plan.getDataCenterId();
         Long clusterId = plan.getClusterId();
         short capacityType;
         if(pools != null && pools.size() != 0){
@@ -102,7 +103,7 @@ protected List<StoragePool> reorderPoolsByCapacity(DeploymentPlan plan,
             return null;
         }
 
-        List<Long> poolIdsByCapacity = capacityDao.orderHostsByFreeCapacity(clusterId, capacityType);
+        List<Long> poolIdsByCapacity = capacityDao.orderHostsByFreeCapacity(zoneId, clusterId, capacityType);
         if (s_logger.isDebugEnabled()) {
             s_logger.debug("List of pools in descending order of free capacity: "+ poolIdsByCapacity);
         }

engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/ZoneWideStoragePoolAllocator.java

@@ -29,6 +29,8 @@
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
 
+import com.cloud.capacity.Capacity;
+import com.cloud.capacity.dao.CapacityDao;
 import com.cloud.deploy.DeploymentPlan;
 import com.cloud.deploy.DeploymentPlanner.ExcludeList;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
@@ -43,6 +45,8 @@ public class ZoneWideStoragePoolAllocator extends AbstractStoragePoolAllocator {
     private static final Logger LOGGER = Logger.getLogger(ZoneWideStoragePoolAllocator.class);
     @Inject
     private DataStoreManager dataStoreMgr;
+    @Inject
+    private CapacityDao capacityDao;
 
 
     @Override
@@ -110,6 +114,40 @@ private boolean canAddStoragePoolToAvoidSet(StoragePoolVO storagePoolVO) {
         return !ScopeType.ZONE.equals(storagePoolVO.getScope()) || !storagePoolVO.isManaged();
     }
 
+
+    @Override
+    protected List<StoragePool> reorderPoolsByCapacity(DeploymentPlan plan,
+        List<StoragePool> pools) {
+        Long zoneId = plan.getDataCenterId();
+        short capacityType;
+        if(pools != null && pools.size() != 0){
+            capacityType = pools.get(0).getPoolType().isShared() ? Capacity.CAPACITY_TYPE_STORAGE_ALLOCATED : Capacity.CAPACITY_TYPE_LOCAL_STORAGE;
+        } else{
+            return null;
+        }
+
+        List<Long> poolIdsByCapacity = capacityDao.orderHostsByFreeCapacity(zoneId, null, capacityType);
+        if (LOGGER.isDebugEnabled()) {
+            LOGGER.debug("List of zone-wide storage pools in descending order of free capacity: "+ poolIdsByCapacity);
+        }
+
+      //now filter the given list of Pools by this ordered list
+      Map<Long, StoragePool> poolMap = new HashMap<>();
+      for (StoragePool pool : pools) {
+          poolMap.put(pool.getId(), pool);
+      }
+      List<Long> matchingPoolIds = new ArrayList<>(poolMap.keySet());
+
+      poolIdsByCapacity.retainAll(matchingPoolIds);
+
+      List<StoragePool> reorderedPools = new ArrayList<>();
+      for(Long id: poolIdsByCapacity){
+          reorderedPools.add(poolMap.get(id));
+      }
+
+      return reorderedPools;
+    }
+
     @Override
     protected List<StoragePool> reorderPoolsByNumberOfVolumes(DeploymentPlan plan, List<StoragePool> pools, Account account) {
         if (account == null) {

server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java

@@ -345,14 +345,15 @@ protected List<Host> allocateTo(DeploymentPlan plan, ServiceOffering offering, V
 
     // Reorder hosts in the decreasing order of free capacity.
     private List<? extends Host> reorderHostsByCapacity(DeploymentPlan plan, List<? extends Host> hosts) {
+        Long zoneId = plan.getDataCenterId();
         Long clusterId = plan.getClusterId();
         //Get capacity by which we should reorder
         String capacityTypeToOrder = _configDao.getValue(Config.HostCapacityTypeToOrderClusters.key());
         short capacityType = CapacityVO.CAPACITY_TYPE_CPU;
         if("RAM".equalsIgnoreCase(capacityTypeToOrder)){
             capacityType = CapacityVO.CAPACITY_TYPE_MEMORY;
         }
-        List<Long> hostIdsByFreeCapacity = _capacityDao.orderHostsByFreeCapacity(clusterId, capacityType);
+        List<Long> hostIdsByFreeCapacity = _capacityDao.orderHostsByFreeCapacity(zoneId, clusterId, capacityType);
         if (s_logger.isDebugEnabled()) {
             s_logger.debug("List of hosts in descending order of free capacity in the cluster: "+ hostIdsByFreeCapacity);
         }