diff --git a/compute/src/main/java/org/zstack/compute/vm/VmAllocateHostFlow.java b/compute/src/main/java/org/zstack/compute/vm/VmAllocateHostFlow.java index 8993b278e9b..73d389d5807 100755 --- a/compute/src/main/java/org/zstack/compute/vm/VmAllocateHostFlow.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmAllocateHostFlow.java @@ -52,33 +52,23 @@ public class VmAllocateHostFlow implements Flow { @Autowired protected VmInstanceExtensionPointEmitter extEmitter; - private long getTotalDataDiskSize(VmInstanceSpec spec) { - long size = 0; - for (DiskOfferingInventory dinv : spec.getDataDiskOfferings()) { - size += dinv.getDiskSize(); - } - return size; - } - protected AllocateHostMsg prepareMsg(VmInstanceSpec spec) { DesignatedAllocateHostMsg msg = new DesignatedAllocateHostMsg(); List diskOfferings = new ArrayList<>(); ImageInventory image = spec.getImageSpec().getInventory(); - long diskSize; + long rootDiskSize; if (image == null || (image.getMediaType() != null && image.getMediaType().equals(ImageMediaType.ISO.toString()))) { DiskOfferingVO dvo = dbf.findByUuid(spec.getRootDiskOffering().getUuid(), DiskOfferingVO.class); - diskSize = dvo.getDiskSize(); + rootDiskSize = dvo.getDiskSize(); diskOfferings.add(DiskOfferingInventory.valueOf(dvo)); } else { - diskSize = image.getSize(); + rootDiskSize = image.getSize(); } - diskSize += getTotalDataDiskSize(spec); diskOfferings.addAll(spec.getDataDiskOfferings()); msg.setSoftAvoidHostUuids(spec.getSoftAvoidHostUuids()); msg.setAvoidHostUuids(spec.getAvoidHostUuids()); msg.setDiskOfferings(diskOfferings); - msg.setDiskSize(diskSize); msg.setCpuCapacity(spec.getVmInventory().getCpuNum()); msg.setMemoryCapacity(spec.getVmInventory().getMemorySize()); msg.setClusterUuids(spec.getRequiredClusterUuids()); @@ -136,6 +126,15 @@ public String call(L3NetworkInventory arg) { msg.getRequiredPrimaryStorageUuids().addAll(spec.getDiskAOs().stream() .map(APICreateVmInstanceMsg.DiskAO::getPrimaryStorageUuid).filter(Objects::nonNull).collect(Collectors.toList())); } + String rootPsUuid = spec.getCandidatePrimaryStorageUuidsForRootVolume().size() == 1 ? + spec.getCandidatePrimaryStorageUuidsForRootVolume().get(0) : null; + msg.addRequiredDiskCapacity(rootPsUuid, rootDiskSize); + + String dataPsUuid = spec.getCandidatePrimaryStorageUuidsForDataVolume().size() == 1 ? + spec.getCandidatePrimaryStorageUuidsForDataVolume().get(0) : null; + for (DiskOfferingInventory dinv : spec.getDataDiskOfferings()) { + msg.addRequiredDiskCapacity(dataPsUuid, dinv.getDiskSize()); + } return msg; } diff --git a/header/src/main/java/org/zstack/header/allocator/AllocateHostMsg.java b/header/src/main/java/org/zstack/header/allocator/AllocateHostMsg.java index 132637b8635..b399a3d736e 100755 --- a/header/src/main/java/org/zstack/header/allocator/AllocateHostMsg.java +++ b/header/src/main/java/org/zstack/header/allocator/AllocateHostMsg.java @@ -10,7 +10,6 @@ public class AllocateHostMsg extends NeedReplyMessage { private long cpuCapacity; private long memoryCapacity; - private long diskSize; private String allocatorStrategy; private List avoidHostUuids; private List softAvoidHostUuids; @@ -27,6 +26,7 @@ public class AllocateHostMsg extends NeedReplyMessage { private Set requiredPrimaryStorageUuids = new HashSet<>(); // for each set in the list, the primary storage inside is optional private final List> optionalPrimaryStorageUuids = new ArrayList<>(); + private final List requiredDiskCapacities = new ArrayList<>(); private boolean fullAllocate = true; private long oldMemoryCapacity = 0; private AllocationScene allocationScene; @@ -70,6 +70,14 @@ public void addRequiredPrimaryStorageUuid(String requiredPrimaryStorageUuid) { this.requiredPrimaryStorageUuids.add(requiredPrimaryStorageUuid); } + public List getRequiredDiskCapacities() { + return requiredDiskCapacities; + } + + public void addRequiredDiskCapacity(String primaryStorageUuid, long size) { + requiredDiskCapacities.add(new RequiredDiskCapacity(primaryStorageUuid, size)); + } + public String getRequiredBackupStorageUuid() { return requiredBackupStorageUuid; } @@ -143,11 +151,13 @@ public void setMemoryCapacity(long memoryCapacity) { } public long getDiskSize() { - return diskSize; + return requiredDiskCapacities.stream().mapToLong(RequiredDiskCapacity::getSize).sum(); } + // Compatibility entry for callers that cannot determine primary storage yet. public void setDiskSize(long diskSize) { - this.diskSize = diskSize; + requiredDiskCapacities.clear(); + requiredDiskCapacities.add(new RequiredDiskCapacity(null, diskSize)); } public String getAllocatorStrategy() { diff --git a/header/src/main/java/org/zstack/header/allocator/HostAllocatorSpec.java b/header/src/main/java/org/zstack/header/allocator/HostAllocatorSpec.java index 33d39896c82..162ae3b1de0 100755 --- a/header/src/main/java/org/zstack/header/allocator/HostAllocatorSpec.java +++ b/header/src/main/java/org/zstack/header/allocator/HostAllocatorSpec.java @@ -14,7 +14,6 @@ public class HostAllocatorSpec { private long cpuCapacity; private long memoryCapacity; private List l3NetworkUuids; - private long diskSize; private String hypervisorType; private String allocatorStrategy; private VmInstanceInventory vmInstance; @@ -29,6 +28,7 @@ public class HostAllocatorSpec { private Set requiredPrimaryStorageUuids = new HashSet<>(); // for each set in the list, the primary storage inside is optional private final List> optionalPrimaryStorageUuids = new ArrayList<>(); + private final List requiredDiskCapacities = new ArrayList<>(); private Map> backupStoragePrimaryStorageMetrics; private boolean dryRun; private List systemTags; @@ -89,6 +89,18 @@ public Set getRequiredPrimaryStorageUuids() { return requiredPrimaryStorageUuids; } + public List getRequiredDiskCapacities() { + return requiredDiskCapacities; + } + + public void setRequiredDiskCapacities(List requiredDiskCapacities) { + this.requiredDiskCapacities.clear(); + if (requiredDiskCapacities != null) { + requiredDiskCapacities.forEach(it -> + this.requiredDiskCapacities.add(new RequiredDiskCapacity(it.getPrimaryStorageUuid(), it.getSize()))); + } + } + public List> getOptionalPrimaryStorageUuids() { return optionalPrimaryStorageUuids; } @@ -192,11 +204,13 @@ public void setL3NetworkUuids(List l3NetworkUuids) { } public long getDiskSize() { - return diskSize; + return requiredDiskCapacities.stream().mapToLong(RequiredDiskCapacity::getSize).sum(); } + // Compatibility entry for callers that cannot determine primary storage yet. public void setDiskSize(long diskSize) { - this.diskSize = diskSize; + requiredDiskCapacities.clear(); + requiredDiskCapacities.add(new RequiredDiskCapacity(null, diskSize)); } public String getHypervisorType() { @@ -261,7 +275,6 @@ public static HostAllocatorSpec fromAllocationMsg(AllocateHostMsg msg) { spec.setAvoidHostUuids(msg.getAvoidHostUuids()); spec.setSoftAvoidHostUuids(msg.getSoftAvoidHostUuids()); spec.setCpuCapacity(msg.getCpuCapacity()); - spec.setDiskSize(msg.getDiskSize()); spec.setListAllHosts(msg.isListAllHosts()); spec.setDryRun(msg.isDryRun()); spec.setFullAllocate(msg.isFullAllocate()); @@ -280,6 +293,7 @@ public static HostAllocatorSpec fromAllocationMsg(AllocateHostMsg msg) { spec.setAllowNoL3Networks(msg.isAllowNoL3Networks()); spec.setRequiredBackupStorageUuid(msg.getRequiredBackupStorageUuid()); spec.setRequiredPrimaryStorageUuids(msg.getRequiredPrimaryStorageUuids()); + spec.setRequiredDiskCapacities(msg.getRequiredDiskCapacities()); msg.getOptionalPrimaryStorageUuids().forEach(spec::addOptionalPrimaryStorageUuids); spec.setAllocationScene(msg.getAllocationScene()); spec.setArchitecture(msg.getArchitecture()); diff --git a/header/src/main/java/org/zstack/header/allocator/RequiredDiskCapacity.java b/header/src/main/java/org/zstack/header/allocator/RequiredDiskCapacity.java new file mode 100644 index 00000000000..99000483f47 --- /dev/null +++ b/header/src/main/java/org/zstack/header/allocator/RequiredDiskCapacity.java @@ -0,0 +1,31 @@ +package org.zstack.header.allocator; + +public class RequiredDiskCapacity { + // Null means the destination primary storage has not been determined yet. + private String primaryStorageUuid; + private long size; + + public RequiredDiskCapacity() { + } + + public RequiredDiskCapacity(String primaryStorageUuid, long size) { + this.primaryStorageUuid = primaryStorageUuid; + this.size = size; + } + + public String getPrimaryStorageUuid() { + return primaryStorageUuid; + } + + public void setPrimaryStorageUuid(String primaryStorageUuid) { + this.primaryStorageUuid = primaryStorageUuid; + } + + public long getSize() { + return size; + } + + public void setSize(long size) { + this.size = size; + } +} diff --git a/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageAllocatorFactory.java b/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageAllocatorFactory.java index 0c90fd3d9bb..e10589e0df6 100755 --- a/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageAllocatorFactory.java +++ b/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageAllocatorFactory.java @@ -15,6 +15,7 @@ import org.zstack.header.allocator.HostAllocatorFilterExtensionPoint; import org.zstack.header.allocator.HostAllocatorSpec; import org.zstack.header.allocator.HostAllocatorStrategyExtensionPoint; +import org.zstack.header.allocator.RequiredDiskCapacity; import org.zstack.header.errorcode.OperationFailureException; import org.zstack.header.host.HostInventory; import org.zstack.header.host.HostVO; @@ -35,6 +36,7 @@ import org.zstack.utils.function.Function; import org.zstack.utils.logging.CLogger; +import javax.persistence.Tuple; import javax.persistence.TypedQuery; import java.net.URI; import java.net.URISyntaxException; @@ -113,36 +115,58 @@ public List filterHostCandidates(List candidates, HostAllocatorS if (huuids.isEmpty()) { return candidates; } - SimpleQuery q = dbf.createQuery(LocalStorageHostRefVO.class); q.add(LocalStorageHostRefVO_.hostUuid, Op.IN, huuids); if (!spec.getRequiredPrimaryStorageUuids().isEmpty()) { q.add(LocalStorageHostRefVO_.primaryStorageUuid, Op.IN, spec.getRequiredPrimaryStorageUuids()); } List refs = q.list(); + boolean hasUndeterminedPrimaryStorageCapacity = spec.getRequiredDiskCapacities().stream() + .anyMatch(it -> it.getPrimaryStorageUuid() == null); + Map> hostAccessiblePrimaryStorageUuids = VmOperation.NewCreate.toString().equals(spec.getVmOperation()) && hasUndeterminedPrimaryStorageCapacity ? + getHostAccessiblePrimaryStorageUuids(huuids) : Collections.emptyMap(); final Set toRemoveHuuids = new HashSet<>(); final Set toAddHuuids = new HashSet<>(); + final Set requiredLocalCapacityFailedHuuids = new HashSet<>(); + long requiredSizeForError = 0; for (LocalStorageHostRefVO ref : refs) { String huuid = ref.getHostUuid(); String psUuid = ref.getPrimaryStorageUuid(); + long requiredSize; + if (VmOperation.MigrateVolume.toString().equals(spec.getVmOperation())) { + // Volume migration has one destination primary storage, so diskSize is the capacity required on the target local storage. + requiredSize = spec.getDiskSize(); + } else { + requiredSize = getRequiredLocalStorageSize(spec, huuid, psUuid, hostAccessiblePrimaryStorageUuids); + } + if (requiredSize == 0) { + continue; + } + // check primary storage capacity and host physical capacity boolean capacityChecked = PrimaryStorageCapacityChecker.New(psUuid, ref.getAvailableCapacity(), ref.getTotalPhysicalCapacity(), ref.getAvailablePhysicalCapacity()) - .checkRequiredSize(spec.getDiskSize()); + .checkRequiredSize(requiredSize); if (!capacityChecked) { addHostPrimaryStorageBlacklist(huuid, psUuid, spec); - toRemoveHuuids.add(huuid); + if (spec.getRequiredPrimaryStorageUuids().contains(psUuid)) { + requiredLocalCapacityFailedHuuids.add(huuid); + } else { + toRemoveHuuids.add(huuid); + } + requiredSizeForError = Math.max(requiredSizeForError, requiredSize); } else { toAddHuuids.add(huuid); } } - // for more than one local storage, maybe one of it fit the requirement + // for more than one optional local storage, maybe one of it fit the requirement toRemoveHuuids.removeAll(toAddHuuids); + toRemoveHuuids.addAll(requiredLocalCapacityFailedHuuids); if (!toRemoveHuuids.isEmpty()) { logger.debug(String.format("local storage filters out hosts%s, because they don't have required disk capacity[%s bytes]", - toRemoveHuuids, spec.getDiskSize())); + toRemoveHuuids, requiredSizeForError)); candidates = CollectionUtils.transformToList(candidates, new Function() { @Override @@ -154,7 +178,7 @@ public HostVO call(HostVO arg) { if (candidates.isEmpty()) { throw new OperationFailureException(err(ORG_ZSTACK_STORAGE_PRIMARY_LOCAL_10020, HostAllocatorError.NO_AVAILABLE_HOST, "the local primary storage has no hosts with enough disk capacity[%s bytes] required by the vm[uuid:%s]", - spec.getDiskSize(), spec.getVmInstance().getUuid() + requiredSizeForError, spec.getVmInstance().getUuid() )); } } @@ -178,6 +202,45 @@ else if (VmOperation.Migrate.toString().equals(spec.getVmOperation())) { return candidates; } + private long getRequiredLocalStorageSize(HostAllocatorSpec spec, String hostUuid, String psUuid, + Map> hostAccessiblePrimaryStorageUuids) { + long requiredSize = spec.getRequiredDiskCapacities().stream() + .filter(it -> psUuid.equals(it.getPrimaryStorageUuid())) + .mapToLong(RequiredDiskCapacity::getSize) + .sum(); + + Set psUuids = hostAccessiblePrimaryStorageUuids.get(hostUuid); + if (psUuids != null && psUuids.size() == 1 && psUuids.contains(psUuid)) { + // Undetermined disks can only be placed on this local storage when it is the host's only accessible primary storage. + requiredSize += spec.getRequiredDiskCapacities().stream() + .filter(it -> it.getPrimaryStorageUuid() == null) + .mapToLong(RequiredDiskCapacity::getSize) + .sum(); + } + + return requiredSize; + } + + private Map> getHostAccessiblePrimaryStorageUuids(List hostUuids) { + String sql = "select h.uuid, ps.uuid" + + " from HostVO h, PrimaryStorageClusterRefVO ref, PrimaryStorageVO ps" + + " where h.uuid in (:hostUuids)" + + " and h.clusterUuid = ref.clusterUuid" + + " and ref.primaryStorageUuid = ps.uuid" + + " and ps.state = :state" + + " and ps.status = :status"; + TypedQuery q = dbf.getEntityManager().createQuery(sql, Tuple.class); + q.setParameter("hostUuids", hostUuids); + q.setParameter("state", PrimaryStorageState.Enabled); + q.setParameter("status", PrimaryStorageStatus.Connected); + + Map> ret = new HashMap<>(); + for (Tuple t : q.getResultList()) { + ret.computeIfAbsent(t.get(0, String.class), k -> new HashSet<>()).add(t.get(1, String.class)); + } + return ret; + } + private void checkLocalStorageForVmStart(VmInstanceInventory vm, List candidates) { final List localPS = Q.New(PrimaryStorageVO.class) .select(PrimaryStorageVO_.uuid) diff --git a/test/src/test/groovy/org/zstack/test/integration/storage/primary/local_nfs/allocator/host/CreateVmHostAllocateCase.groovy b/test/src/test/groovy/org/zstack/test/integration/storage/primary/local_nfs/allocator/host/CreateVmHostAllocateCase.groovy index 36709557942..eccfe83e26f 100644 --- a/test/src/test/groovy/org/zstack/test/integration/storage/primary/local_nfs/allocator/host/CreateVmHostAllocateCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/storage/primary/local_nfs/allocator/host/CreateVmHostAllocateCase.groovy @@ -32,6 +32,11 @@ class CreateVmHostAllocateCase extends SubCase { diskSize = SizeUnit.GIGABYTE.toByte(100) } + diskOffering { + name = "localDataDiskOffering" + diskSize = SizeUnit.GIGABYTE.toByte(55) + } + sftpBackupStorage { name = "sftp" url = "/sftp" @@ -116,6 +121,8 @@ class CreateVmHostAllocateCase extends SubCase { env.create { testGetCandidateZonesClustersHostsForCreatingVm() + testCreateVmAssignLocalAndNfs() + testCreateVmAssignNfs() } } @@ -142,6 +149,78 @@ class CreateVmHostAllocateCase extends SubCase { assert 2 == hosts.size() } + void testCreateVmAssignLocalAndNfs() { + InstanceOfferingInventory instanceOffering = env.inventoryByName("instanceOffering") as InstanceOfferingInventory + DiskOfferingInventory diskOffering = env.inventoryByName("diskOffering") as DiskOfferingInventory + DiskOfferingInventory localDataDiskOffering = env.inventoryByName("localDataDiskOffering") as DiskOfferingInventory + ImageInventory image = env.inventoryByName("image") as ImageInventory + L3NetworkInventory l3 = env.inventoryByName("l3") as L3NetworkInventory + HostInventory host = env.inventoryByName("kvm") + HostInventory host1 = env.inventoryByName("kvm1") + PrimaryStorageInventory nfs = env.inventoryByName("nfs") + PrimaryStorageInventory local = env.inventoryByName("local") + + CreateVmInstanceAction rootAndDataLocalAction = new CreateVmInstanceAction( + name : "rootAndDataLocalVm", + instanceOfferingUuid : instanceOffering.uuid, + imageUuid : image.uuid, + l3NetworkUuids : [l3.uuid], + hostUuid : host.uuid, + dataDiskOfferingUuids : [diskOffering.uuid], + primaryStorageUuidForRootVolume : local.uuid, + systemTags : [VmSystemTags.PRIMARY_STORAGE_UUID_FOR_DATA_VOLUME.instantiateTag([(VmSystemTags.PRIMARY_STORAGE_UUID_FOR_DATA_VOLUME_TOKEN): local.uuid])], + sessionId : currentEnvSpec.session.uuid + ) + assert null != rootAndDataLocalAction.call().error + + CreateVmInstanceAction rootLocalDataNfsAction = new CreateVmInstanceAction( + name : "rootLocalDataNfsVm", + instanceOfferingUuid : instanceOffering.uuid, + imageUuid : image.uuid, + l3NetworkUuids : [l3.uuid], + hostUuid : host.uuid, + dataDiskOfferingUuids : [diskOffering.uuid], + primaryStorageUuidForRootVolume : local.uuid, + systemTags : [VmSystemTags.PRIMARY_STORAGE_UUID_FOR_DATA_VOLUME.instantiateTag([(VmSystemTags.PRIMARY_STORAGE_UUID_FOR_DATA_VOLUME_TOKEN): nfs.uuid])], + sessionId : currentEnvSpec.session.uuid + ) + CreateVmInstanceAction.Result rootLocalDataNfsResult = rootLocalDataNfsAction.call() + assert null == rootLocalDataNfsResult.error + checkVmRootDiskPs(rootLocalDataNfsResult.value.inventory, local.uuid) + checkVmDataDiskPs(rootLocalDataNfsResult.value.inventory, nfs.uuid) + + CreateVmInstanceAction rootLocalDataUnspecifiedAction = new CreateVmInstanceAction( + name : "rootLocalDataUnspecifiedVm", + instanceOfferingUuid : instanceOffering.uuid, + imageUuid : image.uuid, + l3NetworkUuids : [l3.uuid], + hostUuid : host.uuid, + dataDiskOfferingUuids : [diskOffering.uuid], + primaryStorageUuidForRootVolume : local.uuid, + sessionId : currentEnvSpec.session.uuid + ) + CreateVmInstanceAction.Result rootLocalDataUnspecifiedResult = rootLocalDataUnspecifiedAction.call() + assert null == rootLocalDataUnspecifiedResult.error + checkVmRootDiskPs(rootLocalDataUnspecifiedResult.value.inventory, local.uuid) + checkVmDataDiskPs(rootLocalDataUnspecifiedResult.value.inventory, nfs.uuid) + + CreateVmInstanceAction rootNfsDataLocalAction = new CreateVmInstanceAction( + name : "rootNfsDataLocalVm", + instanceOfferingUuid : instanceOffering.uuid, + imageUuid : image.uuid, + l3NetworkUuids : [l3.uuid], + hostUuid : host1.uuid, + dataDiskOfferingUuids : [localDataDiskOffering.uuid], + primaryStorageUuidForRootVolume : nfs.uuid, + systemTags : [VmSystemTags.PRIMARY_STORAGE_UUID_FOR_DATA_VOLUME.instantiateTag([(VmSystemTags.PRIMARY_STORAGE_UUID_FOR_DATA_VOLUME_TOKEN): local.uuid])], + sessionId : currentEnvSpec.session.uuid + ) + CreateVmInstanceAction.Result rootNfsDataLocalResult = rootNfsDataLocalAction.call() + assert null == rootNfsDataLocalResult.error + checkVmRootDiskPs(rootNfsDataLocalResult.value.inventory, nfs.uuid) + checkVmDataDiskPs(rootNfsDataLocalResult.value.inventory, local.uuid) + } + void testCreateVmAssignNfs(){ InstanceOfferingInventory instanceOffering = env.inventoryByName("instanceOffering") as InstanceOfferingInventory DiskOfferingInventory diskOffering = env.inventoryByName("diskOffering") as DiskOfferingInventory @@ -184,4 +263,23 @@ class CreateVmHostAllocateCase extends SubCase { ) assert null != createVmInstanceAction.call().error } + + void checkVmRootDiskPs(VmInstanceInventory vm, String psUuid) { + assert vm.allVolumes.size() > 0 + for (VolumeInventory disk : vm.allVolumes) { + if (disk.uuid == vm.rootVolumeUuid) { + assert psUuid == disk.primaryStorageUuid + return + } + } + } + + void checkVmDataDiskPs(VmInstanceInventory vm, String psUuid) { + assert vm.allVolumes.size() > 1 + for (VolumeInventory disk : vm.allVolumes) { + if (disk.uuid != vm.rootVolumeUuid) { + assert psUuid == disk.primaryStorageUuid + } + } + } }