Skip to content

fix(localstorage): check local capacity by disk requirements #4669

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
MatheMatrix wants to merge 1 commit into
base: 5.5.38
Choose a base branch
from
Open

fix(localstorage): check local capacity by disk requirements #4669

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 12 additions & 13 deletions compute/src/main/java/org/zstack/compute/vm/VmAllocateHostFlow.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,33 +52,23 @@ public class VmAllocateHostFlow implements Flow {
@Autowired
protected VmInstanceExtensionPointEmitter extEmitter;

private long getTotalDataDiskSize(VmInstanceSpec spec) {
long size = 0;
for (DiskOfferingInventory dinv : spec.getDataDiskOfferings()) {
size += dinv.getDiskSize();
}
return size;
}

protected AllocateHostMsg prepareMsg(VmInstanceSpec spec) {
DesignatedAllocateHostMsg msg = new DesignatedAllocateHostMsg();

List<DiskOfferingInventory> diskOfferings = new ArrayList<>();
ImageInventory image = spec.getImageSpec().getInventory();
long diskSize;
long rootDiskSize;
if (image == null || (image.getMediaType() != null && image.getMediaType().equals(ImageMediaType.ISO.toString()))) {
DiskOfferingVO dvo = dbf.findByUuid(spec.getRootDiskOffering().getUuid(), DiskOfferingVO.class);
diskSize = dvo.getDiskSize();
rootDiskSize = dvo.getDiskSize();
diskOfferings.add(DiskOfferingInventory.valueOf(dvo));
} else {
diskSize = image.getSize();
rootDiskSize = image.getSize();
}
diskSize += getTotalDataDiskSize(spec);
diskOfferings.addAll(spec.getDataDiskOfferings());
msg.setSoftAvoidHostUuids(spec.getSoftAvoidHostUuids());
msg.setAvoidHostUuids(spec.getAvoidHostUuids());
msg.setDiskOfferings(diskOfferings);
msg.setDiskSize(diskSize);
msg.setCpuCapacity(spec.getVmInventory().getCpuNum());
msg.setMemoryCapacity(spec.getVmInventory().getMemorySize());
msg.setClusterUuids(spec.getRequiredClusterUuids());
Expand Down Expand Up @@ -136,6 +126,15 @@ public String call(L3NetworkInventory arg) {
msg.getRequiredPrimaryStorageUuids().addAll(spec.getDiskAOs().stream()
.map(APICreateVmInstanceMsg.DiskAO::getPrimaryStorageUuid).filter(Objects::nonNull).collect(Collectors.toList()));
}
String rootPsUuid = spec.getCandidatePrimaryStorageUuidsForRootVolume().size() == 1 ?
spec.getCandidatePrimaryStorageUuidsForRootVolume().get(0) : null;
msg.addRequiredDiskCapacity(rootPsUuid, rootDiskSize);

String dataPsUuid = spec.getCandidatePrimaryStorageUuidsForDataVolume().size() == 1 ?
spec.getCandidatePrimaryStorageUuidsForDataVolume().get(0) : null;
for (DiskOfferingInventory dinv : spec.getDataDiskOfferings()) {
msg.addRequiredDiskCapacity(dataPsUuid, dinv.getDiskSize());
}
return msg;
}

Expand Down
16 changes: 13 additions & 3 deletions header/src/main/java/org/zstack/header/allocator/AllocateHostMsg.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
public class AllocateHostMsg extends NeedReplyMessage {
private long cpuCapacity;
private long memoryCapacity;
private long diskSize;
private String allocatorStrategy;
private List<String> avoidHostUuids;
private List<String> softAvoidHostUuids;
Expand All @@ -27,6 +26,7 @@ public class AllocateHostMsg extends NeedReplyMessage {
private Set<String> requiredPrimaryStorageUuids = new HashSet<>();
// for each set in the list, the primary storage inside is optional
private final List<Set<String>> optionalPrimaryStorageUuids = new ArrayList<>();
private final List<RequiredDiskCapacity> requiredDiskCapacities = new ArrayList<>();
private boolean fullAllocate = true;
private long oldMemoryCapacity = 0;
private AllocationScene allocationScene;
Expand Down Expand Up @@ -70,6 +70,14 @@ public void addRequiredPrimaryStorageUuid(String requiredPrimaryStorageUuid) {
this.requiredPrimaryStorageUuids.add(requiredPrimaryStorageUuid);
}

public List<RequiredDiskCapacity> getRequiredDiskCapacities() {
return requiredDiskCapacities;
}

public void addRequiredDiskCapacity(String primaryStorageUuid, long size) {
requiredDiskCapacities.add(new RequiredDiskCapacity(primaryStorageUuid, size));
}

public String getRequiredBackupStorageUuid() {
return requiredBackupStorageUuid;
}
Expand Down Expand Up @@ -143,11 +151,13 @@ public void setMemoryCapacity(long memoryCapacity) {
}

public long getDiskSize() {
return diskSize;
return requiredDiskCapacities.stream().mapToLong(RequiredDiskCapacity::getSize).sum();
}

// Compatibility entry for callers that cannot determine primary storage yet.
public void setDiskSize(long diskSize) {
this.diskSize = diskSize;
requiredDiskCapacities.clear();
requiredDiskCapacities.add(new RequiredDiskCapacity(null, diskSize));
}

public String getAllocatorStrategy() {
Expand Down
22 changes: 18 additions & 4 deletions header/src/main/java/org/zstack/header/allocator/HostAllocatorSpec.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ public class HostAllocatorSpec {
private long cpuCapacity;
private long memoryCapacity;
private List<String> l3NetworkUuids;
private long diskSize;
private String hypervisorType;
private String allocatorStrategy;
private VmInstanceInventory vmInstance;
Expand All @@ -29,6 +28,7 @@ public class HostAllocatorSpec {
private Set<String> requiredPrimaryStorageUuids = new HashSet<>();
// for each set in the list, the primary storage inside is optional
private final List<Set<String>> optionalPrimaryStorageUuids = new ArrayList<>();
private final List<RequiredDiskCapacity> requiredDiskCapacities = new ArrayList<>();
private Map<String, List<String>> backupStoragePrimaryStorageMetrics;
private boolean dryRun;
private List<String> systemTags;
Expand Down Expand Up @@ -89,6 +89,18 @@ public Set<String> getRequiredPrimaryStorageUuids() {
return requiredPrimaryStorageUuids;
}

public List<RequiredDiskCapacity> getRequiredDiskCapacities() {
return requiredDiskCapacities;
}

public void setRequiredDiskCapacities(List<RequiredDiskCapacity> requiredDiskCapacities) {
this.requiredDiskCapacities.clear();
if (requiredDiskCapacities != null) {
requiredDiskCapacities.forEach(it ->
this.requiredDiskCapacities.add(new RequiredDiskCapacity(it.getPrimaryStorageUuid(), it.getSize())));
}
}

public List<Set<String>> getOptionalPrimaryStorageUuids() {
return optionalPrimaryStorageUuids;
}
Expand Down Expand Up @@ -192,11 +204,13 @@ public void setL3NetworkUuids(List<String> l3NetworkUuids) {
}

public long getDiskSize() {
return diskSize;
return requiredDiskCapacities.stream().mapToLong(RequiredDiskCapacity::getSize).sum();
}

// Compatibility entry for callers that cannot determine primary storage yet.
public void setDiskSize(long diskSize) {
this.diskSize = diskSize;
requiredDiskCapacities.clear();
requiredDiskCapacities.add(new RequiredDiskCapacity(null, diskSize));
}

public String getHypervisorType() {
Expand Down Expand Up @@ -261,7 +275,6 @@ public static HostAllocatorSpec fromAllocationMsg(AllocateHostMsg msg) {
spec.setAvoidHostUuids(msg.getAvoidHostUuids());
spec.setSoftAvoidHostUuids(msg.getSoftAvoidHostUuids());
spec.setCpuCapacity(msg.getCpuCapacity());
spec.setDiskSize(msg.getDiskSize());
spec.setListAllHosts(msg.isListAllHosts());
spec.setDryRun(msg.isDryRun());
spec.setFullAllocate(msg.isFullAllocate());
Expand All @@ -280,6 +293,7 @@ public static HostAllocatorSpec fromAllocationMsg(AllocateHostMsg msg) {
spec.setAllowNoL3Networks(msg.isAllowNoL3Networks());
spec.setRequiredBackupStorageUuid(msg.getRequiredBackupStorageUuid());
spec.setRequiredPrimaryStorageUuids(msg.getRequiredPrimaryStorageUuids());
spec.setRequiredDiskCapacities(msg.getRequiredDiskCapacities());
msg.getOptionalPrimaryStorageUuids().forEach(spec::addOptionalPrimaryStorageUuids);
spec.setAllocationScene(msg.getAllocationScene());
spec.setArchitecture(msg.getArchitecture());
Expand Down
31 changes: 31 additions & 0 deletions header/src/main/java/org/zstack/header/allocator/RequiredDiskCapacity.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
package org.zstack.header.allocator;

public class RequiredDiskCapacity {
// Null means the destination primary storage has not been determined yet.
private String primaryStorageUuid;
private long size;

public RequiredDiskCapacity() {
}

public RequiredDiskCapacity(String primaryStorageUuid, long size) {
this.primaryStorageUuid = primaryStorageUuid;
this.size = size;
}

public String getPrimaryStorageUuid() {
return primaryStorageUuid;
}

public void setPrimaryStorageUuid(String primaryStorageUuid) {
this.primaryStorageUuid = primaryStorageUuid;
}

public long getSize() {
return size;
}

public void setSize(long size) {
this.size = size;
}
}
75 changes: 69 additions & 6 deletions ...lstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageAllocatorFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import org.zstack.header.allocator.HostAllocatorFilterExtensionPoint;
import org.zstack.header.allocator.HostAllocatorSpec;
import org.zstack.header.allocator.HostAllocatorStrategyExtensionPoint;
import org.zstack.header.allocator.RequiredDiskCapacity;
import org.zstack.header.errorcode.OperationFailureException;
import org.zstack.header.host.HostInventory;
import org.zstack.header.host.HostVO;
Expand All @@ -35,6 +36,7 @@
import org.zstack.utils.function.Function;
import org.zstack.utils.logging.CLogger;

import javax.persistence.Tuple;
import javax.persistence.TypedQuery;
import java.net.URI;
import java.net.URISyntaxException;
Expand Down Expand Up @@ -113,36 +115,58 @@ public List<HostVO> filterHostCandidates(List<HostVO> candidates, HostAllocatorS
if (huuids.isEmpty()) {
return candidates;
}

SimpleQuery<LocalStorageHostRefVO> q = dbf.createQuery(LocalStorageHostRefVO.class);
q.add(LocalStorageHostRefVO_.hostUuid, Op.IN, huuids);
if (!spec.getRequiredPrimaryStorageUuids().isEmpty()) {
q.add(LocalStorageHostRefVO_.primaryStorageUuid, Op.IN, spec.getRequiredPrimaryStorageUuids());
}
List<LocalStorageHostRefVO> refs = q.list();
boolean hasUndeterminedPrimaryStorageCapacity = spec.getRequiredDiskCapacities().stream()
.anyMatch(it -> it.getPrimaryStorageUuid() == null);
Map<String, Set<String>> hostAccessiblePrimaryStorageUuids = VmOperation.NewCreate.toString().equals(spec.getVmOperation()) && hasUndeterminedPrimaryStorageCapacity ?
getHostAccessiblePrimaryStorageUuids(huuids) : Collections.emptyMap();

final Set<String> toRemoveHuuids = new HashSet<>();
final Set<String> toAddHuuids = new HashSet<>();
final Set<String> requiredLocalCapacityFailedHuuids = new HashSet<>();
long requiredSizeForError = 0;
for (LocalStorageHostRefVO ref : refs) {
String huuid = ref.getHostUuid();
String psUuid = ref.getPrimaryStorageUuid();
long requiredSize;
if (VmOperation.MigrateVolume.toString().equals(spec.getVmOperation())) {
// Volume migration has one destination primary storage, so diskSize is the capacity required on the target local storage.
requiredSize = spec.getDiskSize();
} else {
requiredSize = getRequiredLocalStorageSize(spec, huuid, psUuid, hostAccessiblePrimaryStorageUuids);
}
if (requiredSize == 0) {
continue;
}

// check primary storage capacity and host physical capacity
boolean capacityChecked = PrimaryStorageCapacityChecker.New(psUuid,
ref.getAvailableCapacity(), ref.getTotalPhysicalCapacity(), ref.getAvailablePhysicalCapacity())
.checkRequiredSize(spec.getDiskSize());
.checkRequiredSize(requiredSize);

if (!capacityChecked) {
addHostPrimaryStorageBlacklist(huuid, psUuid, spec);
toRemoveHuuids.add(huuid);
if (spec.getRequiredPrimaryStorageUuids().contains(psUuid)) {
requiredLocalCapacityFailedHuuids.add(huuid);
} else {
toRemoveHuuids.add(huuid);
}
requiredSizeForError = Math.max(requiredSizeForError, requiredSize);
} else {
toAddHuuids.add(huuid);
}
}
// for more than one local storage, maybe one of it fit the requirement
// for more than one optional local storage, maybe one of it fit the requirement
toRemoveHuuids.removeAll(toAddHuuids);
toRemoveHuuids.addAll(requiredLocalCapacityFailedHuuids);
if (!toRemoveHuuids.isEmpty()) {
logger.debug(String.format("local storage filters out hosts%s, because they don't have required disk capacity[%s bytes]",
toRemoveHuuids, spec.getDiskSize()));
toRemoveHuuids, requiredSizeForError));

candidates = CollectionUtils.transformToList(candidates, new Function<HostVO, HostVO>() {
@Override
Expand All @@ -154,7 +178,7 @@ public HostVO call(HostVO arg) {
if (candidates.isEmpty()) {
throw new OperationFailureException(err(ORG_ZSTACK_STORAGE_PRIMARY_LOCAL_10020, HostAllocatorError.NO_AVAILABLE_HOST,
"the local primary storage has no hosts with enough disk capacity[%s bytes] required by the vm[uuid:%s]",
spec.getDiskSize(), spec.getVmInstance().getUuid()
requiredSizeForError, spec.getVmInstance().getUuid()
));
}
}
Expand All @@ -178,6 +202,45 @@ else if (VmOperation.Migrate.toString().equals(spec.getVmOperation())) {
return candidates;
}

private long getRequiredLocalStorageSize(HostAllocatorSpec spec, String hostUuid, String psUuid,
Map<String, Set<String>> hostAccessiblePrimaryStorageUuids) {
long requiredSize = spec.getRequiredDiskCapacities().stream()
.filter(it -> psUuid.equals(it.getPrimaryStorageUuid()))
.mapToLong(RequiredDiskCapacity::getSize)
.sum();

Set<String> psUuids = hostAccessiblePrimaryStorageUuids.get(hostUuid);
if (psUuids != null && psUuids.size() == 1 && psUuids.contains(psUuid)) {
// Undetermined disks can only be placed on this local storage when it is the host's only accessible primary storage.
requiredSize += spec.getRequiredDiskCapacities().stream()
.filter(it -> it.getPrimaryStorageUuid() == null)
.mapToLong(RequiredDiskCapacity::getSize)
.sum();
}

return requiredSize;
}

private Map<String, Set<String>> getHostAccessiblePrimaryStorageUuids(List<String> hostUuids) {
String sql = "select h.uuid, ps.uuid" +
" from HostVO h, PrimaryStorageClusterRefVO ref, PrimaryStorageVO ps" +
" where h.uuid in (:hostUuids)" +
" and h.clusterUuid = ref.clusterUuid" +
" and ref.primaryStorageUuid = ps.uuid" +
" and ps.state = :state" +
" and ps.status = :status";
TypedQuery<Tuple> q = dbf.getEntityManager().createQuery(sql, Tuple.class);
q.setParameter("hostUuids", hostUuids);
q.setParameter("state", PrimaryStorageState.Enabled);
q.setParameter("status", PrimaryStorageStatus.Connected);

Map<String, Set<String>> ret = new HashMap<>();
for (Tuple t : q.getResultList()) {
ret.computeIfAbsent(t.get(0, String.class), k -> new HashSet<>()).add(t.get(1, String.class));
}
return ret;
}

private void checkLocalStorageForVmStart(VmInstanceInventory vm, List<HostVO> candidates) {
final List<String> localPS = Q.New(PrimaryStorageVO.class)
.select(PrimaryStorageVO_.uuid)
Expand Down
Loading