Skip to content

<fix>[storage]: repair 4.4.52-sm baseline failures #4670

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ZStack-Robot wants to merge 1 commit into
base: 4.4.52-sm
Choose a base branch
from
+43 −23
Open

<fix>[storage]: repair 4.4.52-sm baseline failures #4670

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions image/src/main/java/org/zstack/image/ImageManagerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -246,6 +246,9 @@ public void run(MessageReply r) {

private void copyVmSecurityLevelIfNeeded(String volumeUuid, String imageUuid) {
String vmInstanceUuid = Q.New(VolumeVO.class).eq(VolumeVO_.uuid, volumeUuid).select(VolumeVO_.vmInstanceUuid).findValue();
if (vmInstanceUuid == null) {
return;
}
VmInstanceVO vmInstanceVO = dbf.findByUuid(vmInstanceUuid, VmInstanceVO.class);
if (vmInstanceVO != null && Q.New(SecurityLevelResourceRefVO.class).eq(SecurityLevelResourceRefVO_.resourceUuid, vmInstanceVO.getUuid()).isExists()) {
String currentImageSecurityLevel = Q.New(SecurityLevelResourceRefVO.class)
Expand Down
59 changes: 38 additions & 21 deletions storage/src/main/java/org/zstack/storage/backup/BackupStoragePingTracker.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,10 +76,12 @@ public void handleReply(String resourceUuid, MessageReply reply) {
protected void untrackHook(String resUuid) {
statusMap.remove(resUuid);

BackupStorageReconnectTask reconnectTask = reconnectTaskMap.get(resUuid);
if (reconnectTask != null) {
reconnectTask.cancel();
reconnectTaskMap.remove(resUuid);
synchronized (reconnectTaskMap) {
BackupStorageReconnectTask reconnectTask = reconnectTaskMap.get(resUuid);
if (reconnectTask != null) {
reconnectTask.cancel();
reconnectTaskMap.remove(resUuid);
}
}
}

Expand Down Expand Up @@ -160,11 +162,20 @@ protected void run(Map tokens, Object data) {
} else if (BackupStorageStatus.Disconnected.toString().equals(d.getNewStatus()) &&
BackupStorageStatus.Connecting.toString().equals(d.getOldStatus())) {
backupStorageDisconnectCount.computeIfAbsent(d.getBackupStorageUuid(), key -> new AtomicInteger(0)).addAndGet(1);
if (isReconnectLimitReached(d.getBackupStorageUuid())) {
cancel(d.getBackupStorageUuid());
}
}
}
});
}

private boolean isReconnectLimitReached(String uuid) {
AtomicInteger disconnectCount = backupStorageDisconnectCount.get(uuid);
int threshold = BackupStorageGlobalConfig.AUTO_RECONNECT_ON_ERROR_MAX_ATTEMPT_NUM.value(Integer.class);
return threshold > 0 && disconnectCount != null && disconnectCount.get() >= threshold;
}

private void decideWhatToDoNext(String resUuid, ReconnectDecision decision) {
if (decision == ReconnectDecision.SubmitReconnectTask) {
submitReconnectTask(resUuid);
Expand All @@ -186,9 +197,7 @@ private ReconnectDecision makeReconnectDecision(String uuid, MessageReply reply)
return ReconnectDecision.StopReconnectTask;
}

AtomicInteger disconnectCount = backupStorageDisconnectCount.get(uuid);
int threshold = BackupStorageGlobalConfig.AUTO_RECONNECT_ON_ERROR_MAX_ATTEMPT_NUM.value(Integer.class);
if (threshold > 0 && disconnectCount != null && disconnectCount.get() >= threshold) {
if (isReconnectLimitReached(uuid)) {
logger.warn(String.format("[Backup storage Tracker]: stop pinging backup storage[uuid: %s] because it fail to reconnect too many times.", uuid));
return ReconnectDecision.StopReconnectTask;
}
Expand All @@ -212,27 +221,35 @@ private ReconnectDecision makeReconnectDecision(String uuid, MessageReply reply)
}

private void submitReconnectTask(String uuid) {
BackupStorageReconnectTask reconnectTask = reconnectTaskMap.get(uuid);
synchronized (reconnectTaskMap) {
if (isReconnectLimitReached(uuid)) {
return;
}

if (reconnectTask != null && !reconnectTask.taskIsCanceled()) {
return;
}
BackupStorageReconnectTask reconnectTask = reconnectTaskMap.get(uuid);

reconnectTask = new BackupStorageReconnectTask(uuid, new NoErrorCompletion() {
@Override
public void done() {
logger.debug(String.format("[Backup storage Tracker]: successfully create reconnect backup storage[uuid: %s] task.", uuid));
if (reconnectTask != null && !reconnectTask.taskIsCanceled()) {
return;
}
});

reconnectTaskMap.put(uuid, reconnectTask);
reconnectTask.start();
reconnectTask = new BackupStorageReconnectTask(uuid, new NoErrorCompletion() {
@Override
public void done() {
logger.debug(String.format("[Backup storage Tracker]: successfully create reconnect backup storage[uuid: %s] task.", uuid));
}
});

reconnectTaskMap.put(uuid, reconnectTask);
reconnectTask.start();
}
}

public void cancel(String resUuid) {
BackupStorageReconnectTask reconnectTask = reconnectTaskMap.get(resUuid);
if (reconnectTask != null && !reconnectTask.taskIsCanceled()) {
untrackHook(resUuid);
synchronized (reconnectTaskMap) {
BackupStorageReconnectTask reconnectTask = reconnectTaskMap.get(resUuid);
if (reconnectTask != null && !reconnectTask.taskIsCanceled()) {
untrackHook(resUuid);
}
}
}
}
4 changes: 2 additions & 2 deletions test/src/test/groovy/org/zstack/test/integration/kvm/vm/VmPmuConfigCase.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,7 @@ class VmPmuConfigCase extends SubCase {
assert cmd.pmu
}

KVMAgentCommands.StartVmCmd createVmAndCaptureStartCmd(String name, String imageName) {
KVMAgentCommands.StartVmCmd createVmAndCaptureStartCmd(String vmName, String imageName) {
ImageInventory image = env.inventoryByName(imageName) as ImageInventory
L3NetworkInventory l3 = env.inventoryByName("l3") as L3NetworkInventory
InstanceOfferingInventory instanceOffering = env.inventoryByName("instanceOffering") as InstanceOfferingInventory
Expand All @@ -200,7 +200,7 @@ class VmPmuConfigCase extends SubCase {
}

createVmInstance {
name = name
name = vmName
imageUuid = image.uuid
l3NetworkUuids = [l3.uuid]
instanceOfferingUuid = instanceOffering.uuid
Expand Down