From 76be837156bf2a2fa28e8603c87811c904b8c3e9 Mon Sep 17 00:00:00 2001 From: J M Date: Mon, 3 Aug 2026 17:21:22 +0800 Subject: [PATCH] [storage]: repair 4.4.52-sm baseline failures Handle snapshots from unattached volumes before copying VM security level. Avoid Groovy closure self-assignment in the PMU Case. Stop backup storage reconnect timers as soon as the configured attempt limit is reached and serialize their lifecycle. Tests: CreateVmFromVolumeSnapshotCase Tests: VmPmuConfigCase Tests: BackupStorageAutoConnectCase Related: ZSTAC-87030 / premium!14908 Resolves: ZCF-0 Change-Id: I91ff6204062811ec6bce1fce6ec313ee72c83032 --- .../org/zstack/image/ImageManagerImpl.java | 3 + .../backup/BackupStoragePingTracker.java | 59 ++++++++++++------- .../integration/kvm/vm/VmPmuConfigCase.groovy | 4 +- 3 files changed, 43 insertions(+), 23 deletions(-) diff --git a/image/src/main/java/org/zstack/image/ImageManagerImpl.java b/image/src/main/java/org/zstack/image/ImageManagerImpl.java index cc7b42096b0..6c5d8283c78 100755 --- a/image/src/main/java/org/zstack/image/ImageManagerImpl.java +++ b/image/src/main/java/org/zstack/image/ImageManagerImpl.java @@ -246,6 +246,9 @@ public void run(MessageReply r) { private void copyVmSecurityLevelIfNeeded(String volumeUuid, String imageUuid) { String vmInstanceUuid = Q.New(VolumeVO.class).eq(VolumeVO_.uuid, volumeUuid).select(VolumeVO_.vmInstanceUuid).findValue(); + if (vmInstanceUuid == null) { + return; + } VmInstanceVO vmInstanceVO = dbf.findByUuid(vmInstanceUuid, VmInstanceVO.class); if (vmInstanceVO != null && Q.New(SecurityLevelResourceRefVO.class).eq(SecurityLevelResourceRefVO_.resourceUuid, vmInstanceVO.getUuid()).isExists()) { String currentImageSecurityLevel = Q.New(SecurityLevelResourceRefVO.class) diff --git a/storage/src/main/java/org/zstack/storage/backup/BackupStoragePingTracker.java b/storage/src/main/java/org/zstack/storage/backup/BackupStoragePingTracker.java index d338026429e..402f6e95ab3 100755 --- a/storage/src/main/java/org/zstack/storage/backup/BackupStoragePingTracker.java +++ b/storage/src/main/java/org/zstack/storage/backup/BackupStoragePingTracker.java @@ -76,10 +76,12 @@ public void handleReply(String resourceUuid, MessageReply reply) { protected void untrackHook(String resUuid) { statusMap.remove(resUuid); - BackupStorageReconnectTask reconnectTask = reconnectTaskMap.get(resUuid); - if (reconnectTask != null) { - reconnectTask.cancel(); - reconnectTaskMap.remove(resUuid); + synchronized (reconnectTaskMap) { + BackupStorageReconnectTask reconnectTask = reconnectTaskMap.get(resUuid); + if (reconnectTask != null) { + reconnectTask.cancel(); + reconnectTaskMap.remove(resUuid); + } } } @@ -160,11 +162,20 @@ protected void run(Map tokens, Object data) { } else if (BackupStorageStatus.Disconnected.toString().equals(d.getNewStatus()) && BackupStorageStatus.Connecting.toString().equals(d.getOldStatus())) { backupStorageDisconnectCount.computeIfAbsent(d.getBackupStorageUuid(), key -> new AtomicInteger(0)).addAndGet(1); + if (isReconnectLimitReached(d.getBackupStorageUuid())) { + cancel(d.getBackupStorageUuid()); + } } } }); } + private boolean isReconnectLimitReached(String uuid) { + AtomicInteger disconnectCount = backupStorageDisconnectCount.get(uuid); + int threshold = BackupStorageGlobalConfig.AUTO_RECONNECT_ON_ERROR_MAX_ATTEMPT_NUM.value(Integer.class); + return threshold > 0 && disconnectCount != null && disconnectCount.get() >= threshold; + } + private void decideWhatToDoNext(String resUuid, ReconnectDecision decision) { if (decision == ReconnectDecision.SubmitReconnectTask) { submitReconnectTask(resUuid); @@ -186,9 +197,7 @@ private ReconnectDecision makeReconnectDecision(String uuid, MessageReply reply) return ReconnectDecision.StopReconnectTask; } - AtomicInteger disconnectCount = backupStorageDisconnectCount.get(uuid); - int threshold = BackupStorageGlobalConfig.AUTO_RECONNECT_ON_ERROR_MAX_ATTEMPT_NUM.value(Integer.class); - if (threshold > 0 && disconnectCount != null && disconnectCount.get() >= threshold) { + if (isReconnectLimitReached(uuid)) { logger.warn(String.format("[Backup storage Tracker]: stop pinging backup storage[uuid: %s] because it fail to reconnect too many times.", uuid)); return ReconnectDecision.StopReconnectTask; } @@ -212,27 +221,35 @@ private ReconnectDecision makeReconnectDecision(String uuid, MessageReply reply) } private void submitReconnectTask(String uuid) { - BackupStorageReconnectTask reconnectTask = reconnectTaskMap.get(uuid); + synchronized (reconnectTaskMap) { + if (isReconnectLimitReached(uuid)) { + return; + } - if (reconnectTask != null && !reconnectTask.taskIsCanceled()) { - return; - } + BackupStorageReconnectTask reconnectTask = reconnectTaskMap.get(uuid); - reconnectTask = new BackupStorageReconnectTask(uuid, new NoErrorCompletion() { - @Override - public void done() { - logger.debug(String.format("[Backup storage Tracker]: successfully create reconnect backup storage[uuid: %s] task.", uuid)); + if (reconnectTask != null && !reconnectTask.taskIsCanceled()) { + return; } - }); - reconnectTaskMap.put(uuid, reconnectTask); - reconnectTask.start(); + reconnectTask = new BackupStorageReconnectTask(uuid, new NoErrorCompletion() { + @Override + public void done() { + logger.debug(String.format("[Backup storage Tracker]: successfully create reconnect backup storage[uuid: %s] task.", uuid)); + } + }); + + reconnectTaskMap.put(uuid, reconnectTask); + reconnectTask.start(); + } } public void cancel(String resUuid) { - BackupStorageReconnectTask reconnectTask = reconnectTaskMap.get(resUuid); - if (reconnectTask != null && !reconnectTask.taskIsCanceled()) { - untrackHook(resUuid); + synchronized (reconnectTaskMap) { + BackupStorageReconnectTask reconnectTask = reconnectTaskMap.get(resUuid); + if (reconnectTask != null && !reconnectTask.taskIsCanceled()) { + untrackHook(resUuid); + } } } } diff --git a/test/src/test/groovy/org/zstack/test/integration/kvm/vm/VmPmuConfigCase.groovy b/test/src/test/groovy/org/zstack/test/integration/kvm/vm/VmPmuConfigCase.groovy index 94e1be11ce5..1e348c951a5 100644 --- a/test/src/test/groovy/org/zstack/test/integration/kvm/vm/VmPmuConfigCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/kvm/vm/VmPmuConfigCase.groovy @@ -188,7 +188,7 @@ class VmPmuConfigCase extends SubCase { assert cmd.pmu } - KVMAgentCommands.StartVmCmd createVmAndCaptureStartCmd(String name, String imageName) { + KVMAgentCommands.StartVmCmd createVmAndCaptureStartCmd(String vmName, String imageName) { ImageInventory image = env.inventoryByName(imageName) as ImageInventory L3NetworkInventory l3 = env.inventoryByName("l3") as L3NetworkInventory InstanceOfferingInventory instanceOffering = env.inventoryByName("instanceOffering") as InstanceOfferingInventory @@ -200,7 +200,7 @@ class VmPmuConfigCase extends SubCase { } createVmInstance { - name = name + name = vmName imageUuid = image.uuid l3NetworkUuids = [l3.uuid] instanceOfferingUuid = instanceOffering.uuid