From 1a4fbf5e0d3b1c23458138bc81d4bda6c14b33d9 Mon Sep 17 00:00:00 2001 From: lianghy Date: Tue, 30 Dec 2025 18:15:19 +0800 Subject: [PATCH 01/11] [conf]: bump version to 4.8.37 DBImpact Resolves: ZSTAC-81065 Change-Id: I6563736d6769787876777a63796b697474617a66 --- VERSION | 2 +- conf/db/upgrade/V4.8.37__schema.sql | 0 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 conf/db/upgrade/V4.8.37__schema.sql diff --git a/VERSION b/VERSION index e4e2c43794d..d8e0378cfd7 100755 --- a/VERSION +++ b/VERSION @@ -1,3 +1,3 @@ MAJOR=4 MINOR=8 -UPDATE=36 +UPDATE=37 diff --git a/conf/db/upgrade/V4.8.37__schema.sql b/conf/db/upgrade/V4.8.37__schema.sql new file mode 100644 index 00000000000..e69de29bb2d From 8aa0a2f365174f00abe31421d66395fc93907341 Mon Sep 17 00:00:00 2001 From: "ye.zou" Date: Thu, 12 Feb 2026 13:29:00 +0800 Subject: [PATCH 02/11] [ha]: defer skip-trace list cleanup on MN departure to prevent split-brain When a management node departs, its VM skip-trace entries were immediately removed. If VMs were still being started by kvmagent, the next VM sync would falsely detect them as Stopped and trigger HA, causing split-brain. Fix: transfer departed MN skip-trace entries to an orphaned set with 10-minute TTL instead of immediate deletion. VMs in the orphaned set remain skip-traced until the TTL expires or they are explicitly continued, preventing false HA triggers during MN restart scenarios. Resolves: ZSTAC-80821 Change-Id: I3222e260b2d7b33dc43aba0431ce59a788566b34 Conflicts: plugin/kvm/src/main/java/org/zstack/kvm/KvmVmSyncPingTask.java --- .../org/zstack/kvm/KvmVmSyncPingTask.java | 66 ++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KvmVmSyncPingTask.java b/plugin/kvm/src/main/java/org/zstack/kvm/KvmVmSyncPingTask.java index 487c0664af8..b4374d9e8c1 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KvmVmSyncPingTask.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KvmVmSyncPingTask.java @@ -68,6 +68,12 @@ public class KvmVmSyncPingTask extends VmTracer implements KVMPingAgentNoFailure private List skipVmTracerReplies = new ArrayList<>(); private Map vmInShutdownMap = new ConcurrentHashMap<>(); + // Orphaned skip entries from departed MN nodes. Key=vmUuid, Value=timestamp when orphaned. + // These VMs remain in skip-trace state for ORPHAN_TTL_MS to avoid false HA triggers + // when a MN restarts and its in-flight VM operations haven't completed yet. See ZSTAC-80821. + private final ConcurrentHashMap orphanedSkipVms = new ConcurrentHashMap<>(); + private static final long ORPHAN_TTL_MS = 10 * 60 * 1000; // 10 minutes + { getReflections().getTypesAnnotatedWith(SkipVmTracer.class).forEach(clz -> { skipVmTracerMessages.add(clz.asSubclass(Message.class)); @@ -195,8 +201,13 @@ private void syncVm(final HostInventory host, final Completion completion) { // Get vms to skip before send command to host to confirm the vm will be skipped after sync command finished. // The problem is if one vm-sync skipped operation is started and finished during vm sync command's handling // vm state would still be sync to mn + // ZSTAC-80821: clean up expired orphaned entries each sync cycle + cleanupExpiredOrphanedSkipVms(); + Set vmsToSkipSetHostSide = new HashSet<>(); vmsToSkip.values().forEach(vmsToSkipSetHostSide::addAll); + // ZSTAC-80821: also skip VMs from departed MN nodes that are still within TTL + vmsToSkipSetHostSide.addAll(orphanedSkipVms.keySet()); // if the vm is not running on host when sync command executing but started as soon as possible // before response handling of vm sync, mgmtSideStates will including the running vm but not result in @@ -227,6 +238,8 @@ public void run(MessageReply reply) { // Get vms to skip after sync result returned. vmsToSkip.values().forEach(vmsToSkipSetHostSide::addAll); + // ZSTAC-80821: include orphaned entries from departed MN nodes + vmsToSkipSetHostSide.addAll(orphanedSkipVms.keySet()); Collection vmUuidsInDeleteVmGC = DeleteVmGC.queryVmInGC(host.getUuid(), ret.getStates().keySet()); @@ -445,7 +458,19 @@ public void nodeJoin(ManagementNodeInventory inv) { @Override public void nodeLeft(ManagementNodeInventory inv) { vmApis.remove(inv.getUuid()); - vmsToSkip.remove(inv.getUuid()); + + // ZSTAC-80821: Instead of immediately removing skip list entries, move them + // to the orphaned set with a TTL. This prevents false HA triggers for VMs that + // are still being started by kvmagent but whose controlling MN has restarted. + Set skippedVms = vmsToSkip.remove(inv.getUuid()); + if (skippedVms != null && !skippedVms.isEmpty()) { + long now = System.currentTimeMillis(); + for (String vmUuid : skippedVms) { + orphanedSkipVms.put(vmUuid, now); + logger.info(String.format("moved VM[uuid:%s] from departed MN[uuid:%s] skip list to orphaned set" + + " (will expire in %d minutes)", vmUuid, inv.getUuid(), ORPHAN_TTL_MS / 60000)); + } + } } @Override @@ -457,4 +482,43 @@ public void iJoin(ManagementNodeInventory inv) { vmApis.putIfAbsent(inv.getUuid(), new ConcurrentHashMap<>()); vmsToSkip.putIfAbsent(inv.getUuid(), ConcurrentHashMap.newKeySet()); } + + public boolean isVmDoNotNeedToTrace(String vmUuid) { + if (vmsToSkip.values().stream().anyMatch(vmsToSkipSet -> vmsToSkipSet.contains(vmUuid))) { + return true; + } + + // ZSTAC-80821: Also check orphaned skip entries from departed MN nodes + Long orphanedAt = orphanedSkipVms.get(vmUuid); + if (orphanedAt != null) { + if (System.currentTimeMillis() - orphanedAt < ORPHAN_TTL_MS) { + logger.debug(String.format("VM[uuid:%s] is in orphaned skip set, skipping trace", vmUuid)); + return true; + } else { + // Expired, clean up + orphanedSkipVms.remove(vmUuid); + logger.info(String.format("orphaned skip entry for VM[uuid:%s] expired after %d minutes, resuming trace", + vmUuid, ORPHAN_TTL_MS / 60000)); + } + } + + return false; + } + + // Periodically clean up expired orphaned entries. Called from VM sync cycle. + private void cleanupExpiredOrphanedSkipVms() { + if (orphanedSkipVms.isEmpty()) { + return; + } + + long now = System.currentTimeMillis(); + Iterator> it = orphanedSkipVms.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry entry = it.next(); + if (now - entry.getValue() >= ORPHAN_TTL_MS) { + it.remove(); + logger.info(String.format("cleaned up expired orphaned skip entry for VM[uuid:%s]", entry.getKey())); + } + } + } } From 8becdfa2632e3f6b55a7c13a4712243391d8653c Mon Sep 17 00:00:00 2001 From: "ye.zou" Date: Thu, 12 Feb 2026 16:47:28 +0800 Subject: [PATCH 03/11] [kvm]: use CAS remove to fix TOCTOU race in orphaned skip VM cleanup Resolves: ZSTAC-80821 Change-Id: I59284c4e69f5d2ee357b1836b7c243200e30949a --- .../src/main/java/org/zstack/kvm/KvmVmSyncPingTask.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KvmVmSyncPingTask.java b/plugin/kvm/src/main/java/org/zstack/kvm/KvmVmSyncPingTask.java index b4374d9e8c1..4b0152f941b 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KvmVmSyncPingTask.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KvmVmSyncPingTask.java @@ -496,7 +496,7 @@ public boolean isVmDoNotNeedToTrace(String vmUuid) { return true; } else { // Expired, clean up - orphanedSkipVms.remove(vmUuid); + orphanedSkipVms.remove(vmUuid, orphanedAt); logger.info(String.format("orphaned skip entry for VM[uuid:%s] expired after %d minutes, resuming trace", vmUuid, ORPHAN_TTL_MS / 60000)); } @@ -512,11 +512,9 @@ private void cleanupExpiredOrphanedSkipVms() { } long now = System.currentTimeMillis(); - Iterator> it = orphanedSkipVms.entrySet().iterator(); - while (it.hasNext()) { - Map.Entry entry = it.next(); + for (Map.Entry entry : orphanedSkipVms.entrySet()) { if (now - entry.getValue() >= ORPHAN_TTL_MS) { - it.remove(); + orphanedSkipVms.remove(entry.getKey(), entry.getValue()); logger.info(String.format("cleaned up expired orphaned skip entry for VM[uuid:%s]", entry.getKey())); } } From 2939f820eb34178707d47b4e163b057e9b1dd87f Mon Sep 17 00:00:00 2001 From: "ye.zou" Date: Mon, 16 Feb 2026 21:55:50 +0800 Subject: [PATCH 04/11] [kvm]: configurable orphan skip timeout Resolves: ZSTAC-80821 Change-Id: Ia9a9597feceb96b3e6e22259e2d0be7bde8ae499 --- .../main/java/org/zstack/kvm/KVMGlobalConfig.java | 4 ++++ .../java/org/zstack/kvm/KvmVmSyncPingTask.java | 15 +++++++++------ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMGlobalConfig.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMGlobalConfig.java index 71e9cc8d20d..0e4bc622abd 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMGlobalConfig.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMGlobalConfig.java @@ -120,6 +120,10 @@ public class KVMGlobalConfig { @GlobalConfigDef(defaultValue = "false", type = Boolean.class, description = "enable install host shutdown hook") public static GlobalConfig INSTALL_HOST_SHUTDOWN_HOOK = new GlobalConfig(CATEGORY, "install.host.shutdown.hook"); + @GlobalConfigValidation(numberGreaterThan = 0) + @GlobalConfigDef(defaultValue = "600", type = Long.class, description = "timeout in seconds for orphaned VM skip entries from departed management nodes") + public static GlobalConfig ORPHANED_VM_SKIP_TIMEOUT = new GlobalConfig(CATEGORY, "vm.orphanedSkipTimeout"); + @GlobalConfigValidation(validValues = {"true", "false"}) @GlobalConfigDef(defaultValue = "false", type = Boolean.class, description = "enable memory auto balloon") @BindResourceConfig({VmInstanceVO.class}) diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KvmVmSyncPingTask.java b/plugin/kvm/src/main/java/org/zstack/kvm/KvmVmSyncPingTask.java index 4b0152f941b..d727cbd4cd4 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KvmVmSyncPingTask.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KvmVmSyncPingTask.java @@ -69,10 +69,13 @@ public class KvmVmSyncPingTask extends VmTracer implements KVMPingAgentNoFailure private Map vmInShutdownMap = new ConcurrentHashMap<>(); // Orphaned skip entries from departed MN nodes. Key=vmUuid, Value=timestamp when orphaned. - // These VMs remain in skip-trace state for ORPHAN_TTL_MS to avoid false HA triggers + // These VMs remain in skip-trace state to avoid false HA triggers // when a MN restarts and its in-flight VM operations haven't completed yet. See ZSTAC-80821. private final ConcurrentHashMap orphanedSkipVms = new ConcurrentHashMap<>(); - private static final long ORPHAN_TTL_MS = 10 * 60 * 1000; // 10 minutes + + private long getOrphanTtlMs() { + return KVMGlobalConfig.ORPHANED_VM_SKIP_TIMEOUT.value(Long.class) * 1000; + } { getReflections().getTypesAnnotatedWith(SkipVmTracer.class).forEach(clz -> { @@ -468,7 +471,7 @@ public void nodeLeft(ManagementNodeInventory inv) { for (String vmUuid : skippedVms) { orphanedSkipVms.put(vmUuid, now); logger.info(String.format("moved VM[uuid:%s] from departed MN[uuid:%s] skip list to orphaned set" + - " (will expire in %d minutes)", vmUuid, inv.getUuid(), ORPHAN_TTL_MS / 60000)); + " (will expire in %d minutes)", vmUuid, inv.getUuid(), getOrphanTtlMs() / 60000)); } } } @@ -491,14 +494,14 @@ public boolean isVmDoNotNeedToTrace(String vmUuid) { // ZSTAC-80821: Also check orphaned skip entries from departed MN nodes Long orphanedAt = orphanedSkipVms.get(vmUuid); if (orphanedAt != null) { - if (System.currentTimeMillis() - orphanedAt < ORPHAN_TTL_MS) { + if (System.currentTimeMillis() - orphanedAt < getOrphanTtlMs()) { logger.debug(String.format("VM[uuid:%s] is in orphaned skip set, skipping trace", vmUuid)); return true; } else { // Expired, clean up orphanedSkipVms.remove(vmUuid, orphanedAt); logger.info(String.format("orphaned skip entry for VM[uuid:%s] expired after %d minutes, resuming trace", - vmUuid, ORPHAN_TTL_MS / 60000)); + vmUuid, getOrphanTtlMs() / 60000)); } } @@ -513,7 +516,7 @@ private void cleanupExpiredOrphanedSkipVms() { long now = System.currentTimeMillis(); for (Map.Entry entry : orphanedSkipVms.entrySet()) { - if (now - entry.getValue() >= ORPHAN_TTL_MS) { + if (now - entry.getValue() >= getOrphanTtlMs()) { orphanedSkipVms.remove(entry.getKey(), entry.getValue()); logger.info(String.format("cleaned up expired orphaned skip entry for VM[uuid:%s]", entry.getKey())); } From 1c57b6a37d04f013b4d943f5e6239599b1468f18 Mon Sep 17 00:00:00 2001 From: gitlab Date: Thu, 14 May 2026 06:53:00 +0000 Subject: [PATCH 05/11] bump version to 4.8.38 DBImpact Change-Id: I7473677a7265777a616f6d706776706e67626173 --- VERSION | 2 +- conf/db/upgrade/V4.8.38__schema.sql | 0 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 conf/db/upgrade/V4.8.38__schema.sql diff --git a/VERSION b/VERSION index d8e0378cfd7..56798053668 100755 --- a/VERSION +++ b/VERSION @@ -1,3 +1,3 @@ MAJOR=4 MINOR=8 -UPDATE=37 +UPDATE=38 diff --git a/conf/db/upgrade/V4.8.38__schema.sql b/conf/db/upgrade/V4.8.38__schema.sql new file mode 100644 index 00000000000..e69de29bb2d From 977f3feb3c369e4e9824ca74385520a7bfff5915 Mon Sep 17 00:00:00 2001 From: J M Date: Wed, 21 Aug 2024 18:15:33 +0800 Subject: [PATCH 06/11] [ansible]: fix host pswd shell Resolves: ZSTAC-69124 Change-Id: I616e64727376796e696d6f6779776e7378727076 (cherry picked from commit 85c9b9a17729e47c0be97c1f97dae7d2a4de10b5) --- .../core/ansible/CallBackNetworkChecker.java | 3 ++- .../core/ansible/SshChronyConfigChecker.java | 4 +++- .../core/ansible/SshFileExistChecker.java | 8 +------ .../core/ansible/SshFileMd5Checker.java | 17 +++++++++++---- .../core/ansible/SshFilesMd5Checker.java | 11 ++++++++-- .../core/ansible/SshFolderMd5Checker.java | 3 ++- .../zstack/core/ansible/SshYamlChecker.java | 4 +++- .../core/ansible/SshYumRepoChecker.java | 7 ++++--- .../org/zstack/kvm/KvmHostConfigChecker.java | 5 +++-- .../main/java/org/zstack/utils/ssh/Ssh.java | 13 +++++++++++- .../org/zstack/utils/ssh/SshCmdHelper.java | 21 +++++++++++++++++++ 11 files changed, 73 insertions(+), 23 deletions(-) create mode 100644 utils/src/main/java/org/zstack/utils/ssh/SshCmdHelper.java diff --git a/core/src/main/java/org/zstack/core/ansible/CallBackNetworkChecker.java b/core/src/main/java/org/zstack/core/ansible/CallBackNetworkChecker.java index e178265f4a8..c872703e154 100644 --- a/core/src/main/java/org/zstack/core/ansible/CallBackNetworkChecker.java +++ b/core/src/main/java/org/zstack/core/ansible/CallBackNetworkChecker.java @@ -7,6 +7,7 @@ import org.zstack.utils.Utils; import org.zstack.utils.logging.CLogger; import org.zstack.utils.ssh.Ssh; +import org.zstack.utils.ssh.SshCmdHelper; import org.zstack.utils.ssh.SshException; import org.zstack.utils.ssh.SshResult; @@ -47,7 +48,7 @@ public void deleteDestFile() { * if failed, use nmap to try again. */ private ErrorCode useNcatAndNmapToTestConnection(Ssh ssh) { - String srcScript = script.format(password, callBackPort, callbackIp); + String srcScript = script.format(SshCmdHelper.shellQuote(password), callBackPort, callbackIp); SshResult ret = ssh.setExecTimeout(60).shell(srcScript).setTimeout(60).runAndClose(); ret.raiseExceptionIfFailed(); diff --git a/core/src/main/java/org/zstack/core/ansible/SshChronyConfigChecker.java b/core/src/main/java/org/zstack/core/ansible/SshChronyConfigChecker.java index 37fad948abb..e9afbb1ee09 100644 --- a/core/src/main/java/org/zstack/core/ansible/SshChronyConfigChecker.java +++ b/core/src/main/java/org/zstack/core/ansible/SshChronyConfigChecker.java @@ -29,11 +29,13 @@ public boolean needDeploy() { .setPassword(password).setPort(sshPort) .setHostname(targetIp); try { - ssh.command("awk '/^\\s*server/{print $2}' /etc/chrony.conf"); + ssh.sudoCommand("awk '/^\\s*server/{print $2}' /etc/chrony.conf"); SshResult ret = ssh.run(); int returnCode = ret.getReturnCode(); ssh.reset(); if (returnCode != 0) { + logger.warn(String.format("exec ssh command failed, return code: %d, stdout: %s, stderr: %s", + ret.getReturnCode(), ret.getStdout(), ret.getStderr())); return true; } diff --git a/core/src/main/java/org/zstack/core/ansible/SshFileExistChecker.java b/core/src/main/java/org/zstack/core/ansible/SshFileExistChecker.java index 16d1b6eb181..0d3dd7d1e53 100755 --- a/core/src/main/java/org/zstack/core/ansible/SshFileExistChecker.java +++ b/core/src/main/java/org/zstack/core/ansible/SshFileExistChecker.java @@ -1,16 +1,10 @@ package org.zstack.core.ansible; -import org.zstack.utils.ShellResult; -import org.zstack.utils.ShellUtils; import org.zstack.utils.Utils; import org.zstack.utils.logging.CLogger; -import org.zstack.utils.path.PathUtil; import org.zstack.utils.ssh.Ssh; import org.zstack.utils.ssh.SshResult; -import java.util.ArrayList; -import java.util.List; - /** */ public class SshFileExistChecker implements AnsibleChecker { @@ -31,7 +25,7 @@ public boolean needDeploy() { .setHostname(targetIp) .setTimeout(5); try { - ssh.command(String.format("echo %s | sudo -S stat %s 2>/dev/null", password, filePath)); + ssh.sudoCommand(String.format("stat %s 2>/dev/null", filePath)); SshResult ret = ssh.run(); if (ret.getReturnCode() != 0) { logger.debug(String.format("file not exist, file: %s", filePath)); diff --git a/core/src/main/java/org/zstack/core/ansible/SshFileMd5Checker.java b/core/src/main/java/org/zstack/core/ansible/SshFileMd5Checker.java index 2beb2fb5058..812e02553ab 100755 --- a/core/src/main/java/org/zstack/core/ansible/SshFileMd5Checker.java +++ b/core/src/main/java/org/zstack/core/ansible/SshFileMd5Checker.java @@ -1,5 +1,6 @@ package org.zstack.core.ansible; +import org.zstack.core.CoreGlobalProperty; import org.zstack.utils.ShellResult; import org.zstack.utils.ShellUtils; import org.zstack.utils.Utils; @@ -33,7 +34,8 @@ private SrcDestPair(String srcPath, String destPath) { String destPath; } - public static final String ZSTACKLIB_SRC_PATH = PathUtil.findFileOnClassPath(String.format("ansible/zstacklib/%s", AnsibleGlobalProperty.ZSTACKLIB_PACKAGE_NAME), true).getAbsolutePath(); + public static final String ZSTACKLIB_SRC_PATH = CoreGlobalProperty.UNIT_TEST_ON ? "/tmp" : + PathUtil.findFileOnClassPath(String.format("ansible/zstacklib/%s", AnsibleGlobalProperty.ZSTACKLIB_PACKAGE_NAME), true).getAbsolutePath(); @Override public boolean needDeploy() { @@ -47,9 +49,11 @@ public boolean needDeploy() { String sourceFilePath = b.srcPath; String destFilePath = b.destPath; - ssh.command(String.format("echo %s | sudo -S md5sum %s 2>/dev/null", password, destFilePath)); + ssh.sudoCommand(String.format("md5sum %s 2>/dev/null", destFilePath)); SshResult ret = ssh.run(); if (ret.getReturnCode() != 0) { + logger.warn(String.format("exec ssh command failed, return code: %d, stdout: %s, stderr: %s", + ret.getReturnCode(), ret.getStdout(), ret.getStderr())); return true; } ssh.reset(); @@ -59,7 +63,7 @@ public boolean needDeploy() { sret.raiseExceptionIfFail(); String srcMd5 = sret.getStdout().split(" ")[0]; if (!destMd5.equals(srcMd5)) { - logger.debug(String.format("file MD5 changed, src[%s, md5:%s] dest[%s, md5, %s]", sourceFilePath, + logger.debug(String.format("file MD5 changed, src[%s, md5:%s] dest[%s, md5: %s]", sourceFilePath, srcMd5, destFilePath, destMd5)); return true; } @@ -75,10 +79,15 @@ public boolean needDeploy() { public void deleteDestFile() { for (SrcDestPair b : srcDestPairs) { String destFilePath = b.destPath; + if (!destFilePath.contains("zstack")) { + logger.debug(String.format("skip delete dest file[%s] which is not zstack file", destFilePath)); + continue; + } + Ssh ssh = new Ssh(); ssh.setUsername(username).setPrivateKey(privateKey) .setPassword(password).setPort(sshPort) - .setHostname(targetIp).command(String.format("rm -f %s", destFilePath)).runAndClose(); + .setHostname(targetIp).sudoCommand(String.format("rm -f %s", destFilePath)).runAndClose(); logger.debug(String.format("delete dest file[%s]", destFilePath)); } } diff --git a/core/src/main/java/org/zstack/core/ansible/SshFilesMd5Checker.java b/core/src/main/java/org/zstack/core/ansible/SshFilesMd5Checker.java index 238a0411e6a..0cd2f2bd034 100644 --- a/core/src/main/java/org/zstack/core/ansible/SshFilesMd5Checker.java +++ b/core/src/main/java/org/zstack/core/ansible/SshFilesMd5Checker.java @@ -28,9 +28,11 @@ public boolean needDeploy() { .setHostname(ip) .setTimeout(5); try { - ssh.command(String.format("echo %s | sudo -S md5sum %s 2>/dev/null", password, filePath)); + ssh.sudoCommand(String.format("md5sum %s 2>/dev/null", filePath)); SshResult ret = ssh.run(); if (ret.getReturnCode() != 0) { + logger.warn(String.format("exec ssh command failed, return code: %d, stdout: %s, stderr: %s", + ret.getReturnCode(), ret.getStdout(), ret.getStderr())); return true; } ssh.reset(); @@ -48,10 +50,15 @@ public boolean needDeploy() { @Override public void deleteDestFile() { + if (!filePath.contains("zstack")) { + logger.debug(String.format("skip delete dest file[%s] which is not zstack file", filePath)); + return; + } + Ssh ssh = new Ssh(); ssh.setUsername(username).setPrivateKey(privateKey) .setPassword(password).setPort(sshPort) - .setHostname(ip).command(String.format("rm -f %s", filePath)).runAndClose(); + .setHostname(ip).sudoCommand(String.format("rm -f %s", filePath)).runAndClose(); logger.debug(String.format("delete dest file[%s]", filePath)); } diff --git a/core/src/main/java/org/zstack/core/ansible/SshFolderMd5Checker.java b/core/src/main/java/org/zstack/core/ansible/SshFolderMd5Checker.java index ed6b0e4a1d2..e9d7e318b17 100755 --- a/core/src/main/java/org/zstack/core/ansible/SshFolderMd5Checker.java +++ b/core/src/main/java/org/zstack/core/ansible/SshFolderMd5Checker.java @@ -11,6 +11,7 @@ import org.zstack.utils.StringDSL.StringWrapper; import org.zstack.utils.Utils; import org.zstack.utils.logging.CLogger; +import org.zstack.utils.ssh.SshCmdHelper; import org.zstack.utils.ssh.SshResult; import org.zstack.utils.ssh.SshShell; @@ -107,7 +108,7 @@ public boolean needDeploy() { srcRes.getStdout(), srcRes.getStderr())); } - String dstScript = script.format(dstFolder, password); + String dstScript = script.format(dstFolder, SshCmdHelper.shellQuote(password)); SshShell ssh = new SshShell(); ssh.setHostname(hostname); ssh.setUsername(username); diff --git a/core/src/main/java/org/zstack/core/ansible/SshYamlChecker.java b/core/src/main/java/org/zstack/core/ansible/SshYamlChecker.java index 80f48782f66..32e6ca5ae06 100644 --- a/core/src/main/java/org/zstack/core/ansible/SshYamlChecker.java +++ b/core/src/main/java/org/zstack/core/ansible/SshYamlChecker.java @@ -36,9 +36,11 @@ public boolean needDeploy() { .setHostname(targetIp); try { - ssh.command(String.format("grep -o '%s' %s | uniq | wc -l", getGrepArgs(), yamlFilePath)); + ssh.sudoCommand(String.format("grep -o '%s' %s | uniq | wc -l", getGrepArgs(), yamlFilePath)); SshResult ret = ssh.run(); if (ret.getReturnCode() != 0) { + logger.warn(String.format("exec ssh command failed, return code: %d, stdout: %s, stderr: %s", + ret.getReturnCode(), ret.getStdout(), ret.getStderr())); return true; } diff --git a/core/src/main/java/org/zstack/core/ansible/SshYumRepoChecker.java b/core/src/main/java/org/zstack/core/ansible/SshYumRepoChecker.java index 8732f2067e6..8438fce4eac 100644 --- a/core/src/main/java/org/zstack/core/ansible/SshYumRepoChecker.java +++ b/core/src/main/java/org/zstack/core/ansible/SshYumRepoChecker.java @@ -36,12 +36,13 @@ public boolean needDeploy() { .setPassword(password).setPort(sshPort) .setHostname(targetIp); try { - ssh.command(String.format( - "echo %s | sudo -S sed -i '/baseurl/s/\\([0-9]\\{1,3\\}\\.\\)\\{3\\}[0-9]\\{1,3\\}:\\([0-9]\\+\\)/%s/g' /etc/yum.repos.d/{zstack,qemu-kvm-ev}-mn.repo", - password, restf.getHostName() + ":" + restf.getPort() + ssh.sudoCommand(String.format("sed -i '/baseurl/s/\\([0-9]\\{1,3\\}\\.\\)\\{3\\}[0-9]\\{1,3\\}:\\([0-9]\\+\\)/%s/g' /etc/yum.repos.d/{zstack,qemu-kvm-ev}-mn.repo", + restf.getHostName() + ":" + restf.getPort() )); SshResult ret = ssh.setTimeout(60).runAndClose(); if (ret.getReturnCode() != 0) { + logger.warn(String.format("exec ssh command failed, return code: %d, stdout: %s, stderr: %s", + ret.getReturnCode(), ret.getStdout(), ret.getStderr())); return true; } diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KvmHostConfigChecker.java b/plugin/kvm/src/main/java/org/zstack/kvm/KvmHostConfigChecker.java index 3f20bd13c11..6184933f5e2 100644 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KvmHostConfigChecker.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KvmHostConfigChecker.java @@ -3,7 +3,6 @@ import org.springframework.beans.factory.annotation.Autowire; import org.springframework.beans.factory.annotation.Configurable; import org.zstack.core.ansible.AnsibleChecker; -import org.zstack.core.ansible.CallBackNetworkChecker; import org.zstack.utils.Utils; import org.zstack.utils.logging.CLogger; import org.zstack.utils.ssh.Ssh; @@ -31,9 +30,11 @@ public boolean needDeploy() { .setPassword(password).setPort(sshPort) .setHostname(targetIp); try { - ssh.command("cat /sys/kernel/mm/ksm/run"); + ssh.sudoCommand("cat /sys/kernel/mm/ksm/run"); SshResult ret = ssh.setTimeout(60).runAndClose(); if (ret.getReturnCode() != 0) { + logger.warn(String.format("exec ssh command failed, return code: %d, stdout: %s, stderr: %s", + ret.getReturnCode(), ret.getStdout(), ret.getStderr())); return true; } diff --git a/utils/src/main/java/org/zstack/utils/ssh/Ssh.java b/utils/src/main/java/org/zstack/utils/ssh/Ssh.java index 3cf897075ee..69de299e258 100755 --- a/utils/src/main/java/org/zstack/utils/ssh/Ssh.java +++ b/utils/src/main/java/org/zstack/utils/ssh/Ssh.java @@ -212,6 +212,10 @@ public Ssh setSuppressException(boolean suppressException) { return this; } + /** + make sure user has permissions or use sudoCommand + */ + @Deprecated public Ssh command(String...cmds) { for (String cmd : cmds) { commands.add(createCommand(cmd)); @@ -219,6 +223,13 @@ public Ssh command(String...cmds) { return this; } + public Ssh sudoCommand(String...cmds) { + for (String cmd : cmds) { + commands.add(createCommand(SshCmdHelper.wrapSudoCmd(cmd, username, password))); + } + return this; + } + private SshRunner createCommand(final String cmdWithoutPrefix) { final String cmd = language + cmdWithoutPrefix; return new SshRunner() { @@ -280,7 +291,7 @@ public String getCommand() { @Override public String getCommandWithoutPassword() { - return cmd.replaceAll("echo .*?\\s*\\|\\s*sudo -S", "echo ****** | sudo -S"); + return SshCmdHelper.removeSensitiveInfoFromCmd(cmd); } }; } diff --git a/utils/src/main/java/org/zstack/utils/ssh/SshCmdHelper.java b/utils/src/main/java/org/zstack/utils/ssh/SshCmdHelper.java new file mode 100644 index 00000000000..27c9c031445 --- /dev/null +++ b/utils/src/main/java/org/zstack/utils/ssh/SshCmdHelper.java @@ -0,0 +1,21 @@ +package org.zstack.utils.ssh; + +public class SshCmdHelper { + public static String wrapSudoCmd(String cmd, String username, String password) { + if ("root".equals(username)) { + return cmd; + } else if (password == null) { + return String.format("sudo %s", cmd); + } else { + return String.format("echo %s | sudo -S %s", shellQuote(password), cmd); + } + } + + public static String removeSensitiveInfoFromCmd(String cmd) { + return cmd.replaceAll("echo .*?\\s*\\|\\s*sudo -S", "echo ****** | sudo -S"); + } + + public static String shellQuote(String s) { + return "'" + s.replace("'", "'\\''") + "'"; + } +} From bf4a83d0cc91d1d683157bdf82a4c278b32ccff3 Mon Sep 17 00:00:00 2001 From: J M Date: Wed, 4 Sep 2024 19:22:06 +0800 Subject: [PATCH 07/11] [ansible]: fix sudo password echo stdout and stderr will both write into ssh output Resolves: ZSTAC-69124 Change-Id: I6b68767166687072756e64776d6d6377746a6a6a (cherry picked from commit 5bf5b0a513e1e9233745075b1d53053df6c10201) --- .../org/zstack/core/ansible/SshFileExistChecker.java | 2 +- .../org/zstack/core/ansible/SshFileMd5Checker.java | 2 +- .../org/zstack/core/ansible/SshFilesMd5Checker.java | 2 +- plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java | 10 ++++++---- .../main/java/org/zstack/utils/ssh/SshCmdHelper.java | 2 +- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/org/zstack/core/ansible/SshFileExistChecker.java b/core/src/main/java/org/zstack/core/ansible/SshFileExistChecker.java index 0d3dd7d1e53..ccd78f74592 100755 --- a/core/src/main/java/org/zstack/core/ansible/SshFileExistChecker.java +++ b/core/src/main/java/org/zstack/core/ansible/SshFileExistChecker.java @@ -25,7 +25,7 @@ public boolean needDeploy() { .setHostname(targetIp) .setTimeout(5); try { - ssh.sudoCommand(String.format("stat %s 2>/dev/null", filePath)); + ssh.sudoCommand(String.format("stat %s", filePath)); SshResult ret = ssh.run(); if (ret.getReturnCode() != 0) { logger.debug(String.format("file not exist, file: %s", filePath)); diff --git a/core/src/main/java/org/zstack/core/ansible/SshFileMd5Checker.java b/core/src/main/java/org/zstack/core/ansible/SshFileMd5Checker.java index 812e02553ab..63fd823f833 100755 --- a/core/src/main/java/org/zstack/core/ansible/SshFileMd5Checker.java +++ b/core/src/main/java/org/zstack/core/ansible/SshFileMd5Checker.java @@ -49,7 +49,7 @@ public boolean needDeploy() { String sourceFilePath = b.srcPath; String destFilePath = b.destPath; - ssh.sudoCommand(String.format("md5sum %s 2>/dev/null", destFilePath)); + ssh.sudoCommand(String.format("md5sum %s", destFilePath)); SshResult ret = ssh.run(); if (ret.getReturnCode() != 0) { logger.warn(String.format("exec ssh command failed, return code: %d, stdout: %s, stderr: %s", diff --git a/core/src/main/java/org/zstack/core/ansible/SshFilesMd5Checker.java b/core/src/main/java/org/zstack/core/ansible/SshFilesMd5Checker.java index 0cd2f2bd034..2d69ab6f7e2 100644 --- a/core/src/main/java/org/zstack/core/ansible/SshFilesMd5Checker.java +++ b/core/src/main/java/org/zstack/core/ansible/SshFilesMd5Checker.java @@ -28,7 +28,7 @@ public boolean needDeploy() { .setHostname(ip) .setTimeout(5); try { - ssh.sudoCommand(String.format("md5sum %s 2>/dev/null", filePath)); + ssh.sudoCommand(String.format("md5sum %s", filePath)); SshResult ret = ssh.run(); if (ret.getReturnCode() != 0) { logger.warn(String.format("exec ssh command failed, return code: %d, stdout: %s, stderr: %s", diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java index 664480a26b9..7750c06dce7 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java @@ -5707,14 +5707,14 @@ public boolean skip(Map data) { public void run(FlowTrigger trigger, Map data) { StringBuilder builder = new StringBuilder(); if (!KVMGlobalProperty.MN_NETWORKS.isEmpty()) { - builder.append(String.format("sudo bash %s -m %s -p %s -s %s -c %s", + builder.append(String.format("bash %s -m %s -p %s -s %s -c %s", "/var/lib/zstack/kvm/kvmagent-iptables", KVMConstant.IPTABLES_COMMENTS, KVMGlobalConfig.KVMAGENT_ALLOW_PORTS_LIST.value(String.class), KVMGlobalProperty.AGENT_PORT, String.join(",", KVMGlobalProperty.MN_NETWORKS))); } else { - builder.append(String.format("sudo bash %s -m %s -p %s -s %s", + builder.append(String.format("bash %s -m %s -p %s -s %s", "/var/lib/zstack/kvm/kvmagent-iptables", KVMConstant.IPTABLES_COMMENTS, KVMGlobalConfig.KVMAGENT_ALLOW_PORTS_LIST.value(String.class), @@ -5722,11 +5722,13 @@ public void run(FlowTrigger trigger, Map data) { } try { - new Ssh().shell(builder.toString()) + new Ssh() .setUsername(getSelf().getUsername()) .setPassword(getSelf().getPassword()) .setHostname(getSelf().getManagementIp()) - .setPort(getSelf().getPort()).runErrorByExceptionAndClose(); + .setPort(getSelf().getPort()) + .sudoCommand(builder.toString()) + .runErrorByExceptionAndClose(); } catch (SshException ex) { throw new OperationFailureException(operr(ex.toString())); } diff --git a/utils/src/main/java/org/zstack/utils/ssh/SshCmdHelper.java b/utils/src/main/java/org/zstack/utils/ssh/SshCmdHelper.java index 27c9c031445..6c2fd3d5df1 100644 --- a/utils/src/main/java/org/zstack/utils/ssh/SshCmdHelper.java +++ b/utils/src/main/java/org/zstack/utils/ssh/SshCmdHelper.java @@ -7,7 +7,7 @@ public static String wrapSudoCmd(String cmd, String username, String password) { } else if (password == null) { return String.format("sudo %s", cmd); } else { - return String.format("echo %s | sudo -S %s", shellQuote(password), cmd); + return String.format("echo %s | sudo -S %s 2>/dev/null", shellQuote(password), cmd); } } From 175ba2b4e4ba00c056cd8ee45dfec9a6e70e856a Mon Sep 17 00:00:00 2001 From: J M Date: Thu, 29 May 2025 14:45:31 +0800 Subject: [PATCH 08/11] [storage]: trash top after storage live migration top is unused after storage migration even if its snapshot referenced by other volumes. only skip trash volume when its snapshots are inner snapshot. Resolves: ZSTAC-71514 Change-Id: I636b6d726d6c757478737869726f6a777a736663 (cherry picked from commit 866b137a8a89349d2b66cc9bf0fe8b0031b3729a) --- .../storage/snapshot/VolumeSnapshotTree.java | 18 +++++++ .../VolumeSnapshotReferenceInventory.java | 3 ++ .../reference/VolumeSnapshotReferenceVO.java | 3 ++ .../VolumeSnapshotReferenceUtils.java | 50 +++++++++++++------ 4 files changed, 60 insertions(+), 14 deletions(-) diff --git a/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotTree.java b/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotTree.java index fbb8a670d83..57b4fab4099 100755 --- a/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotTree.java +++ b/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotTree.java @@ -348,6 +348,7 @@ private VolumeSnapshotInventory getInventory(Set filterUuids) { private SnapshotLeaf root; private String volumeUuid; + private String uuid; public static VolumeSnapshotTree fromInventories(List invs) { VolumeSnapshotTree tree = new VolumeSnapshotTree(); @@ -377,6 +378,7 @@ public static VolumeSnapshotTree fromInventories(List i } tree.volumeUuid = inv.getVolumeUuid(); + tree.uuid = inv.getTreeUuid(); } DebugUtils.Assert(tree.root != null, "why tree root is null???"); @@ -403,6 +405,14 @@ public void setVolumeUuid(String volumeUuid) { this.volumeUuid = volumeUuid; } + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + private SnapshotLeaf findSnapshot(final List leafs, final Function func) { for (SnapshotLeaf leaf : leafs) { SnapshotLeaf ret = findSnapshot(leaf.children, func); @@ -424,4 +434,12 @@ public SnapshotLeaf findSnapshot(Function func } return findSnapshot(root.children, func); } + + public SnapshotLeaf findSnapshot(String snapshotUuid) { + if (snapshotUuid == null) { + return null; + } + + return findSnapshot(arg -> arg.getUuid().equals(snapshotUuid)); + } } diff --git a/header/src/main/java/org/zstack/header/storage/snapshot/reference/VolumeSnapshotReferenceInventory.java b/header/src/main/java/org/zstack/header/storage/snapshot/reference/VolumeSnapshotReferenceInventory.java index a3db08d459c..e8b12800b12 100644 --- a/header/src/main/java/org/zstack/header/storage/snapshot/reference/VolumeSnapshotReferenceInventory.java +++ b/header/src/main/java/org/zstack/header/storage/snapshot/reference/VolumeSnapshotReferenceInventory.java @@ -36,6 +36,9 @@ public class VolumeSnapshotReferenceInventory { private String directSnapshotInstallUrl; + /** + * the UUID of resource referencing @volumeUuid. + */ private String referenceUuid; private String referenceType; diff --git a/header/src/main/java/org/zstack/header/storage/snapshot/reference/VolumeSnapshotReferenceVO.java b/header/src/main/java/org/zstack/header/storage/snapshot/reference/VolumeSnapshotReferenceVO.java index 25fe22ee9e0..c646c440847 100644 --- a/header/src/main/java/org/zstack/header/storage/snapshot/reference/VolumeSnapshotReferenceVO.java +++ b/header/src/main/java/org/zstack/header/storage/snapshot/reference/VolumeSnapshotReferenceVO.java @@ -52,6 +52,9 @@ public class VolumeSnapshotReferenceVO { private Long parentId; + /** + * the UUID of resource referencing @volumeUuid. + */ @Column private String referenceUuid; diff --git a/storage/src/main/java/org/zstack/storage/snapshot/reference/VolumeSnapshotReferenceUtils.java b/storage/src/main/java/org/zstack/storage/snapshot/reference/VolumeSnapshotReferenceUtils.java index 921bef48516..5dca83ede7e 100644 --- a/storage/src/main/java/org/zstack/storage/snapshot/reference/VolumeSnapshotReferenceUtils.java +++ b/storage/src/main/java/org/zstack/storage/snapshot/reference/VolumeSnapshotReferenceUtils.java @@ -82,33 +82,55 @@ public static String getVolumeInstallUrlBackingOtherVolume(String volumeUuid) { return null; } - public static List getVolumeInstallUrlsReferenceByOtherVolumes(String volumeUuid) { + // use getVolumeAllSnapshotsReferencedOtherVolumes or isVolumeDirectlyReferenceByOthers + @Deprecated + public static List getVolumeSnapshotsReferencedByOtherVolumes(String volumeUuid) { return getVolumeReferenceRef(volumeUuid).stream() .map(VolumeSnapshotReferenceVO::getVolumeSnapshotInstallUrl).distinct() .collect(Collectors.toList()); } // get volume snapshotUuids referenced by other volumes directly or indirectly + // FIXME split different primary storage snapshot reference public static Set getVolumeAllSnapshotsReferencedByOtherVolumes(String volumeUuid) { - List refVolumeSnapshotUuids = getVolumeReferenceRef(volumeUuid).stream() - .map(VolumeSnapshotReferenceVO::getVolumeSnapshotUuid).distinct() - .collect(Collectors.toList()); - if (refVolumeSnapshotUuids.isEmpty()) { + Set refSnapshotUuids = getVolumeReferenceRef(volumeUuid).stream() + .map(VolumeSnapshotReferenceVO::getVolumeSnapshotUuid) + .collect(Collectors.toSet()); + if (refSnapshotUuids.isEmpty()) { return Collections.emptySet(); } List allSnapshots = Q.New(VolumeSnapshotVO.class).eq(VolumeSnapshotVO_.volumeUuid, volumeUuid).list(); + // snapshotUuid in VolumeSnapshotReferenceVO may not exist in VolumeSnapshotVO, + // maybe because the snapshot has been deleted db only after volume migration, + // so we need to filter them out + refSnapshotUuids.retainAll(allSnapshots.stream().map(VolumeSnapshotVO::getUuid).collect(Collectors.toSet())); + if (refSnapshotUuids.isEmpty()) { + return Collections.emptySet(); + } + + Set allRefSnapshotUuids = new HashSet<>(refSnapshotUuids); + Map trees = allSnapshots.stream() + .collect(Collectors.groupingBy(VolumeSnapshotVO::getTreeUuid, Collectors.toList())) + .values().stream() + .collect(Collectors.toMap( + snaps -> snaps.get(0).getTreeUuid(), + VolumeSnapshotTree::fromVOs + )); - Map> treeSnapshotsMap = allSnapshots.stream().collect(Collectors.groupingBy(VolumeSnapshotVO::getTreeUuid)); - List refVolumeSnapshotUuidsInTree = new ArrayList<>(); - for (VolumeSnapshotVO refSnapshot : allSnapshots.stream().filter(sp -> refVolumeSnapshotUuids.contains(sp.getUuid())).collect(Collectors.toList())) { - refVolumeSnapshotUuidsInTree.add(refSnapshot.getUuid()); - VolumeSnapshotTree tree = VolumeSnapshotTree.fromVOs(treeSnapshotsMap.get(refSnapshot.getTreeUuid())); - VolumeSnapshotTree.SnapshotLeaf snapshotLeaf = tree.findSnapshot(arg -> arg.getUuid().equals(refSnapshot.getUuid())); - refVolumeSnapshotUuidsInTree.addAll(snapshotLeaf.getAncestors().stream() - .map(VolumeSnapshotInventory::getUuid).collect(Collectors.toList())); + List refSnapshots = allSnapshots.stream() + .filter(sp -> refSnapshotUuids.contains(sp.getUuid())) + .collect(Collectors.toList()); + for (VolumeSnapshotVO refSnapshot : refSnapshots) { + Set ancestorUuids = trees.get(refSnapshot.getTreeUuid()) + .findSnapshot(refSnapshot.getUuid()) + .getAncestors().stream() + .map(VolumeSnapshotInventory::getUuid) + .collect(Collectors.toSet()); + allRefSnapshotUuids.addAll(ancestorUuids); } - return new HashSet<>(refVolumeSnapshotUuidsInTree); + + return allRefSnapshotUuids; } public static List getReferenceVolume(String volumeUuid) { From fbe41685d38bf6b9fcad4e84d9b90a2b8e24b55b Mon Sep 17 00:00:00 2001 From: "shan.wu" Date: Fri, 11 Apr 2025 13:17:33 +0800 Subject: [PATCH 09/11] [imagecache]: fix unable to clear image cache If fast cloning is performed on the data volume, it will result in image cache filtering error, further causing the image cache to be unable to be cleared Resolves/Related: ZSTAC-71233 Change-Id: I6d716679646d74726e69726b7968636667746b6f (cherry picked from commit ff9bf211259d26a2d8898d089f2abc1b339b6f50) --- .../VolumeSnapshotReferenceUtils.java | 3 +- ...CleanImageCacheOnPrimaryStorageCase.groovy | 35 ++++++++++++++++--- 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/storage/src/main/java/org/zstack/storage/snapshot/reference/VolumeSnapshotReferenceUtils.java b/storage/src/main/java/org/zstack/storage/snapshot/reference/VolumeSnapshotReferenceUtils.java index 921bef48516..8d6b442d7cd 100644 --- a/storage/src/main/java/org/zstack/storage/snapshot/reference/VolumeSnapshotReferenceUtils.java +++ b/storage/src/main/java/org/zstack/storage/snapshot/reference/VolumeSnapshotReferenceUtils.java @@ -339,7 +339,8 @@ public static List filterStaleImageCache(List ids) { return SQL.New("select c.id from ImageCacheVO c" + " where c.id in (:ids)" + - " and c.imageUuid not in (select tree.rootImageUuid from VolumeSnapshotReferenceTreeVO tree)", Long.class) + " and c.imageUuid not in (select tree.rootImageUuid from VolumeSnapshotReferenceTreeVO tree" + + " where tree.rootImageUuid is not null)", Long.class) .param("ids", ids) .list(); } diff --git a/test/src/test/groovy/org/zstack/test/integration/storage/primary/nfs/imagecleaner/imagecache/CleanImageCacheOnPrimaryStorageCase.groovy b/test/src/test/groovy/org/zstack/test/integration/storage/primary/nfs/imagecleaner/imagecache/CleanImageCacheOnPrimaryStorageCase.groovy index bd4746aee79..113d78caa8a 100644 --- a/test/src/test/groovy/org/zstack/test/integration/storage/primary/nfs/imagecleaner/imagecache/CleanImageCacheOnPrimaryStorageCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/storage/primary/nfs/imagecleaner/imagecache/CleanImageCacheOnPrimaryStorageCase.groovy @@ -3,28 +3,28 @@ package org.zstack.test.integration.storage.primary.nfs.imagecleaner.imagecache import org.springframework.http.HttpEntity import org.zstack.compute.vm.VmGlobalConfig import org.zstack.core.db.DatabaseFacade +import org.zstack.core.db.Q import org.zstack.core.db.SimpleQuery import org.zstack.header.image.ImageDeletionPolicyManager import org.zstack.header.network.service.NetworkServiceType import org.zstack.header.storage.primary.ImageCacheVO import org.zstack.header.storage.primary.ImageCacheVO_ +import org.zstack.header.storage.snapshot.reference.VolumeSnapshotReferenceTreeVO +import org.zstack.header.storage.snapshot.reference.VolumeSnapshotReferenceTreeVO_ import org.zstack.header.vm.VmInstanceDeletionPolicyManager import org.zstack.image.ImageGlobalConfig import org.zstack.network.securitygroup.SecurityGroupConstant import org.zstack.network.service.virtualrouter.VirtualRouterConstant -import org.zstack.sdk.ImageInventory -import org.zstack.sdk.PrimaryStorageInventory -import org.zstack.sdk.VmInstanceInventory +import org.zstack.sdk.* import org.zstack.storage.primary.nfs.NfsPrimaryStorageKVMBackend import org.zstack.storage.primary.nfs.NfsPrimaryStorageKVMBackendCommands +import org.zstack.storage.volume.VolumeSystemTags import org.zstack.test.integration.storage.StorageTest import org.zstack.testlib.EnvSpec import org.zstack.testlib.SubCase import org.zstack.utils.data.SizeUnit import org.zstack.utils.gson.JSONObjectUtil import org.zstack.utils.path.PathUtil -import java.util.concurrent.TimeUnit - /** * 1. two NFS storage running two VMs with the same image * 2. delete the image and two VMs @@ -207,7 +207,31 @@ class CleanImageCacheOnPrimaryStorageCase extends SubCase{ } } + void fastCloneVmBeforeDeletingImageCache() { + def volume = createDataVolume { + name = "test fast clone" + diskOfferingUuid = env.inventoryByName("diskOffering").uuid + primaryStorageUuid = env.inventoryByName("nfs").uuid + } as VolumeInventory + + def sp = createVolumeSnapshot { + name = "sp" + volumeUuid = volume.uuid + } as VolumeSnapshotInventory + + createDataVolumeFromVolumeSnapshot { + name = "data-vol-from-sp" + volumeSnapshotUuid = sp.uuid + systemTags = [VolumeSystemTags.FAST_CREATE.tagFormat] + primaryStorageUuid = env.inventoryByName("nfs").uuid + } + + assert Q.New(VolumeSnapshotReferenceTreeVO.class).isNull(VolumeSnapshotReferenceTreeVO_.rootImageUuid).isExists() + } + void testDelete(){ + fastCloneVmBeforeDeletingImageCache() + dbf = bean(DatabaseFacade.class) PrimaryStorageInventory nfs = env.inventoryByName("nfs") @@ -264,6 +288,7 @@ class CleanImageCacheOnPrimaryStorageCase extends SubCase{ c = q.find() assert null != c + q = dbf.createQuery(ImageCacheVO.class) q.add(ImageCacheVO_.imageUuid, SimpleQuery.Op.EQ, image1.getUuid()) q.add(ImageCacheVO_.primaryStorageUuid, SimpleQuery.Op.EQ, nfs.getUuid()) c = q.find() From 7805e366147780041e5a0818a752e15756f09bf0 Mon Sep 17 00:00:00 2001 From: "yaohua.wu" Date: Wed, 13 Aug 2025 17:33:27 +0800 Subject: [PATCH 10/11] [storage]: markSnapshotAsVolume before delete origin volume bits Resolves: ZSTAC-76704 Change-Id: I6d66726c636e6e707076667772707a7578777379 (cherry picked from commit 6efc547428f64657001cb2eed2d7a27b715a6a0e) --- .../org/zstack/storage/volume/VolumeBase.java | 64 +++++++++++-------- 1 file changed, 36 insertions(+), 28 deletions(-) diff --git a/storage/src/main/java/org/zstack/storage/volume/VolumeBase.java b/storage/src/main/java/org/zstack/storage/volume/VolumeBase.java index 95d7bb86060..4c40b7a95dc 100755 --- a/storage/src/main/java/org/zstack/storage/volume/VolumeBase.java +++ b/storage/src/main/java/org/zstack/storage/volume/VolumeBase.java @@ -46,6 +46,8 @@ import org.zstack.identity.AccountManager; import org.zstack.storage.primary.EstimateVolumeTemplateSizeOnPrimaryStorageMsg; import org.zstack.storage.primary.EstimateVolumeTemplateSizeOnPrimaryStorageReply; +import org.zstack.storage.primary.PrimaryStorageDeleteBitGC; +import org.zstack.storage.primary.PrimaryStorageGlobalConfig; import org.zstack.storage.snapshot.group.VolumeSnapshotGroupOperationValidator; import org.zstack.storage.snapshot.reference.VolumeSnapshotReferenceUtils; import org.zstack.tag.SystemTagCreator; @@ -59,6 +61,7 @@ import javax.persistence.TypedQuery; import java.util.*; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.zstack.core.Platform.*; @@ -3561,7 +3564,7 @@ public void run(MessageReply reply) { }); flow(new NoRollbackFlow() { - String __name__ = "delete-origin-volume-bits"; + String __name__ = "update-db-install-path"; @Override public boolean skip(Map data) { @@ -3570,24 +3573,18 @@ public boolean skip(Map data) { @Override public void run(FlowTrigger trigger, Map data) { - DeleteVolumeBitsOnPrimaryStorageMsg dmsg = new DeleteVolumeBitsOnPrimaryStorageMsg(); - dmsg.setPrimaryStorageUuid(self.getPrimaryStorageUuid()); - dmsg.setInstallPath(originVolumePath); - dmsg.setSize(self.getSize()); - dmsg.setBitsType(VolumeVO.class.getSimpleName()); - dmsg.setBitsUuid(self.getUuid()); - dmsg.setHypervisorType(VolumeFormat.getMasterHypervisorTypeByVolumeFormat(getSelfInventory().getFormat()).toString()); - dmsg.setFolder(false); - dmsg.setFromRecycle(true); - bus.makeTargetServiceIdByResourceUuid(dmsg, PrimaryStorageConstant.SERVICE_ID, self.getPrimaryStorageUuid()); - bus.send(dmsg, new CloudBusCallBack(trigger) { + MarkSnapshotAsVolumeMsg mmsg = new MarkSnapshotAsVolumeMsg(); + mmsg.setVolumeUuid(self.getUuid()); + mmsg.setSnapshotUuid(snapShot.getUuid()); + mmsg.setSize(size); + mmsg.setVolumePath(newVolumeInstallPath); + mmsg.setTreeUuid(snapShot.getTreeUuid()); + bus.makeTargetServiceIdByResourceUuid(mmsg, VolumeSnapshotConstant.SERVICE_ID, snapShot.getUuid()); + bus.send(mmsg, new CloudBusCallBack(trigger) { @Override public void run(MessageReply reply) { if (!reply.isSuccess()) { - if (reply.getError().isError(VolumeErrors.VOLUME_IN_USE)) { - logger.warn(String.format("unable to delete path:%s right now", originVolumePath)); - } - + logger.warn(String.format("mark snapshot:%s as volume failed", snapShot.getUuid())); trigger.fail(reply.getError()); return; } @@ -3599,7 +3596,7 @@ public void run(MessageReply reply) { }); flow(new NoRollbackFlow() { - String __name__ = "update-db-install-path"; + String __name__ = "delete-origin-volume-bits"; @Override public boolean skip(Map data) { @@ -3608,20 +3605,31 @@ public boolean skip(Map data) { @Override public void run(FlowTrigger trigger, Map data) { - MarkSnapshotAsVolumeMsg mmsg = new MarkSnapshotAsVolumeMsg(); - mmsg.setVolumeUuid(self.getUuid()); - mmsg.setSnapshotUuid(snapShot.getUuid()); - mmsg.setSize(size); - mmsg.setVolumePath(newVolumeInstallPath); - mmsg.setTreeUuid(snapShot.getTreeUuid()); - bus.makeTargetServiceIdByResourceUuid(mmsg, VolumeSnapshotConstant.SERVICE_ID, snapShot.getUuid()); - bus.send(mmsg, new CloudBusCallBack(trigger) { + DeleteVolumeBitsOnPrimaryStorageMsg dmsg = new DeleteVolumeBitsOnPrimaryStorageMsg(); + dmsg.setPrimaryStorageUuid(self.getPrimaryStorageUuid()); + dmsg.setInstallPath(originVolumePath); + dmsg.setSize(self.getSize()); + dmsg.setBitsType(VolumeVO.class.getSimpleName()); + dmsg.setBitsUuid(self.getUuid()); + dmsg.setHypervisorType(VolumeFormat.getMasterHypervisorTypeByVolumeFormat(getSelfInventory().getFormat()).toString()); + dmsg.setFolder(false); + dmsg.setFromRecycle(true); + bus.makeTargetServiceIdByResourceUuid(dmsg, PrimaryStorageConstant.SERVICE_ID, self.getPrimaryStorageUuid()); + bus.send(dmsg, new CloudBusCallBack(trigger) { @Override public void run(MessageReply reply) { if (!reply.isSuccess()) { - logger.warn(String.format("mark snapshot:%s as volume failed", snapShot.getUuid())); - trigger.fail(reply.getError()); - return; + if (reply.getError().isError(VolumeErrors.VOLUME_IN_USE)) { + logger.warn(String.format("unable to delete path:%s right now", originVolumePath)); + } + + PrimaryStorageDeleteBitGC gc = new PrimaryStorageDeleteBitGC(); + gc.NAME = String.format("gc-delete-bits-volume-%s-on-primary-storage-%s", self.getUuid(), self.getPrimaryStorageUuid()); + gc.primaryStorageInstallPath = originVolumePath; + gc.primaryStorageUuid = self.getPrimaryStorageUuid(); + gc.volume = self; + gc.submit(PrimaryStorageGlobalConfig.PRIMARY_STORAGE_DELETEBITS_GARBAGE_COLLECTOR_INTERVAL.value(Long.class), + TimeUnit.SECONDS); } trigger.next(); From 115ac2b7841b6553011d28c6d3874cab625148df Mon Sep 17 00:00:00 2001 From: "hanyu.liang" Date: Tue, 23 Sep 2025 14:15:59 +0800 Subject: [PATCH 11/11] [host]: Supports returning cpuCoreNum when getting capacity Resolves: ZSTAC-77257 Change-Id: I737577257765547655441897574646678757a7a734578 (cherry picked from commit 9c99f997165a3b6717cd8e42b697c02b575a3c06) --- .../compute/allocator/HostAllocatorManagerImpl.java | 6 ++++-- conf/db/upgrade/V4.8.38__schema.sql | 2 ++ .../header/allocator/HostCapacityInventory.java | 10 ++++++++++ .../org/zstack/header/allocator/HostCapacityVO.java | 11 +++++++++++ .../org/zstack/header/allocator/HostCapacityVO_.java | 1 + .../header/cluster/ReportHostCapacityMessage.java | 9 +++++++++ .../main/java/org/zstack/kvm/KVMAgentCommands.java | 10 ++++++++++ plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java | 1 + .../main/java/org/zstack/sdk/LicenseInventory.java | 8 ++++++++ .../api/server/RequestLicenseCapacityAction.java | 2 +- .../capacity/CheckHostCapacityWhenAddHostCase.groovy | 9 +++++++++ 11 files changed, 66 insertions(+), 3 deletions(-) diff --git a/compute/src/main/java/org/zstack/compute/allocator/HostAllocatorManagerImpl.java b/compute/src/main/java/org/zstack/compute/allocator/HostAllocatorManagerImpl.java index daf3a9259a2..814cf91b83a 100755 --- a/compute/src/main/java/org/zstack/compute/allocator/HostAllocatorManagerImpl.java +++ b/compute/src/main/java/org/zstack/compute/allocator/HostAllocatorManagerImpl.java @@ -286,6 +286,7 @@ private void handle(ReportHostCapacityMessage msg) { vo.setAvailablePhysicalMemory(availMem); vo.setCpuNum(msg.getCpuNum()); vo.setCpuSockets(msg.getCpuSockets()); + vo.setCpuCoreNum(msg.getCpuCoreNum()); HostCapacityStruct s = new HostCapacityStruct(); s.setCpuSockets(vo.getCpuSockets()); @@ -308,6 +309,7 @@ private void handle(ReportHostCapacityMessage msg) { vo.setAvailablePhysicalMemory(availMem); vo.setTotalMemory(msg.getTotalMemory()); vo.setCpuSockets(msg.getCpuSockets()); + vo.setCpuCoreNum(msg.getCpuCoreNum()); HostCapacityStruct s = new HostCapacityStruct(); s.setCapacityVO(vo); @@ -327,10 +329,10 @@ private void handle(ReportHostCapacityMessage msg) { } private boolean needUpdateCapacity(HostCapacityVO vo, ReportHostCapacityMessage msg, long totalCpu, long avaliCpu, long availMem) { - return vo.getCpuNum() != msg.getCpuNum() || vo.getTotalCpu() != totalCpu + return vo.getCpuNum() != msg.getCpuNum() || vo.getTotalCpu() != totalCpu || vo.getAvailableCpu() != avaliCpu || vo.getTotalPhysicalMemory() != msg.getTotalMemory() || vo.getAvailablePhysicalMemory() != availMem || vo.getTotalMemory() != msg.getTotalMemory() - || vo.getCpuSockets() != msg.getCpuSockets(); + || vo.getCpuSockets() != msg.getCpuSockets() || vo.getCpuCoreNum() != msg.getCpuCoreNum(); } private void handle(final AllocateHostMsg msg) { diff --git a/conf/db/upgrade/V4.8.38__schema.sql b/conf/db/upgrade/V4.8.38__schema.sql index e69de29bb2d..da155043107 100644 --- a/conf/db/upgrade/V4.8.38__schema.sql +++ b/conf/db/upgrade/V4.8.38__schema.sql @@ -0,0 +1,2 @@ +CALL ADD_COLUMN('LicenseHistoryVO', 'quotaType', 'varchar(64)', 1, 'None'); +CALL ADD_COLUMN('HostCapacityVO', 'cpuCoreNum', 'int unsigned', 0, '0'); diff --git a/header/src/main/java/org/zstack/header/allocator/HostCapacityInventory.java b/header/src/main/java/org/zstack/header/allocator/HostCapacityInventory.java index 8fd888110f2..ca76b7234ff 100755 --- a/header/src/main/java/org/zstack/header/allocator/HostCapacityInventory.java +++ b/header/src/main/java/org/zstack/header/allocator/HostCapacityInventory.java @@ -15,6 +15,7 @@ public class HostCapacityInventory { private Long totalCpu; private Integer cpuNum; private Integer cpuSockets; + private Integer cpuCoreNum; private Long availableMemory; private Long availableCpu; private Long totalPhysicalMemory; @@ -31,6 +32,7 @@ public static HostCapacityInventory valueOf(HostCapacityVO vo) { inv.setTotalPhysicalMemory(vo.getTotalPhysicalMemory()); inv.setCpuNum(vo.getCpuNum()); inv.setCpuSockets(vo.getCpuSockets()); + inv.setCpuCoreNum(vo.getCpuCoreNum()); return inv; } @@ -42,6 +44,14 @@ public static List valueOf(Collection vos return invs; } + public Integer getCpuCoreNum() { + return cpuCoreNum; + } + + public void setCpuCoreNum(Integer cpuCoreNum) { + this.cpuCoreNum = cpuCoreNum; + } + public Integer getCpuSockets() { return cpuSockets; } diff --git a/header/src/main/java/org/zstack/header/allocator/HostCapacityVO.java b/header/src/main/java/org/zstack/header/allocator/HostCapacityVO.java index 18a66f7af99..b6a17d1a91f 100755 --- a/header/src/main/java/org/zstack/header/allocator/HostCapacityVO.java +++ b/header/src/main/java/org/zstack/header/allocator/HostCapacityVO.java @@ -40,6 +40,9 @@ public class HostCapacityVO { @Column private int cpuSockets; + @Column + private int cpuCoreNum; + @Column @Index private long availableMemory; @@ -63,6 +66,14 @@ public int getCpuSockets() { return cpuSockets; } + public int getCpuCoreNum() { + return cpuCoreNum; + } + + public void setCpuCoreNum(int cpuCoreNum) { + this.cpuCoreNum = cpuCoreNum; + } + public void setCpuSockets(int cpuSockets) { this.cpuSockets = cpuSockets; } diff --git a/header/src/main/java/org/zstack/header/allocator/HostCapacityVO_.java b/header/src/main/java/org/zstack/header/allocator/HostCapacityVO_.java index 0e256752159..d521909ad5a 100755 --- a/header/src/main/java/org/zstack/header/allocator/HostCapacityVO_.java +++ b/header/src/main/java/org/zstack/header/allocator/HostCapacityVO_.java @@ -10,6 +10,7 @@ public class HostCapacityVO_ { public static volatile SingularAttribute totalCpu; public static volatile SingularAttribute cpuNum; public static volatile SingularAttribute cpuSockets; + public static volatile SingularAttribute cpuCoreNum; public static volatile SingularAttribute availableMemory; public static volatile SingularAttribute availableCpu; public static volatile SingularAttribute totalPhysicalMemory; diff --git a/header/src/main/java/org/zstack/header/cluster/ReportHostCapacityMessage.java b/header/src/main/java/org/zstack/header/cluster/ReportHostCapacityMessage.java index bd41d6a2cca..e2d069c6a10 100755 --- a/header/src/main/java/org/zstack/header/cluster/ReportHostCapacityMessage.java +++ b/header/src/main/java/org/zstack/header/cluster/ReportHostCapacityMessage.java @@ -10,6 +10,15 @@ public class ReportHostCapacityMessage extends NeedReplyMessage { private String hostUuid; private int cpuNum; private int cpuSockets; + private int cpuCoreNum; + + public int getCpuCoreNum() { + return cpuCoreNum; + } + + public void setCpuCoreNum(int cpuCoreNum) { + this.cpuCoreNum = cpuCoreNum; + } public int getCpuSockets() { return cpuSockets; diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java index 5ad9b5e7f7c..0a43ba44092 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java @@ -778,6 +778,16 @@ public static class HostCapacityResponse extends AgentResponse { private long usedMemory; @GrayVersion(value = "5.0.0") private int cpuSockets; + @GrayVersion(value = "5.4.0") + private int cpuCoreNum; + + public int getCpuCoreNum() { + return cpuCoreNum; + } + + public void setCpuCoreNum(int cpuCoreNum) { + this.cpuCoreNum = cpuCoreNum; + } public int getCpuSockets() { return cpuSockets; diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java index 7750c06dce7..3d63c180ea5 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java @@ -1701,6 +1701,7 @@ public void run(MessageReply reply) { rmsg.setTotalMemory(rsp.getTotalMemory()); rmsg.setUsedMemory(rsp.getUsedMemory()); rmsg.setCpuSockets(rsp.getCpuSockets()); + rmsg.setCpuCoreNum(rsp.getCpuCoreNum()); rmsg.setServiceId(bus.makeLocalServiceId(HostAllocatorConstant.SERVICE_ID)); bus.send(rmsg, new CloudBusCallBack(msg) { @Override diff --git a/sdk/src/main/java/org/zstack/sdk/LicenseInventory.java b/sdk/src/main/java/org/zstack/sdk/LicenseInventory.java index 204094611e3..d4217d9b607 100644 --- a/sdk/src/main/java/org/zstack/sdk/LicenseInventory.java +++ b/sdk/src/main/java/org/zstack/sdk/LicenseInventory.java @@ -68,6 +68,14 @@ public java.lang.String getLicenseType() { return this.licenseType; } + public java.lang.String quotaType; + public void setQuotaType(java.lang.String quotaType) { + this.quotaType = quotaType; + } + public java.lang.String getQuotaType() { + return this.quotaType; + } + public java.lang.String expiredDate; public void setExpiredDate(java.lang.String expiredDate) { this.expiredDate = expiredDate; diff --git a/sdk/src/main/java/org/zstack/sdk/license/api/server/RequestLicenseCapacityAction.java b/sdk/src/main/java/org/zstack/sdk/license/api/server/RequestLicenseCapacityAction.java index 3a5805d0b44..8e632d9428c 100644 --- a/sdk/src/main/java/org/zstack/sdk/license/api/server/RequestLicenseCapacityAction.java +++ b/sdk/src/main/java/org/zstack/sdk/license/api/server/RequestLicenseCapacityAction.java @@ -28,7 +28,7 @@ public Result throwExceptionIfError() { @Param(required = true, nonempty = false, nullElements = false, emptyString = true, noTrim = false) public java.lang.String resourceUuid; - @Param(required = true, validValues = {"CPU","VM","Host","Capacity","None"}, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + @Param(required = true, validValues = {"CPU_Socket","CPU_Core","VM","Host","Capacity","None"}, nonempty = false, nullElements = false, emptyString = true, noTrim = false) public java.lang.String quotaType; @Param(required = true, nonempty = false, nullElements = false, emptyString = true, numberRange = {0L,9223372036854775807L}, noTrim = false) diff --git a/test/src/test/groovy/org/zstack/test/integration/kvm/capacity/CheckHostCapacityWhenAddHostCase.groovy b/test/src/test/groovy/org/zstack/test/integration/kvm/capacity/CheckHostCapacityWhenAddHostCase.groovy index b8c046775a2..021b5eaf823 100755 --- a/test/src/test/groovy/org/zstack/test/integration/kvm/capacity/CheckHostCapacityWhenAddHostCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/kvm/capacity/CheckHostCapacityWhenAddHostCase.groovy @@ -1,6 +1,9 @@ package org.zstack.test.integration.kvm.capacity import org.springframework.http.HttpEntity +import org.zstack.core.db.Q +import org.zstack.header.allocator.HostCapacityVO +import org.zstack.header.allocator.HostCapacityVO_ import org.zstack.kvm.KVMAgentCommands import org.zstack.kvm.KVMConstant import org.zstack.kvm.KVMGlobalConfig @@ -81,6 +84,7 @@ class CheckHostCapacityWhenAddHostCase extends SubCase { env.afterSimulator(KVMConstant.KVM_HOST_CAPACITY_PATH) { rsp, HttpEntity e -> rsp as KVMAgentCommands.HostCapacityResponse + rsp.setCpuCoreNum(20) rsp.setTotalMemory(SizeUnit.GIGABYTE.toByte(10)) return rsp } @@ -101,6 +105,11 @@ class CheckHostCapacityWhenAddHostCase extends SubCase { res = action.call() assert res.error == null + retryInSecs { + Integer cpuCoreNum = Q.New(HostCapacityVO.class).eq(HostCapacityVO_.uuid, res.value.getInventory().uuid).select(HostCapacityVO_.cpuCoreNum).findValue() + assert cpuCoreNum == 20 + } + deleteHost { uuid = res.value.inventory.uuid }