<fix>[host]: Supports returning cpuCoreNum when getting capacity

VERSION

@@ -1,3 +1,3 @@
 MAJOR=4
 MINOR=8
-UPDATE=36
+UPDATE=38

compute/src/main/java/org/zstack/compute/allocator/HostAllocatorManagerImpl.java

@@ -286,6 +286,7 @@ private void handle(ReportHostCapacityMessage msg) {
             vo.setAvailablePhysicalMemory(availMem);
             vo.setCpuNum(msg.getCpuNum());
             vo.setCpuSockets(msg.getCpuSockets());
+            vo.setCpuCoreNum(msg.getCpuCoreNum());
 
             HostCapacityStruct s = new HostCapacityStruct();
             s.setCpuSockets(vo.getCpuSockets());
@@ -308,6 +309,7 @@ private void handle(ReportHostCapacityMessage msg) {
             vo.setAvailablePhysicalMemory(availMem);
             vo.setTotalMemory(msg.getTotalMemory());
             vo.setCpuSockets(msg.getCpuSockets());
+            vo.setCpuCoreNum(msg.getCpuCoreNum());
 
             HostCapacityStruct s = new HostCapacityStruct();
             s.setCapacityVO(vo);
@@ -327,10 +329,10 @@ private void handle(ReportHostCapacityMessage msg) {
     }
 
     private boolean needUpdateCapacity(HostCapacityVO vo, ReportHostCapacityMessage msg, long totalCpu, long avaliCpu, long availMem) {
-        return vo.getCpuNum() != msg.getCpuNum() || vo.getTotalCpu() != totalCpu 
+        return vo.getCpuNum() != msg.getCpuNum() || vo.getTotalCpu() != totalCpu
                 || vo.getAvailableCpu() != avaliCpu || vo.getTotalPhysicalMemory() != msg.getTotalMemory()
                 || vo.getAvailablePhysicalMemory() != availMem || vo.getTotalMemory() != msg.getTotalMemory()
-                || vo.getCpuSockets() != msg.getCpuSockets();
+                || vo.getCpuSockets() != msg.getCpuSockets() || vo.getCpuCoreNum() != msg.getCpuCoreNum();
     }
 
     private void handle(final AllocateHostMsg msg) {

conf/db/upgrade/V4.8.38__schema.sql

@@ -0,0 +1,2 @@
+CALL ADD_COLUMN('LicenseHistoryVO', 'quotaType', 'varchar(64)', 1, 'None');
+CALL ADD_COLUMN('HostCapacityVO', 'cpuCoreNum', 'int unsigned', 0, '0');

core/src/main/java/org/zstack/core/ansible/CallBackNetworkChecker.java

@@ -7,6 +7,7 @@
 import org.zstack.utils.Utils;
 import org.zstack.utils.logging.CLogger;
 import org.zstack.utils.ssh.Ssh;
+import org.zstack.utils.ssh.SshCmdHelper;
 import org.zstack.utils.ssh.SshException;
 import org.zstack.utils.ssh.SshResult;
 
@@ -47,7 +48,7 @@ public void deleteDestFile() {
      * if failed, use nmap to try again.
      */
     private ErrorCode useNcatAndNmapToTestConnection(Ssh ssh) {
-        String srcScript = script.format(password, callBackPort, callbackIp);
+        String srcScript = script.format(SshCmdHelper.shellQuote(password), callBackPort, callbackIp);
 
         SshResult ret = ssh.setExecTimeout(60).shell(srcScript).setTimeout(60).runAndClose();
         ret.raiseExceptionIfFailed();

core/src/main/java/org/zstack/core/ansible/SshChronyConfigChecker.java

@@ -29,11 +29,13 @@ public boolean needDeploy() {
                 .setPassword(password).setPort(sshPort)
                 .setHostname(targetIp);
         try {
-            ssh.command("awk '/^\\s*server/{print $2}' /etc/chrony.conf");
+            ssh.sudoCommand("awk '/^\\s*server/{print $2}' /etc/chrony.conf");
             SshResult ret = ssh.run();
             int returnCode = ret.getReturnCode();
             ssh.reset();
             if (returnCode != 0) {
+                logger.warn(String.format("exec ssh command failed, return code: %d, stdout: %s, stderr: %s",
+                        ret.getReturnCode(), ret.getStdout(), ret.getStderr()));
                 return true;
             }
 

core/src/main/java/org/zstack/core/ansible/SshFileExistChecker.java

@@ -1,16 +1,10 @@
 package org.zstack.core.ansible;
 
-import org.zstack.utils.ShellResult;
-import org.zstack.utils.ShellUtils;
 import org.zstack.utils.Utils;
 import org.zstack.utils.logging.CLogger;
-import org.zstack.utils.path.PathUtil;
 import org.zstack.utils.ssh.Ssh;
 import org.zstack.utils.ssh.SshResult;
 
-import java.util.ArrayList;
-import java.util.List;
-
 /**
  */
 public class SshFileExistChecker implements AnsibleChecker {
@@ -31,7 +25,7 @@ public boolean needDeploy() {
                 .setHostname(targetIp)
                 .setTimeout(5);
         try {
-            ssh.command(String.format("echo %s | sudo -S stat %s 2>/dev/null", password, filePath));
+            ssh.sudoCommand(String.format("stat %s", filePath));
             SshResult ret = ssh.run();
             if (ret.getReturnCode() != 0) {
                 logger.debug(String.format("file not exist, file: %s", filePath));

core/src/main/java/org/zstack/core/ansible/SshFileMd5Checker.java

@@ -1,5 +1,6 @@
 package org.zstack.core.ansible;
 
+import org.zstack.core.CoreGlobalProperty;
 import org.zstack.utils.ShellResult;
 import org.zstack.utils.ShellUtils;
 import org.zstack.utils.Utils;
@@ -33,7 +34,8 @@ private SrcDestPair(String srcPath, String destPath) {
         String destPath;
     }
 
-    public static final String ZSTACKLIB_SRC_PATH = PathUtil.findFileOnClassPath(String.format("ansible/zstacklib/%s", AnsibleGlobalProperty.ZSTACKLIB_PACKAGE_NAME), true).getAbsolutePath();
+    public static final String ZSTACKLIB_SRC_PATH = CoreGlobalProperty.UNIT_TEST_ON ? "/tmp" :
+            PathUtil.findFileOnClassPath(String.format("ansible/zstacklib/%s", AnsibleGlobalProperty.ZSTACKLIB_PACKAGE_NAME), true).getAbsolutePath();
 
     @Override
     public boolean needDeploy() {
@@ -47,9 +49,11 @@ public boolean needDeploy() {
                 String sourceFilePath = b.srcPath;
                 String destFilePath = b.destPath;
 
-                ssh.command(String.format("echo %s | sudo -S md5sum %s 2>/dev/null", password, destFilePath));
+                ssh.sudoCommand(String.format("md5sum %s", destFilePath));
                 SshResult ret = ssh.run();
                 if (ret.getReturnCode() != 0) {
+                    logger.warn(String.format("exec ssh command failed, return code: %d, stdout: %s, stderr: %s",
+                            ret.getReturnCode(), ret.getStdout(), ret.getStderr()));
                     return true;
                 }
                 ssh.reset();
@@ -59,7 +63,7 @@ public boolean needDeploy() {
                 sret.raiseExceptionIfFail();
                 String srcMd5 = sret.getStdout().split(" ")[0];
                 if (!destMd5.equals(srcMd5)) {
-                    logger.debug(String.format("file MD5 changed, src[%s, md5:%s] dest[%s, md5, %s]", sourceFilePath,
+                    logger.debug(String.format("file MD5 changed, src[%s, md5:%s] dest[%s, md5: %s]", sourceFilePath,
                             srcMd5, destFilePath, destMd5));
                     return true;
                 }
@@ -75,10 +79,15 @@ public boolean needDeploy() {
     public void deleteDestFile() {
         for (SrcDestPair b : srcDestPairs) {
             String destFilePath = b.destPath;
+            if (!destFilePath.contains("zstack")) {
+                logger.debug(String.format("skip delete dest file[%s] which is not zstack file", destFilePath));
+                continue;
+            }
+
             Ssh ssh = new Ssh();
             ssh.setUsername(username).setPrivateKey(privateKey)
                     .setPassword(password).setPort(sshPort)
-                    .setHostname(targetIp).command(String.format("rm -f %s", destFilePath)).runAndClose();
+                    .setHostname(targetIp).sudoCommand(String.format("rm -f %s", destFilePath)).runAndClose();
             logger.debug(String.format("delete dest file[%s]", destFilePath));
         }
     }

core/src/main/java/org/zstack/core/ansible/SshFilesMd5Checker.java

@@ -28,9 +28,11 @@ public boolean needDeploy() {
                 .setHostname(ip)
                 .setTimeout(5);
         try {
-            ssh.command(String.format("echo %s | sudo -S md5sum %s 2>/dev/null", password, filePath));
+            ssh.sudoCommand(String.format("md5sum %s", filePath));
             SshResult ret = ssh.run();
             if (ret.getReturnCode() != 0) {
+                logger.warn(String.format("exec ssh command failed, return code: %d, stdout: %s, stderr: %s",
+                        ret.getReturnCode(), ret.getStdout(), ret.getStderr()));
                 return true;
             }
             ssh.reset();
@@ -48,10 +50,15 @@ public boolean needDeploy() {
 
     @Override
     public void deleteDestFile() {
+        if (!filePath.contains("zstack")) {
+            logger.debug(String.format("skip delete dest file[%s] which is not zstack file", filePath));
+            return;
+        }
+
         Ssh ssh = new Ssh();
         ssh.setUsername(username).setPrivateKey(privateKey)
                 .setPassword(password).setPort(sshPort)
-                .setHostname(ip).command(String.format("rm -f %s", filePath)).runAndClose();
+                .setHostname(ip).sudoCommand(String.format("rm -f %s", filePath)).runAndClose();
         logger.debug(String.format("delete dest file[%s]", filePath));
     }
 

core/src/main/java/org/zstack/core/ansible/SshFolderMd5Checker.java

@@ -11,6 +11,7 @@
 import org.zstack.utils.StringDSL.StringWrapper;
 import org.zstack.utils.Utils;
 import org.zstack.utils.logging.CLogger;
+import org.zstack.utils.ssh.SshCmdHelper;
 import org.zstack.utils.ssh.SshResult;
 import org.zstack.utils.ssh.SshShell;
 
@@ -107,7 +108,7 @@ public boolean needDeploy() {
                             srcRes.getStdout(), srcRes.getStderr()));
         }
 
-        String dstScript = script.format(dstFolder, password);
+        String dstScript = script.format(dstFolder, SshCmdHelper.shellQuote(password));
         SshShell ssh = new SshShell();
         ssh.setHostname(hostname);
         ssh.setUsername(username);

core/src/main/java/org/zstack/core/ansible/SshYamlChecker.java

@@ -36,9 +36,11 @@ public boolean needDeploy() {
                 .setHostname(targetIp);
         try {
 
-            ssh.command(String.format("grep -o '%s' %s | uniq | wc -l", getGrepArgs(), yamlFilePath));
+            ssh.sudoCommand(String.format("grep -o '%s' %s | uniq | wc -l", getGrepArgs(), yamlFilePath));
             SshResult ret = ssh.run();
             if (ret.getReturnCode() != 0) {
+                logger.warn(String.format("exec ssh command failed, return code: %d, stdout: %s, stderr: %s",
+                        ret.getReturnCode(), ret.getStdout(), ret.getStderr()));
                 return true;
             }
 

core/src/main/java/org/zstack/core/ansible/SshYumRepoChecker.java

@@ -36,12 +36,13 @@ public boolean needDeploy() {
                 .setPassword(password).setPort(sshPort)
                 .setHostname(targetIp);
         try {
-            ssh.command(String.format(
-                    "echo %s | sudo -S sed -i '/baseurl/s/\\([0-9]\\{1,3\\}\\.\\)\\{3\\}[0-9]\\{1,3\\}:\\([0-9]\\+\\)/%s/g' /etc/yum.repos.d/{zstack,qemu-kvm-ev}-mn.repo",
-                    password, restf.getHostName() + ":" + restf.getPort()
+            ssh.sudoCommand(String.format("sed -i '/baseurl/s/\\([0-9]\\{1,3\\}\\.\\)\\{3\\}[0-9]\\{1,3\\}:\\([0-9]\\+\\)/%s/g' /etc/yum.repos.d/{zstack,qemu-kvm-ev}-mn.repo",
+                    restf.getHostName() + ":" + restf.getPort()
             ));
             SshResult ret = ssh.setTimeout(60).runAndClose();
             if (ret.getReturnCode() != 0) {
+                logger.warn(String.format("exec ssh command failed, return code: %d, stdout: %s, stderr: %s",
+                        ret.getReturnCode(), ret.getStdout(), ret.getStderr()));
                 return true;
             }
 

header/src/main/java/org/zstack/header/allocator/HostCapacityInventory.java

@@ -15,6 +15,7 @@ public class HostCapacityInventory {
     private Long totalCpu;
     private Integer cpuNum;
     private Integer cpuSockets;
+    private Integer cpuCoreNum;
     private Long availableMemory;
     private Long availableCpu;
     private Long totalPhysicalMemory;
@@ -31,6 +32,7 @@ public static HostCapacityInventory valueOf(HostCapacityVO vo) {
         inv.setTotalPhysicalMemory(vo.getTotalPhysicalMemory());
         inv.setCpuNum(vo.getCpuNum());
         inv.setCpuSockets(vo.getCpuSockets());
+        inv.setCpuCoreNum(vo.getCpuCoreNum());
         return inv;
     }
 
@@ -42,6 +44,14 @@ public static List<HostCapacityInventory> valueOf(Collection<HostCapacityVO> vos
         return invs;
     }
 
+    public Integer getCpuCoreNum() {
+        return cpuCoreNum;
+    }
+
+    public void setCpuCoreNum(Integer cpuCoreNum) {
+        this.cpuCoreNum = cpuCoreNum;
+    }
+
     public Integer getCpuSockets() {
         return cpuSockets;
     }

header/src/main/java/org/zstack/header/allocator/HostCapacityVO.java

@@ -40,6 +40,9 @@ public class HostCapacityVO {
     @Column
     private int cpuSockets;
 
+    @Column
+    private int cpuCoreNum;
+
     @Column
     @Index
     private long availableMemory;
@@ -63,6 +66,14 @@ public int getCpuSockets() {
         return cpuSockets;
     }
 
+    public int getCpuCoreNum() {
+        return cpuCoreNum;
+    }
+
+    public void setCpuCoreNum(int cpuCoreNum) {
+        this.cpuCoreNum = cpuCoreNum;
+    }
+
     public void setCpuSockets(int cpuSockets) {
         this.cpuSockets = cpuSockets;
     }

header/src/main/java/org/zstack/header/allocator/HostCapacityVO_.java

@@ -10,6 +10,7 @@ public class HostCapacityVO_ {
     public static volatile SingularAttribute<HostCapacityVO, Long> totalCpu;
     public static volatile SingularAttribute<HostCapacityVO, Integer> cpuNum;
     public static volatile SingularAttribute<HostCapacityVO, Integer> cpuSockets;
+    public static volatile SingularAttribute<HostCapacityVO, Integer> cpuCoreNum;
     public static volatile SingularAttribute<HostCapacityVO, Long> availableMemory;
     public static volatile SingularAttribute<HostCapacityVO, Long> availableCpu;
     public static volatile SingularAttribute<HostCapacityVO, Long> totalPhysicalMemory;

header/src/main/java/org/zstack/header/cluster/ReportHostCapacityMessage.java

@@ -10,6 +10,15 @@ public class ReportHostCapacityMessage extends NeedReplyMessage {
     private String hostUuid;
     private int cpuNum;
     private int cpuSockets;
+    private int cpuCoreNum;
+
+    public int getCpuCoreNum() {
+        return cpuCoreNum;
+    }
+
+    public void setCpuCoreNum(int cpuCoreNum) {
+        this.cpuCoreNum = cpuCoreNum;
+    }
 
     public int getCpuSockets() {
         return cpuSockets;

header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotTree.java

@@ -348,6 +348,7 @@ private VolumeSnapshotInventory getInventory(Set<String> filterUuids) {
 
     private SnapshotLeaf root;
     private String volumeUuid;
+    private String uuid;
 
     public static VolumeSnapshotTree fromInventories(List<VolumeSnapshotInventory> invs) {
         VolumeSnapshotTree tree = new VolumeSnapshotTree();
@@ -377,6 +378,7 @@ public static VolumeSnapshotTree fromInventories(List<VolumeSnapshotInventory> i
             }
 
             tree.volumeUuid = inv.getVolumeUuid();
+            tree.uuid = inv.getTreeUuid();
         }
 
         DebugUtils.Assert(tree.root != null, "why tree root is null???");
@@ -403,6 +405,14 @@ public void setVolumeUuid(String volumeUuid) {
         this.volumeUuid = volumeUuid;
     }
 
+    public String getUuid() {
+        return uuid;
+    }
+
+    public void setUuid(String uuid) {
+        this.uuid = uuid;
+    }
+
     private SnapshotLeaf findSnapshot(final List<SnapshotLeaf> leafs, final Function<Boolean, VolumeSnapshotInventory> func) {
         for (SnapshotLeaf leaf : leafs) {
             SnapshotLeaf ret = findSnapshot(leaf.children, func);
@@ -424,4 +434,12 @@ public SnapshotLeaf findSnapshot(Function<Boolean, VolumeSnapshotInventory> func
         }
         return findSnapshot(root.children, func);
     }
+
+    public SnapshotLeaf findSnapshot(String snapshotUuid) {
+        if (snapshotUuid == null) {
+            return null;
+        }
+
+        return findSnapshot(arg -> arg.getUuid().equals(snapshotUuid));
+    }
 }

header/src/main/java/org/zstack/header/storage/snapshot/reference/VolumeSnapshotReferenceInventory.java

@@ -36,6 +36,9 @@ public class VolumeSnapshotReferenceInventory {
 
     private String directSnapshotInstallUrl;
 
+    /**
+     * the UUID of resource referencing @volumeUuid.
+     */
     private String referenceUuid;
 
     private String referenceType;

header/src/main/java/org/zstack/header/storage/snapshot/reference/VolumeSnapshotReferenceVO.java

@@ -52,6 +52,9 @@ public class VolumeSnapshotReferenceVO {
     private Long parentId;
 
 
+    /**
+     * the UUID of resource referencing @volumeUuid.
+     */
     @Column
     private String referenceUuid;
 

plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java

@@ -778,6 +778,16 @@ public static class HostCapacityResponse extends AgentResponse {
         private long usedMemory;
         @GrayVersion(value = "5.0.0")
         private int cpuSockets;
+        @GrayVersion(value = "5.4.0")
+        private int cpuCoreNum;
+
+        public int getCpuCoreNum() {
+            return cpuCoreNum;
+        }
+
+        public void setCpuCoreNum(int cpuCoreNum) {
+            this.cpuCoreNum = cpuCoreNum;
+        }
 
         public int getCpuSockets() {
             return cpuSockets;

plugin/kvm/src/main/java/org/zstack/kvm/KVMGlobalConfig.java

@@ -120,6 +120,10 @@ public class KVMGlobalConfig {
     @GlobalConfigDef(defaultValue = "false", type = Boolean.class, description = "enable install host shutdown hook")
     public static GlobalConfig INSTALL_HOST_SHUTDOWN_HOOK = new GlobalConfig(CATEGORY, "install.host.shutdown.hook");
 
+    @GlobalConfigValidation(numberGreaterThan = 0)
+    @GlobalConfigDef(defaultValue = "600", type = Long.class, description = "timeout in seconds for orphaned VM skip entries from departed management nodes")
+    public static GlobalConfig ORPHANED_VM_SKIP_TIMEOUT = new GlobalConfig(CATEGORY, "vm.orphanedSkipTimeout");
+
     @GlobalConfigValidation(validValues = {"true", "false"})
     @GlobalConfigDef(defaultValue = "false", type = Boolean.class, description = "enable memory auto balloon")
     @BindResourceConfig({VmInstanceVO.class})