<fix>[kvm]: TIC-5776 backport memory hotfix

compute/src/main/java/org/zstack/compute/host/HostTrackImpl.java

@@ -1,5 +1,7 @@
 package org.zstack.compute.host;
 
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.zstack.core.CoreGlobalProperty;
 import org.zstack.core.cloudbus.*;
@@ -35,6 +37,7 @@ public class HostTrackImpl implements HostTracker, ManagementNodeChangeListener,
 
     private Map<String, Tracker> trackers = new ConcurrentHashMap<>();
     private static boolean alwaysStartRightNow = false;
+    private static final Cache<String, Long> skippedPingHostDeadline = CacheBuilder.newBuilder().expireAfterWrite(1, TimeUnit.HOURS).build();
 
     @Autowired
     private DatabaseFacade dbf;
@@ -124,6 +127,12 @@ private void track()  {
                 return;
             }
 
+            if (skippedPingHostDeadline.getIfPresent(uuid) != null && System.currentTimeMillis() / 1000 <= skippedPingHostDeadline.getIfPresent(uuid)) {
+                logger.debug(String.format("skip tracking host[uuid:%s] this time, deadline %s", uuid, skippedPingHostDeadline.getIfPresent(uuid)));
+                continueToRunThisTimer();
+                return;
+            }
+
             PingHostMsg msg = new PingHostMsg();
             msg.setHostUuid(uuid);
             bus.makeLocalServiceId(msg, HostConstant.SERVICE_ID);
@@ -341,6 +350,7 @@ private HostReconnectTaskFactory getHostReconnectTaskFactory(String hvType) {
     public boolean start() {
         populateExtensions();
         onHostStatusChange();
+        onHostPingSkip();
 
         HostGlobalConfig.PING_HOST_INTERVAL.installUpdateExtension((oldConfig, newConfig) -> {
             logger.debug(String.format("%s change from %s to %s, restart host trackers",
@@ -386,6 +396,24 @@ protected void run(Map tokens, Object data) {
         });
     }
 
+    private void onHostPingSkip() {
+        evtf.on(HostCanonicalEvents.HOST_PING_SKIP, new EventCallback() {
+            @Override
+            protected void run(Map tokens, Object data) {
+                HostCanonicalEvents.HostPingSkipData d = (HostCanonicalEvents.HostPingSkipData) data;
+                Long deadline = System.currentTimeMillis() / 1000 + d.getSkipTimeInSec();
+                skippedPingHostDeadline.put(d.getHostUuid(), deadline);
+            }
+        });
+        evtf.on(HostCanonicalEvents.HOST_PING_CANCEL_SKIP, new EventCallback() {
+            @Override
+            protected void run(Map tokens, Object data) {
+                HostCanonicalEvents.HostPingSkipData d = (HostCanonicalEvents.HostPingSkipData) data;
+                skippedPingHostDeadline.invalidate(d.getHostUuid());
+            }
+        });
+    }
+
     @Override
     public boolean stop() {
         return true;

conf/globalConfig/kvm.xml

@@ -257,4 +257,20 @@
         <defaultValue>1800</defaultValue>
         <type>java.lang.Integer</type>
     </config>
+
+    <config>
+        <category>kvm</category>
+        <name>kvmagent.physicalmemory.usage.alarm.threshold</name>
+        <description>The threshold for the physical memory usage of the kvmagent process, exceeding which an alarm will be triggered.</description>
+        <defaultValue>2147483648</defaultValue>
+        <type>java.lang.Long</type>
+    </config>
+
+    <config>
+        <category>kvm</category>
+        <name>kvmagent.physicalmemory.usage.hardlimit</name>
+        <description>The hard limit for the physical memory usage of the kvmagent process, exceeding this value will trigger a kvmagent restart.</description>
+        <defaultValue>10737418240</defaultValue>
+        <type>java.lang.Long</type>
+    </config>
 </globalConfig>

core/src/main/java/org/zstack/core/CoreManagerImpl.java

@@ -98,6 +98,9 @@ private void handle(GetLocalTaskMsg msg) {
         GetLocalTaskReply reply = new GetLocalTaskReply();
         Map<String, ChainInfo> results = msg.getSyncSignatures().stream()
                 .collect(Collectors.toMap(Function.identity(), thdf::getChainTaskInfo));
+        if (msg.isOnlyRunningTask()) {
+            results.values().forEach(c -> c.getPendingTask().clear());
+        }
         reply.setResults(results);
         bus.reply(msg, reply);
     }

header/src/main/java/org/zstack/header/core/GetLocalTaskMsg.java

@@ -6,6 +6,15 @@
 
 public class GetLocalTaskMsg extends NeedReplyMessage {
     private List<String> syncSignatures;
+    private boolean onlyRunningTask;
+
+    public boolean isOnlyRunningTask() {
+        return onlyRunningTask;
+    }
+
+    public void setOnlyRunningTask(boolean onlyRunningTask) {
+        this.onlyRunningTask = onlyRunningTask;
+    }
 
     public void setSyncSignatures(List<String> syncSignatures) {
         this.syncSignatures = syncSignatures;

header/src/main/java/org/zstack/header/host/HostCanonicalEvents.java

@@ -23,6 +23,10 @@ public class HostCanonicalEvents {
     public static final String HOST_PHYSICAL_DISK_STATUS_ABNORMAL = "/host/physicalDisk/status/abnormal";
     public static final String HOST_PHYSICAL_DISK_INSERT_TRIGGERED = "/host/physicalDisk/insert/triggered";
     public static final String HOST_PHYSICAL_DISK_REMOVE_TRIGGERED = "/host/physicalDisk/remove/triggered";
+    public static final String HOST_PROCESS_PHYSICAL_MEMORY_USAGE_ABNORMAL = "/host/process/physicalMemory/usage/abnormal";
+    public static final String HOST_PING_SKIP = "/host/ping/skip";
+    public static final String HOST_PING_CANCEL_SKIP = "/host/ping/cancel/skip";
+
 
     @NeedJsonSchema
     public static class HostPhysicalCpuStatusAbnormalData {
@@ -399,4 +403,66 @@ public void setInterfaceStatus(String interfaceStatus) {
             this.interfaceStatus = interfaceStatus;
         }
     }
+
+    @NeedJsonSchema
+    public static class HostProcessPhysicalMemoryUsageAlarmData {
+        private String hostUuid;
+        private String pid;
+        private String processName;
+        private String memoryUsage;
+
+        public String getHostUuid() {
+            return hostUuid;
+        }
+
+        public void setHostUuid(String hostUuid) {
+            this.hostUuid = hostUuid;
+        }
+
+        public String getPid() {
+            return pid;
+        }
+
+        public void setPid(String pid) {
+            this.pid = pid;
+        }
+
+        public String getProcessName() {
+            return processName;
+        }
+
+        public void setProcessName(String processName) {
+            this.processName = processName;
+        }
+
+        public String getMemoryUsage() {
+            return memoryUsage;
+        }
+
+        public void setMemoryUsage(String memoryUsage) {
+            this.memoryUsage = memoryUsage;
+        }
+    }
+
+    @NeedJsonSchema
+    public static class HostPingSkipData {
+        private String hostUuid;
+        private int skipTimeInSec;
+
+        public String getHostUuid() {
+            return hostUuid;
+        }
+
+        public void setHostUuid(String hostUuid) {
+            this.hostUuid = hostUuid;
+        }
+
+        public int getSkipTimeInSec() {
+            return skipTimeInSec;
+        }
+
+        public void setSkipTimeInSec(int skipTimeInSec) {
+            this.skipTimeInSec = skipTimeInSec;
+        }
+    }
 }

plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java

@@ -330,6 +330,7 @@ public void setQemuVersion(String qemuVersion) {
 
     public static class PingCmd extends AgentCommand {
         public String hostUuid;
+        public Map<String, Object> configs;
     }
 
     public static class PingResponse extends AgentResponse {
@@ -4127,4 +4128,83 @@ public void setImageData(String imageData) {
             this.imageData = imageData;
         }
     }
+
+    public static class HostProcessPhysicalMemoryUsageAlarmCmd {
+        private String hostUuid;
+        private String pid;
+        private String processName;
+        private long memoryUsage;
+        private Map<String, Object> additionalProperties = new HashMap<>();
+
+        public String getHostUuid() {
+            return hostUuid;
+        }
+
+        public void setHostUuid(String hostUuid) {
+            this.hostUuid = hostUuid;
+        }
+
+        public String getPid() {
+            return pid;
+        }
+
+        public void setPid(String pid) {
+            this.pid = pid;
+        }
+
+        public String getProcessName() {
+            return processName;
+        }
+
+        public void setProcessName(String processName) {
+            this.processName = processName;
+        }
+
+        public long getMemoryUsage() {
+            return memoryUsage;
+        }
+
+        public void setMemoryUsage(long memoryUsage) {
+            this.memoryUsage = memoryUsage;
+        }
+
+        public Map<String, Object> getAdditionalProperties() {
+            return additionalProperties;
+        }
+
+        public void setAdditionalProperties(Map<String, Object> additionalProperties) {
+            this.additionalProperties = additionalProperties;
+        }
+    }
+
+    public static class HostKvmagentStatusCmd {
+        private String status;
+        private String hostUuid;
+        private long memoryUsage;
+
+        public String getHostUuid() {
+            return hostUuid;
+        }
+
+        public void setHostUuid(String hostUuid) {
+            this.hostUuid = hostUuid;
+        }
+
+        public String getStatus() {
+            return status;
+        }
+
+        public void setStatus(String status) {
+            this.status = status;
+        }
+
+        public long getMemoryUsage() {
+            return memoryUsage;
+        }
+
+        public void setMemoryUsage(long memoryUsage) {
+            this.memoryUsage = memoryUsage;
+        }
+    }
+
 }

plugin/kvm/src/main/java/org/zstack/kvm/KVMConstant.java

@@ -109,6 +109,9 @@ public interface KVMConstant {
     String KVM_BLOCK_COMMIT_VOLUME_PATH = "/vm/volume/blockcommit";
     String TAKE_VM_CONSOLE_SCREENSHOT_PATH = "/vm/console/screenshot";
 
+    String HOST_PROCESS_PHYSICAL_MEMORY_USAGE_ALARM_PATH = "/host/process/physicalMemory/usage/alarm";
+    String HOST_KVMAGENT_STATUS_PATH = "/host/kvmagent/status";
+
     String KVM_AGENT_OWNER = "kvm";
 
     String ALI_REPO = "ali";

plugin/kvm/src/main/java/org/zstack/kvm/KVMGlobalConfig.java

@@ -1,6 +1,5 @@
 package org.zstack.kvm;
 
-import org.zstack.core.GlobalProperty;
 import org.zstack.core.config.GlobalConfig;
 import org.zstack.core.config.GlobalConfigDef;
 import org.zstack.core.config.GlobalConfigDefinition;
@@ -130,4 +129,15 @@ public class KVMGlobalConfig {
     @GlobalConfigDef(defaultValue = "none", description = "enable host ksm")
     @BindResourceConfig({HostVO.class})
     public static GlobalConfig HOST_KSM = new GlobalConfig(CATEGORY, "host.ksm");
+
+    @GlobalConfigValidation(validValues = {"true", "false"})
+    @GlobalConfigDef(defaultValue = "false", description = "restart kvm host libvirtd service or not")
+    @BindResourceConfig({HostVO.class, ClusterVO.class})
+    public static GlobalConfig RECONNECT_HOST_RESTART_LIBVIRTD_SERVICE = new GlobalConfig(CATEGORY, "reconnect.host.restart.libvirtd.service");
+
+    @GlobalConfigValidation(numberGreaterThan = 0)
+    public static GlobalConfig KVMAGENT_PHYSICAL_MEMORY_USAGE_ALARM_THRESHOLD = new GlobalConfig(CATEGORY, "kvmagent.physicalmemory.usage.alarm.threshold");
+
+    @GlobalConfigValidation(numberGreaterThan = 0)
+    public static GlobalConfig KVMAGENT_PHYSICAL_MEMORY_USAGE_HARD_LIMIT = new GlobalConfig(CATEGORY, "kvmagent.physicalmemory.usage.hardlimit");
 }