<fix>[kvm]: preserve skip list on nodeLeft with TTL orphan mechanism

- nodeLeft moves VMs to orphaned skip list with timestamp (not immediate removal)
- Periodic cleanup task (60s) removes orphaned entries after timeout (default 10min)
- VmTracer double-check: orphaned VMs in Starting/Migrating skip missing-VM handling
- New GlobalConfig: kvm.orphanedVmSkipTimeout
- Prevents split-brain when MN restarts while VMs starting on kvmagent

Resolves: ZSTAC-80821

Change-Id: Ie56e937941034d29598627dcdf88d61260b06434

Author: AlanJager

compute/src/main/java/org/zstack/compute/vm/VmTracer.java

@@ -116,7 +116,8 @@ private void checkFromManagementServerSide() {
                 String vmUuid = e.getKey();
                 VmInstanceState expectedState = e.getValue();
                 if (expectedState != VmInstanceState.Stopped && expectedState != VmInstanceState.Created && !hostSideStates.containsKey(vmUuid)
-                        && (vmsToSkipHostSide == null || !vmsToSkipHostSide.contains(vmUuid))) {
+                        && (vmsToSkipHostSide == null || !vmsToSkipHostSide.contains(vmUuid))
+                        && !shouldSkipMissingVmHandling(vmUuid, expectedState)) {
                     handleMissingVm(vmUuid, expectedState);
                 }
             }
@@ -173,6 +174,17 @@ protected Map<String, VmInstanceState> buildManagementServerSideVmStates(String
         return mgmtSideStates;
     }
 
+    /**
+     * Hook for subclasses to add additional skip logic for missing VM handling.
+     * @param vmUuid the VM uuid
+     * @param expectedState the expected state in DB
+     * @return true if missing VM handling should be skipped
+     */
+    protected boolean shouldSkipMissingVmHandling(String vmUuid, VmInstanceState expectedState) {
+        // Default: no additional skip logic
+        return false;
+    }
+
     protected void reportVmState(final String hostUuid, final Map<String, VmInstanceState> vmStates, final Set<String> vmsToSkipHostSide, final Map<String, VmInstanceState> mgmtSideStates) {
         if (logger.isTraceEnabled()) {
             for (Map.Entry<String, VmInstanceState> e : vmStates.entrySet()) {

plugin/kvm/src/main/java/org/zstack/kvm/KVMGlobalConfig.java

@@ -145,4 +145,8 @@ public class KVMGlobalConfig {
     @BindResourceConfig({HostVO.class, ClusterVO.class})
     public static GlobalConfig MINIMUM_MEMORY_SIZE_BEFORE_START_VM = new GlobalConfig(CATEGORY, "min.free.memory.size");
 
+    @GlobalConfigValidation(numberGreaterThan = 0)
+    @GlobalConfigDef(defaultValue = "600", type = Long.class, description = "timeout in seconds for orphaned VM skip list entries (default 10 minutes)")
+    public static GlobalConfig ORPHANED_VM_SKIP_TIMEOUT = new GlobalConfig(CATEGORY, "orphanedVmSkipTimeout");
+
 }

plugin/kvm/src/main/java/org/zstack/kvm/KvmVmSyncPingTask.java

@@ -12,6 +12,7 @@
 import org.zstack.core.thread.ChainTask;
 import org.zstack.core.thread.SyncTaskChain;
 import org.zstack.core.thread.ThreadFacade;
+import org.zstack.core.thread.PeriodicTask;
 import org.zstack.header.Component;
 import org.zstack.header.apimediator.ApiMediatorConstant;
 import org.zstack.header.core.Completion;
@@ -39,6 +40,7 @@
 
 import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
 
 import static org.zstack.core.Platform.getReflections;
 import static org.zstack.core.Platform.operr;
@@ -65,6 +67,9 @@ public class KvmVmSyncPingTask extends VmTracer implements KVMPingAgentNoFailure
     // A map from apiId to VM instance uuid
     private ConcurrentHashMap<String, ConcurrentHashMap<String, String>> vmApis = new ConcurrentHashMap<>();
     private ConcurrentHashMap<String, Set<String>> vmsToSkip = new ConcurrentHashMap<>();
+    // Orphaned skip list: VMs whose management node left but might still be starting
+    // Map: vmUuid -> timestamp when orphaned
+    private ConcurrentHashMap<String, Long> orphanedVmsToSkip = new ConcurrentHashMap<>();
     private List<Class<? extends Message>> skipVmTracerMessages = new ArrayList<>();
     private List<Class> skipVmTracerReplies = new ArrayList<>();
     private Map<String, Integer> vmInShutdownMap = new ConcurrentHashMap<>();
@@ -142,12 +147,16 @@ protected void run(Map tokens, Object data) {
                     String vmUuid = vmApis.get(data1.getManagementNodeId()).remove(data1.getApiId());
                     logger.info("Continuing tracing VM: " + vmUuid);
                     vmsToSkip.get(data1.getManagementNodeId()).remove(vmUuid);
+                    // Also remove from orphaned list if present
+                    orphanedVmsToSkip.remove(vmUuid);
                     return;
                 }
 
                 if (data1.getVmUuid() != null) {
                     logger.info("Continuing tracing VM: " + data1.getVmUuid());
                     vmsToSkip.get(data1.getManagementNodeId()).remove(data1.getVmUuid());
+                    // Also remove from orphaned list if present
+                    orphanedVmsToSkip.remove(data1.getVmUuid());
                 }
             }
         });
@@ -307,8 +316,45 @@ public HypervisorType getHypervisorTypeForReestablishExtensionPoint() {
         return HypervisorType.valueOf(KVMConstant.KVM_HYPERVISOR_TYPE);
     }
 
+    private void startOrphanedVmCleanupTask() {
+        thdf.submitPeriodicTask(new PeriodicTask() {
+            @Override
+            public TimeUnit getTimeUnit() {
+                return TimeUnit.SECONDS;
+            }
+
+            @Override
+            public long getInterval() {
+                return 60; // Check every 60 seconds
+            }
+
+            @Override
+            public String getName() {
+                return "orphaned-vm-skip-list-cleanup";
+            }
+
+            @Override
+            public void run() {
+                long now = System.currentTimeMillis();
+                long timeoutMs = KVMGlobalConfig.ORPHANED_VM_SKIP_TIMEOUT.value(Long.class) * 1000;
+
+                orphanedVmsToSkip.entrySet().removeIf(entry -> {
+                    long age = now - entry.getValue();
+                    if (age > timeoutMs) {
+                        logger.info(String.format("Removing VM[uuid:%s] from orphaned skip list after %d seconds",
+                                entry.getKey(), age / 1000));
+                        return true;
+                    }
+                    return false;
+                });
+            }
+        });
+    }
+
     @Override
     public boolean start() {
+        startOrphanedVmCleanupTask();
+
         restf.registerSyncHttpCallHandler(KVMConstant.KVM_REPORT_VM_STATE, ReportVmStateCmd.class, new SyncHttpCallHandler<ReportVmStateCmd>() {
             private void reportState(final ReportVmStateCmd cmd) {
                 thdf.chainSubmit(new ChainTask(null) {
@@ -446,7 +492,18 @@ public void nodeJoin(ManagementNodeInventory inv) {
     @Override
     public void nodeLeft(ManagementNodeInventory inv) {
         vmApis.remove(inv.getUuid());
-        vmsToSkip.remove(inv.getUuid());
+
+        // Don't immediately remove skip list - move to orphaned set with timestamp
+        // This prevents split-brain when MN restarts and VMs are still starting on kvmagent
+        Set<String> vmsToOrphan = vmsToSkip.remove(inv.getUuid());
+        if (vmsToOrphan != null && !vmsToOrphan.isEmpty()) {
+            long now = System.currentTimeMillis();
+            for (String vmUuid : vmsToOrphan) {
+                orphanedVmsToSkip.put(vmUuid, now);
+                logger.info(String.format("VM[uuid:%s] moved to orphaned skip list due to management node[uuid:%s] left",
+                        vmUuid, inv.getUuid()));
+            }
+        }
     }
 
     @Override
@@ -460,6 +517,30 @@ public void iJoin(ManagementNodeInventory inv) {
     }
 
     public boolean isVmDoNotNeedToTrace(String vmUuid) {
-        return vmsToSkip.values().stream().anyMatch(vmsToSkipSet -> vmsToSkipSet.contains(vmUuid));
+        return vmsToSkip.values().stream().anyMatch(vmsToSkipSet -> vmsToSkipSet.contains(vmUuid))
+                || orphanedVmsToSkip.containsKey(vmUuid);
+    }
+
+    protected boolean isVmOrphaned(String vmUuid) {
+        return orphanedVmsToSkip.containsKey(vmUuid);
+    }
+
+    @Override
+    protected boolean shouldSkipMissingVmHandling(String vmUuid, VmInstanceState expectedState) {
+        // Double-check: if VM is orphaned and DB state indicates it's in a transient state,
+        // skip this trace cycle to avoid split-brain
+        if (!orphanedVmsToSkip.containsKey(vmUuid)) {
+            return false;
+        }
+
+        // If the VM is orphaned and in Starting/Migrating state, it might still be
+        // starting on kvmagent - don't judge it as Stopped yet
+        if (expectedState == VmInstanceState.Starting || expectedState == VmInstanceState.Migrating) {
+            logger.info(String.format("VM[uuid:%s] is orphaned and in state[%s], skipping missing VM check to avoid split-brain",
+                    vmUuid, expectedState));
+            return true;
+        }
+
+        return false;
     }
 }