Improve sync delete handling for lvol bdevs in lvol_monitor.py

simplyblock_core/models/storage_node.py

@@ -344,6 +344,7 @@ def lvol_del_sync_lock(self) -> bool:
             lock = NodeLVolDelLock({"uuid": self.uuid})
             lock.write_to_db()
             logger.info(f"Created lvol_del_sync_lock on node: {self.get_id()}")
+        time.sleep(0.250)
         return True
 
     def lvol_del_sync_lock_reset(self) -> bool:
@@ -367,6 +368,7 @@ def lvol_del_sync_lock_reset(self) -> bool:
             if lock:
                 lock.remove(db_controller.kv_store)
                 logger.info(f"remove lvol_del_sync_lock from node: {self.get_id()}")
+        time.sleep(0.250)
         return True
 
 

simplyblock_core/services/lvol_monitor.py

@@ -133,9 +133,22 @@ def process_lvol_delete_finish(lvol):
         if not ret:
             logger.error(f"Failed to delete lvol from primary_node node: {primary_node.get_id()}")
 
-    # 3-2 async delete lvol bdev from secondary
-    if sec_node and sec_node.status in [StorageNode.STATUS_ONLINE, StorageNode.STATUS_SUSPENDED, StorageNode.STATUS_DOWN, StorageNode.STATUS_UNREACHABLE]:
-        tasks_controller.add_lvol_sync_del_task(sec_node.cluster_id, sec_node.get_id(), f"{lvol.lvs_name}/{lvol.lvol_bdev}", primary_node.get_id())
+    lvol_bdev_name=f"{lvol.lvs_name}/{lvol.lvol_bdev}"
+    if sec_node:
+        if sec_node.status in [StorageNode.STATUS_ONLINE]:
+            logger.info(f"Sync delete bdev: {lvol_bdev_name} from node: {sec_node.get_id()}")
+            ret, err = sec_node.rpc_client().delete_lvol(lvol_bdev_name, del_async=True)
+            if not ret:
+                if "code" in err and err["code"] == -19:
+                    logger.error(f"Sync delete completed with error: {err}")
+                else:
+                    msg = f"Failed to sync delete bdev: {lvol_bdev_name} from node: {sec_node.get_id()}, ading task..."
+                    logger.error(msg)
+                    tasks_controller.add_lvol_sync_del_task(sec_node.cluster_id, sec_node.get_id(), lvol_bdev_name,
+                                                            primary_node.get_id())
+        elif sec_node.status in [StorageNode.STATUS_SUSPENDED, StorageNode.STATUS_DOWN, StorageNode.STATUS_UNREACHABLE]:
+            # 3-2 async delete lvol bdev from secondary
+            tasks_controller.add_lvol_sync_del_task(sec_node.cluster_id, sec_node.get_id(), lvol_bdev_name, primary_node.get_id())
 
     lvol_events.lvol_delete(lvol)
     lvol.remove(db.kv_store)

simplyblock_core/services/snapshot_monitor.py

@@ -75,12 +75,25 @@ def process_snap_delete_finish(snap, leader_node):
         if not ret:
             logger.error(f"Failed to delete snap from node: {snode.get_id()}")
 
-    # 3-2 async delete lvol bdev from secondary
-    if non_leader and non_leader.status in [StorageNode.STATUS_ONLINE, StorageNode.STATUS_SUSPENDED,
-                                            StorageNode.STATUS_DOWN, StorageNode.STATUS_UNREACHABLE]:
-        tasks_controller.add_lvol_sync_del_task(non_leader.cluster_id, non_leader.get_id(), snap.snap_bdev, primary_node.get_id())
-    snapshot_events.snapshot_delete(snap)
-    snap.remove(db.kv_store)
+    lvol_bdev_name = snap.snap_bdev
+    if non_leader:
+        if non_leader.status in [StorageNode.STATUS_ONLINE]:
+            logger.info(f"Sync delete bdev: {lvol_bdev_name} from node: {non_leader.get_id()}")
+            ret, err = non_leader.rpc_client().delete_lvol(lvol_bdev_name, del_async=True)
+            if not ret:
+                if "code" in err and err["code"] == -19:
+                    logger.error(f"Sync delete completed with error: {err}")
+                else:
+                    msg = f"Failed to sync delete bdev: {lvol_bdev_name} from node: {non_leader.get_id()}, ading task..."
+                    logger.error(msg)
+                    tasks_controller.add_lvol_sync_del_task(non_leader.cluster_id, non_leader.get_id(), lvol_bdev_name, primary_node.get_id())
+
+        elif non_leader.status in [StorageNode.STATUS_SUSPENDED, StorageNode.STATUS_DOWN, StorageNode.STATUS_UNREACHABLE]:
+            # 3-2 async delete lvol bdev from secondary
+            tasks_controller.add_lvol_sync_del_task(non_leader.cluster_id, non_leader.get_id(), lvol_bdev_name, primary_node.get_id())
+            snapshot_events.snapshot_delete(snap)
+            snap.remove(db.kv_store)
+
 
 
 def process_snap_delete_try_again(snap):

simplyblock_core/storage_node_ops.py

@@ -1785,8 +1785,8 @@ def restart_storage_node(
                 if new_ssd not in snode.ssd_pcie:
                     try:
                         snode_api.bind_device_to_spdk(new_ssd)
-                    except:
-                        pass
+                    except Exception as e:
+                        logger.error(e)
                     snode.ssd_pcie.append(new_ssd)
 
         fdb_connection = cluster.db_connection