Linstor: Fix on destroy

linstor-manager:
- fix on get_drbd_volumes
    Failed because the key backing-disk is not always present.

LinstorSR:
- Refactored some hosts variable to reference the OpaqueRef status

linstorvolumemanager:
- Add destroying DRBD remnants on each hosts before destroying resource groups
- Correctly remove DB files (#57)
    `glob` method only returns the dir when not wildcard
    is used in the path.
    Therefore the remove call right after will fail everytime.

Signed-off-by: Damien Thenot <[email protected]>
Co-authored-by: Benjamin Reis <[email protected]>
Co-authored-by: Ronan Abhamon <[email protected]>

Author: 3 people

drivers/LinstorSR.py

@@ -650,17 +650,17 @@ def delete(self, uuid):
                 opterr='Cannot get controller node name'
             )
 
-        host = None
+        host_ref = None
         if node_name == 'localhost':
-            host = util.get_this_host_ref(self.session)
+            host_ref = util.get_this_host_ref(self.session)
         else:
             for slave in util.get_all_slaves(self.session):
                 r_name = self.session.xenapi.host.get_record(slave)['hostname']
                 if r_name == node_name:
-                    host = slave
+                    host_ref = slave
                     break
 
-        if not host:
+        if not host_ref:
             raise xs_errors.XenError(
                 'LinstorSRDelete',
                 opterr='Failed to find host with hostname: {}'.format(
@@ -677,7 +677,7 @@ def delete(self, uuid):
                 'groupName': self._group_name,
             }
             self._exec_manager_command(
-                host, 'destroy', args, 'LinstorSRDelete'
+                host_ref, 'destroy', args, 'LinstorSRDelete'
             )
         except Exception as e:
             try:

drivers/linstor-manager

@@ -241,7 +241,10 @@ def get_drbd_volumes(volume_group=None):
     config = json.loads(stdout)
     for resource in config:
         for volume in resource['_this_host']['volumes']:
-            backing_disk = volume['backing-disk']
+            backing_disk = volume.get('backing-disk')
+            if not backing_disk:
+                continue
+
             match = BACKING_DISK_RE.match(backing_disk)
             if not match:
                 continue

drivers/linstorvolumemanager.py

@@ -18,7 +18,6 @@
 
 import distutils.util
 import errno
-import glob
 import json
 import linstor
 import os.path
@@ -1362,13 +1361,27 @@ def destroy(self):
 
             # 4.4. Refresh linstor connection.
             # Without we get this error:
-            #   "Cannot delete resource group 'xcp-sr-linstor_group_thin_device' because it has existing resource definitions.."
+            # "Cannot delete resource group 'xcp-sr-linstor_group_thin_device' because it has existing resource definitions.."
             # Because the deletion of the databse was not seen by Linstor for some reason.
             # It seems a simple refresh of the Linstor connection make it aware of the deletion.
             self._linstor.disconnect()
             self._linstor.connect()
 
-            # 4.5. Destroy group and storage pools.
+            # 4.5. Destroy remaining drbd nodes on hosts.
+            # We check if there is a DRBD node on hosts that could mean blocking when destroying resource groups.
+            # It needs to be done locally by each host so we go through the linstor-manager plugin.
+            # If we don't do this sometimes, the destroy will fail when trying to destroy the resource groups with:
+            # "linstor-manager:destroy error: Failed to destroy SP `xcp-sr-linstor_group_thin_device` on node `r620-s2`: The specified storage pool 'xcp-sr-linstor_group_thin_device' on node 'r620-s2' can not be deleted as volumes / snapshot-volumes are still using it."
+            session = util.timeout_call(5, util.get_localAPI_session)
+            for host_ref in session.xenapi.host.get_all():
+                try:
+                    response = session.xenapi.host.call_plugin(
+                        host_ref, 'linstor-manager', 'destroyDrbdVolumes', {'volume_group': self._group_name}
+                    )
+                except Exception as e:
+                    util.SMlog('Calling destroyDrbdVolumes on host {} failed with error {}'.format(host_ref, e))
+
+            # 4.6. Destroy group and storage pools.
             self._destroy_resource_group(self._linstor, self._group_name)
             self._destroy_resource_group(self._linstor, self._ha_group_name)
             for pool in self._get_storage_pools(force=True):
@@ -1381,8 +1394,9 @@ def destroy(self):
 
         try:
             self._start_controller(start=False)
-            for file in glob.glob(DATABASE_PATH + '/'):
-                os.remove(file)
+            for file in os.listdir(DATABASE_PATH):
+                if file != 'lost+found':
+                    os.remove(DATABASE_PATH + '/' + file)
         except Exception as e:
             util.SMlog(
                 'Ignoring failure after LINSTOR SR destruction: {}'