Merge pull request #86 from DeliZhangX/private/deliz/CP-32847

CP-32847: Support manually block/unblock path

Author: Deli Zhang

src/XenCert/StorageHandler.py

@@ -21,6 +21,7 @@
 import commands
 import glob
 import random
+import operator
 from xml.dom import minidom
 import StorageHandlerUtil
 from XenCertLog import Print, PrintOnSameLine, XenCertPrint
@@ -88,11 +89,12 @@ def run(self):
                 XenCertPrint("Could not write through the allocated disk space on test disk, please check the storage configuration manually. Exception: %s" % str(e))
 
 class WaitForFailover(Thread):
-    def __init__(self, session, scsiid, activePaths, noOfPaths):
+    def __init__(self, session, scsiid, activePaths, noOfPaths, checkFunc):
         Thread.__init__(self)        
         self.scsiid = scsiid
         self.activePaths = activePaths
         self.noOfPaths = noOfPaths
+        self.checkFunc = checkFunc
 
     def run(self):
         # Here wait for the expected number of paths to fail.        
@@ -104,7 +106,8 @@ def run(self):
         while not pathsFailed and failoverTime < 50:
             try:
                 (retVal, listPathConfigNew) = StorageHandlerUtil.get_path_status(self.scsiid, True)
-                if self.noOfPaths == ((int)(self.activePaths) - len(listPathConfigNew)):
+                currNoOfPaths = (int)(self.activePaths) - len(listPathConfigNew)
+                if self.checkFunc(currNoOfPaths, self.noOfPaths):
                     pathsFailed = True
                 time.sleep(1)
                 failoverTime += 1                
@@ -245,8 +248,9 @@ def MPConfigVerificationTests(self):
 
             if not os.path.exists(self.storage_conf['pathHandlerUtil']):                 
                 raise Exception("Path handler util specified for multipathing tests does not exist!")
-            
-            if self.storage_conf['storage_type'] == 'hba' and self.storage_conf['pathInfo'] is None:
+
+            isManBlock = os.path.basename(self.storage_conf['pathHandlerUtil']) == "blockunblockpaths"
+            if self.storage_conf['storage_type'] == 'hba' and self.storage_conf['pathInfo'] is None and not isManBlock:
                 raise Exception("Path related information not specified for storage type hba.")
 
             if self.storage_conf['count'] is not None:
@@ -306,7 +310,7 @@ def MPConfigVerificationTests(self):
             global speedOfCopy
             Print("")
             Print("Iteration 1:\n")
-            Print(" -> No manual blocking of paths.")
+            Print(" -> No manual/script blocking of paths.")
             s = TimedDeviceIO(self.session.xenapi.VBD.get_device(vbd_ref))
             s.start()
             s.join()
@@ -325,29 +329,37 @@ def MPConfigVerificationTests(self):
                 displayOperationStatus(True)
                 checkPoint += 1
 
-            if len(self.listPathConfig) > 1:                
+            if len(self.listPathConfig) > 1:
                 for i in range(2, iterationCount):
                     maxTimeTaken = 0
                     throughputForMaxTime = ''
                     totalCheckPoints += 2
-                    Print("Iteration %d:\n" % i)                                    
-                    if not self.RandomlyFailPaths():                                            
-                        raise Exception("Failed to block paths.")
-                    
-                    XenCertPrint("Dev Path Config = '%s', no of Blocked switch Paths = '%s'" % (self.listPathConfig, self.noOfPaths))
-
-                    # Fail path calculation needs to be done only in case of hba SRs
-                    if "blockunblockhbapaths" in \
-                            self.storage_conf['pathHandlerUtil'].split('/')[-1]:
-                        #Calculate the number of devices to be found after the path block
-                        devicesToFail = (len(self.listPathConfig)/self.noOfTotalPaths) * self.noOfPaths
-                        XenCertPrint("Expected devices to fail: %s" % devicesToFail)
+                    Print("Iteration %d:\n" % i)
+
+                    if isManBlock:
+                        Print(" -> Wait for manually blocking paths")
+                        self.WaitManualBlockUnblockPaths()
+                        devicesToFail = 1
+                        checkFunc = operator.ge
                     else:
-                        devicesToFail = self.noOfPaths
+                        if not self.RandomlyFailPaths():
+                            raise Exception("Failed to block paths.")
+
+                        XenCertPrint("Dev Path Config = '%s', no of Blocked switch Paths = '%s'" % (self.listPathConfig, self.noOfPaths))
+
+                        # Fail path calculation needs to be done only in case of hba SRs
+                        if "blockunblockhbapaths" in \
+                                self.storage_conf['pathHandlerUtil'].split('/')[-1]:
+                            #Calculate the number of devices to be found after the path block
+                            devicesToFail = (len(self.listPathConfig)/self.noOfTotalPaths) * self.noOfPaths
+                            XenCertPrint("Expected devices to fail: %s" % devicesToFail)
+                        else:
+                            devicesToFail = self.noOfPaths
+                        checkFunc = operator.eq
 
-                    s = WaitForFailover(self.session, device_config['SCSIid'], len(self.listPathConfig), devicesToFail)
+                    s = WaitForFailover(self.session, device_config['SCSIid'], len(self.listPathConfig), devicesToFail, checkFunc)
                     s.start()
-                    
+
                     while s.isAlive():
                         timeTaken = 0
                         s1 = TimedDeviceIO(self.session.xenapi.VBD.get_device(vbd_ref))                    
@@ -363,19 +375,25 @@ def MPConfigVerificationTests(self):
                         if timeTaken > maxTimeTaken:
                             maxTimeTaken = timeTaken
                             throughputForMaxTime = speedOfCopy
-                    
+
                     if pathsFailed:
                         Print("    - Paths failover time: %s seconds" % failoverTime)
                         Print("    - Maximum IO completion time: %s. Data: %s. Throughput: %s" % (maxTimeTaken, '1MB', throughputForMaxTime))
                         displayOperationStatus(True)
                         checkPoint += 1
                     else:
                         displayOperationStatus(False)
-                        self.BlockUnblockPaths(False, self.storage_conf['pathHandlerUtil'], self.noOfPaths, self.blockedpathinfo)
+                        if not isManBlock:
+                            self.BlockUnblockPaths(False, self.storage_conf['pathHandlerUtil'], self.noOfPaths, self.blockedpathinfo)
                         raise Exception("    - Paths did not failover within expected time.")
-                    
-                    self.BlockUnblockPaths(False, self.storage_conf['pathHandlerUtil'], self.noOfPaths, self.blockedpathinfo)
-                    Print(" -> Unblocking paths, waiting for restoration.")
+
+                    if isManBlock:
+                        Print(" -> Wait for manually unblocking paths and restoration")
+                        self.WaitManualBlockUnblockPaths()
+                    else:
+                        self.BlockUnblockPaths(False, self.storage_conf['pathHandlerUtil'], self.noOfPaths, self.blockedpathinfo)
+                        Print(" -> Unblocking paths, waiting for restoration.")
+
                     count = 0
                     pathsMatch = False
                     while not pathsMatch and count < 120:
@@ -652,7 +670,19 @@ def BlockUnblockPaths(self, blockOrUnblock, script, noOfPaths, passthrough):
                 raise Exception("   - The path block/unblock utility returned an error: %s." % stderr)
             return stdout
         except Exception, e:            
-            raise e        
+            raise e
+
+    def WaitManualBlockUnblockPaths(self):
+        try:
+            cmd = [self.storage_conf['pathHandlerUtil']]
+            (rc, stdout, stderr) = util.doexec(cmd, '')
+            XenCertPrint(
+                "The path manually block/unblock utility returned rc: %s stdout: '%s', stderr: '%s'" % (rc, stdout, stderr))
+            if rc != 0:
+                raise Exception("   - The path manually block/unblock utility returned an error: %s." % stderr)
+            return stdout
+        except Exception, e:
+            raise e
 
     def __del__(self):
         XenCertPrint("Reached Storagehandler destructor")

src/XenCert/blockunblockhbapaths-brocade

@@ -42,7 +42,7 @@ def blockUnblockPort(blockOrUnblock, ip, username, password, port):
         XenCertPrint("calling %s" % ' '.join(XenCertCommon.getCmdsWithHiddenPassword(cmd)))
         util.doexec(cmd, '')
     except Exception, e:
-        util.SMlog("There was an exception in blocking port: %s" % port)
+        util.SMlog("There was an exception in blocking port: %s, exception: %s" % (port, str(e)))
 
 # Test Cmdline args
 if len(sys.argv) != 4:

src/XenCert/blockunblockhbapaths-cisco

@@ -42,7 +42,7 @@ def blockUnblockPort(blockOrUnblock, ip, username, password, port):
         XenCertPrint("calling %s" % ' '.join(XenCertCommon.getCmdsWithHiddenPassword(cmd)))
         util.doexec(cmd, '')
     except Exception, e:
-        util.SMlog("There was an exception in blocking port: %s" % port)
+        util.SMlog("There was an exception in blocking port: %s, exception: %s" % (port, str(e)))
 
 # Test Cmdline args
 if len(sys.argv) != 4:

src/XenCert/blockunblockhbapaths-qlogic

@@ -42,7 +42,7 @@ def blockUnblockPort(blockOrUnblock, ip, username, password, port):
         XenCertPrint("calling %s" % ' '.join(XenCertCommon.getCmdsWithHiddenPassword(cmd)))
         util.doexec(cmd, '')
     except Exception, e:
-        util.SMlog("There was an exception in blocking port: %s" % port)
+        util.SMlog("There was an exception in blocking port: %s, exception: %s" % (port, str(e)))
 
 # Test Cmdline args
 if len(sys.argv) != 4:

src/XenCert/blockunblockpaths

@@ -48,17 +48,14 @@ if len(sys.argv) == 5:
     cmd = [sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]]
     XenCertPrint('blockunblockpaths - now call %s and wait for block/unblock to finish.' % cmd)
     (rc, stdout, stderr) = util.doexec(cmd, '')
-    if rc == 0:
-	retVal = stdout
-    else:
-	retVal = stderr
+    retVal = stdout if rc == 0 else stderr
 elif len(sys.argv) == 1:
     XenCertPrint('blockunblockpaths - called without any arguments, just wait for block/unblock to finish.')
-    
+
 while block_unblock_over != '1':
     time.sleep(1)
     block_unblock_over = xs_handle.read('', '/xencert/block-unblock-over')
-    
+
 os.system('xenstore-rm /xencert')
 del xs_handle
 sys.stdout.write(retVal)