Merge branch 'autoupgrade-wait-changemaster' into 'dev'

AutoUpgrade - Change master modifications

See merge request dbdev/dbaas!8

Author: Marcelo Rodrigues Da Silva Soares

dbaas/drivers/base.py

@@ -253,6 +253,9 @@ def is_replication_ok(self, instance):
     def switch_master(self, instance=None, preferred_slave_instance=None):
         raise NotImplementedError()
 
+    def switch_master_with_stepdowntime(self, instance=None, preferred_slave_instance=None, stepdown_time=60):
+        raise NotImplementedError()
+
     def get_database_instances(self, ):
         driver_name = self.name.upper()
         instances = [instance
@@ -369,6 +372,35 @@ def check_replication_and_switch(self, instance, attempts=100,
             "Could not switch master because of replication's delay"
         )
 
+    def check_replication_and_switch_with_stepdown_time(self, instance, attempts=100,
+                                                        check_is_master_attempts=5,
+                                                        preferred_slave_instance=None,
+                                                        stepdown_time=60):
+        LOG.info("Check Replication with StepDown time of %s seconds", stepdown_time)
+        from time import sleep
+        for attempt in range(0, attempts):
+            if self.is_replication_ok(instance):
+                self.switch_master_with_stepdowntime(instance, preferred_slave_instance, stepdown_time)
+                LOG.info("Switch master returned ok...")
+
+                check_is_master_attempts_count = check_is_master_attempts
+                while self.check_instance_is_master(instance,
+                                                    default_timeout=False):
+                    if check_is_master_attempts_count == 0:
+                        break
+                    check_is_master_attempts_count -= 1
+                    sleep(10)
+                else:
+                    return
+
+                raise Exception("Could not change master")
+
+            LOG.info("Waiting 10s to check replication...")
+            sleep(10)
+        raise Exception(
+            "Could not switch master because of replication's delay"
+        )
+
     def get_database_agents(self):
         """ Returns database agents list"""
         raise NotImplementedError()

dbaas/drivers/mongodb.py

@@ -546,6 +546,16 @@ def switch_master(self, instance=None, preferred_slave_instance=None):
         except pymongo.errors.AutoReconnect:
             pass
 
+    def switch_master_with_stepdowntime(self, instance=None, preferred_slave_instance=None, stepdown_time=60):
+        client = self.get_client(None)
+        try:
+            client.admin.command(
+                'replSetStepDown', stepdown_time,
+                secondaryCatchUpPeriodSecs=60
+            )
+        except pymongo.errors.AutoReconnect:
+            pass
+
     def get_database_agents(self):
         return []
 

dbaas/workflow/steps/util/vm.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+import logging
 from time import sleep
 from dbaas_credentials.models import CredentialType
 from util import get_credentials_for
@@ -7,6 +8,8 @@
 CHANGE_MASTER_ATTEMPS = 30
 CHANGE_MASTER_SECONDS = 15
 
+LOG = logging.getLogger(__name__)
+
 
 class HostStatus(object):
     @staticmethod
@@ -159,33 +162,47 @@ class ChangeMasterTemporaryInstance(ChangeMaster):
 
     @property
     def is_valid(self):
-        if self.instance.temporary or self.check_master_is_temporary():
+        master_temporary = self.check_master_is_temporary()
+        # so executa para  a VM tepmoraria, e se a  Master nao eh temporaria
+        if not self.instance.temporary or master_temporary:
             return False
-        return super(ChangeMasterTemporaryInstance, self).is_valid
 
-    def check_master_is_temporary(self):
+        return True
+
+    def check_master_is_temporary(self, wait_seconds=0):
+        LOG.info("Checking master is temporary instance")
+        LOG.debug("Willl sleep for %s seconds before checking", wait_seconds)
+        sleep(wait_seconds)
+
         master = self.driver.get_master_instance()
-        if master.temporary:
-            return True
-        return False
+        LOG.info("Master instance is %s", master)
+        LOG.info("Master is temporary? %s", master.temporary)
+
+        if master is None or not master.temporary:
+            return False
+
+        return True
 
     def change_master(self):
         error = None
 
         for _ in range(CHANGE_MASTER_ATTEMPS):
-            if self.is_slave:
-                return
+            error = None
             try:
-                self.driver.check_replication_and_switch(self.target_instance)
-                if not self.check_master_is_temporary():
+                LOG.info("Trying to change master. Attempt %s", _)
+                self.driver.check_replication_and_switch_with_stepdown_time(self.target_instance, stepdown_time=300)
+                master_is_temporary = self.check_master_is_temporary(wait_seconds=60)
+
+                if not master_is_temporary:
                     raise Exception('Master is not the temporary instance')
+
+                return
             except Exception as e:
                 error = e
                 sleep(CHANGE_MASTER_SECONDS)
-            else:
-                return
 
-        raise error
+        if error is not None:
+            raise error
 
     def do(self):
         if not self.is_valid:
@@ -206,19 +223,18 @@ def change_master(self):
         error = None
 
         for _ in range(CHANGE_MASTER_ATTEMPS):
-            if self.is_slave:
-                return
             try:
                 self.driver.check_replication_and_switch(self.target_instance)
-                if self.check_master_is_temporary():
+                if self.check_master_is_temporary(wait_seconds=60):
                     raise Exception('Master is the temporary instance')
+
+                return
             except Exception as e:
                 error = e
                 sleep(CHANGE_MASTER_SECONDS)
-            else:
-                return
 
-        raise error
+        if error is not None:
+            raise error
 
 
 class ChangeMasterDatabaseMigrate(ChangeMaster):