Merge branch 'dev' into 'master'

Dev > Master

See merge request dbdev/dbaas!9

Author: Marcelo Rodrigues Da Silva Soares

dbaas/api/task.py

@@ -160,6 +160,7 @@ class TaskAPI(viewsets.ReadOnlyModelViewSet):
         'maintenance.tasks.restart_database',
         'notification.tasks.change_database_persistence',
         'maintenance.tasks.task_upgrade_disk_type',
+        'maintenance.tasks.auto_upgrade_database_vm_offering',
     ]
 
     model = TaskHistory

dbaas/drivers/base.py

@@ -253,6 +253,9 @@ def is_replication_ok(self, instance):
     def switch_master(self, instance=None, preferred_slave_instance=None):
         raise NotImplementedError()
 
+    def switch_master_with_stepdowntime(self, instance=None, preferred_slave_instance=None, stepdown_time=60):
+        raise NotImplementedError()
+
     def get_database_instances(self, ):
         driver_name = self.name.upper()
         instances = [instance
@@ -369,6 +372,35 @@ def check_replication_and_switch(self, instance, attempts=100,
             "Could not switch master because of replication's delay"
         )
 
+    def check_replication_and_switch_with_stepdown_time(self, instance, attempts=100,
+                                                        check_is_master_attempts=5,
+                                                        preferred_slave_instance=None,
+                                                        stepdown_time=60):
+        LOG.info("Check Replication with StepDown time of %s seconds", stepdown_time)
+        from time import sleep
+        for attempt in range(0, attempts):
+            if self.is_replication_ok(instance):
+                self.switch_master_with_stepdowntime(instance, preferred_slave_instance, stepdown_time)
+                LOG.info("Switch master returned ok...")
+
+                check_is_master_attempts_count = check_is_master_attempts
+                while self.check_instance_is_master(instance,
+                                                    default_timeout=False):
+                    if check_is_master_attempts_count == 0:
+                        break
+                    check_is_master_attempts_count -= 1
+                    sleep(10)
+                else:
+                    return
+
+                raise Exception("Could not change master")
+
+            LOG.info("Waiting 10s to check replication...")
+            sleep(10)
+        raise Exception(
+            "Could not switch master because of replication's delay"
+        )
+
     def get_database_agents(self):
         """ Returns database agents list"""
         raise NotImplementedError()

dbaas/drivers/mongodb.py

@@ -546,6 +546,16 @@ def switch_master(self, instance=None, preferred_slave_instance=None):
         except pymongo.errors.AutoReconnect:
             pass
 
+    def switch_master_with_stepdowntime(self, instance=None, preferred_slave_instance=None, stepdown_time=60):
+        client = self.get_client(None)
+        try:
+            client.admin.command(
+                'replSetStepDown', stepdown_time,
+                secondaryCatchUpPeriodSecs=60
+            )
+        except pymongo.errors.AutoReconnect:
+            pass
+
     def get_database_agents(self):
         return []
 

dbaas/maintenance/task_auto_upgrade_vm_offering.py

@@ -6,7 +6,7 @@
 from physical.models import (Plan, DatabaseInfra, Instance, Pool)
 from util.providers import get_auto_upgrade_vm_settings
 from workflow.workflow import steps_for_instances
-from util import get_vm_name
+from util import get_vm_name, email_notifications
 
 LOG = logging.getLogger(__name__)
 
@@ -72,6 +72,8 @@ def task_auto_upgrade_vm_offering(database, task, retry_from=None, resize_target
 
         if create_temporary_instance:  # se precisar criar nova temporary instance, cria
             LOG.info("Creating temporary instance")
+            email_notifications.upgrade_offering_notification(database, resize_target)
+
             for i in range(number_of_instances):
                 instance = None
                 last_vm_created += 1

dbaas/notification/templates/email_extras/auto_upgrade_offering_notification.html

@@ -0,0 +1,20 @@
+<h2>Automatic GCP offering upgrade for Database {{ database.name }}</h2>
+
+We would like to inform you that an automatic offer update is being carried out on DBaaS to meet your database <a href="{{database_url}}">{{database.name}}</a> needs.
+<br><br>
+The update will be done automatically from {{ current_offering.name }} to {{ future_offering.name }}.
+<br><br>
+Please be aware that this upgrade may increase the costs.
+Any questions please contact the DBDevops responsible for your team.
+<br><br>
+
+{% if database.team.email %}
+You are receiving this email because in our records you are in team {{ database.team.name }}.<br>
+If this is not right, contact the DBaaS system administrators.
+{% else %}
+<h3>Team {{ database.team.name }} has no email set!</h3>
+{% endif %}
+<br><br><br>
+Regards,<br>
+DBaaS notification robot<br>
+{{domain}}<br>

dbaas/notification/templates/email_extras/auto_upgrade_offering_notification.txt

@@ -0,0 +1,19 @@
+Automatic GCP offering upgrade for Database {{ database.name }}
+
+We would like to inform you that an automatic offer update is being carried out on DBaaS to meet your database {{database.name}} needs.
+
+The update will be done automatically from {{ current_offering.name }} to {{ future_offering.name }}.
+
+Please be aware that this upgrade may increase the costs.
+Any questions please contact the DBDevops responsible for your team.
+
+{% if database.team.email %}
+You are receiving this email because in our records you are in team {{ database.team.name }}.<br>
+If this is not right, contact the DBaaS system administrators.
+{% else %}
+Team {{ database.team.name }} has no email set!
+{% endif %}
+
+Regards,
+DBaaS notification robot
+{{domain}}

dbaas/util/email_notifications.py

@@ -157,8 +157,30 @@ def disk_resize_notification(database, new_disk, usage_percentage):
     )
 
 
-def schedule_task_notification(database, scheduled_task, is_new,
-                               is_task_warning=False):
+def upgrade_offering_notification(database, resize_target):
+    LOG.info('Notifying auto upgrade offering to database: {}'.format(database))
+
+    current_offering = database.databaseinfra.offering
+    future_offering = database.get_future_offering(resize_target)
+
+    subject = _('[DBaaS] Database {} auto upgrade offering to {}').format(database, future_offering.name)
+    template = "auto_upgrade_offering_notification"
+
+    context = {
+        'domain': get_domain(),
+        'database': database,
+        'current_offering': current_offering,
+        'future_offering': future_offering,
+        'database_url': get_database_url(database.id)
+    }
+
+    send_mail_template(
+        subject, template, email_from(), email_to(database.team),
+        fail_silently=False, attachments=None, context=context
+    )
+
+
+def schedule_task_notification(database, scheduled_task, is_new, is_task_warning=False):
 
     subject_tmpl = '[DBaaS] Automatic Task {} for Database {}'
     if is_task_warning:

dbaas/workflow/steps/util/vm.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+import logging
 from time import sleep
 from dbaas_credentials.models import CredentialType
 from util import get_credentials_for
@@ -7,6 +8,8 @@
 CHANGE_MASTER_ATTEMPS = 30
 CHANGE_MASTER_SECONDS = 15
 
+LOG = logging.getLogger(__name__)
+
 
 class HostStatus(object):
     @staticmethod
@@ -159,33 +162,47 @@ class ChangeMasterTemporaryInstance(ChangeMaster):
 
     @property
     def is_valid(self):
-        if self.instance.temporary or self.check_master_is_temporary():
+        master_temporary = self.check_master_is_temporary()
+        # so executa para  a VM tepmoraria, e se a  Master nao eh temporaria
+        if not self.instance.temporary or master_temporary:
             return False
-        return super(ChangeMasterTemporaryInstance, self).is_valid
 
-    def check_master_is_temporary(self):
+        return True
+
+    def check_master_is_temporary(self, wait_seconds=0):
+        LOG.info("Checking master is temporary instance")
+        LOG.debug("Willl sleep for %s seconds before checking", wait_seconds)
+        sleep(wait_seconds)
+
         master = self.driver.get_master_instance()
-        if master.temporary:
-            return True
-        return False
+        LOG.info("Master instance is %s", master)
+        LOG.info("Master is temporary? %s", master.temporary)
+
+        if master is None or not master.temporary:
+            return False
+
+        return True
 
     def change_master(self):
         error = None
 
         for _ in range(CHANGE_MASTER_ATTEMPS):
-            if self.is_slave:
-                return
+            error = None
             try:
-                self.driver.check_replication_and_switch(self.target_instance)
-                if not self.check_master_is_temporary():
+                LOG.info("Trying to change master. Attempt %s", _)
+                self.driver.check_replication_and_switch_with_stepdown_time(self.target_instance, stepdown_time=300)
+                master_is_temporary = self.check_master_is_temporary(wait_seconds=60)
+
+                if not master_is_temporary:
                     raise Exception('Master is not the temporary instance')
+
+                return
             except Exception as e:
                 error = e
                 sleep(CHANGE_MASTER_SECONDS)
-            else:
-                return
 
-        raise error
+        if error is not None:
+            raise error
 
     def do(self):
         if not self.is_valid:
@@ -206,19 +223,18 @@ def change_master(self):
         error = None
 
         for _ in range(CHANGE_MASTER_ATTEMPS):
-            if self.is_slave:
-                return
             try:
                 self.driver.check_replication_and_switch(self.target_instance)
-                if self.check_master_is_temporary():
+                if self.check_master_is_temporary(wait_seconds=60):
                     raise Exception('Master is the temporary instance')
+
+                return
             except Exception as e:
                 error = e
                 sleep(CHANGE_MASTER_SECONDS)
-            else:
-                return
 
-        raise error
+        if error is not None:
+            raise error
 
 
 class ChangeMasterDatabaseMigrate(ChangeMaster):

dbaas/workflow/steps/util/zabbix.py

@@ -334,12 +334,22 @@ def __unicode__(self):
     def do(self):
         if not self.is_valid:
             return
-        self.zabbix_provider.disable_alarms()
+        # Try and Exception to fix error Session terminated
+        try:
+            self.zabbix_provider.disable_alarms()
+        except:
+            self.provider_write = None
+            self.zabbix_provider.disable_alarms()
 
     def undo(self):
         if not self.is_valid:
             return
-        self.zabbix_provider.enable_alarms()
+        # Try and Exception to fix error Session terminated
+        try:
+            self.zabbix_provider.enable_alarms()
+        except:
+            self.provider_write = None
+            self.zabbix_provider.enable_alarms()
 
 
 class EnableAlarms(ZabbixStep):
@@ -354,11 +364,22 @@ def is_valid(self):
     def do(self):
         if not self.is_valid:
             return
-        
-        self.zabbix_provider.enable_alarms()
+        # Try and Exception to fix error Session terminated
+        try:
+            self.zabbix_provider.enable_alarms()
+        except:
+            self.provider_write = None
+            self.zabbix_provider.enable_alarms()
 
     def undo(self):
-        self.zabbix_provider.disable_alarms()
+        if not self.is_valid:
+            return
+        # Try and Exception to fix error Session terminated
+        try:
+            self.zabbix_provider.disable_alarms()
+        except:
+            self.provider_write = None
+            self.zabbix_provider.disable_alarms()
 
 
 class UpdateMonitoring(ZabbixStep):
@@ -422,4 +443,4 @@ def do(self):
             host_name=self.instance.dns,
             macro='{$MONGO_SSL}',
             value=value
-        )
+        )