Merge branch 'master' of github.com:globocom/database-as-a-service

Author: Marcelo RS Soares

dbaas/backup/tasks.py

@@ -86,12 +86,12 @@ def unlock_instance(driver, instance, client):
         return False
 
 
-def make_instance_snapshot_backup(instance, error, group,
-                                  provider_class=VolumeProviderSnapshot,
-                                  target_volume=None,
-                                  current_hour=None,
-                                  task=None,
-                                  persist=0):
+def make_instance_dccm_snapshot_backup(instance, error, group,
+                                       provider_class=VolumeProviderSnapshot,
+                                       target_volume=None,
+                                       current_hour=None,
+                                       task=None,
+                                       persist=0):
     LOG.info("Make instance backup for {}".format(instance))
     provider = provider_class(instance)
     infra = instance.databaseinfra
@@ -212,6 +212,169 @@ def make_instance_snapshot_backup(instance, error, group,
     return snapshot
 
 
+def make_instance_gcp_snapshot_backup(
+    instance, error, group, provider_class=VolumeProviderSnapshot, target_volume=None,
+    current_hour=None, task=None, persist=0
+):
+    LOG.info("Make instance backup for {}".format(instance))
+    provider = provider_class(instance)
+    infra = instance.databaseinfra
+    database = infra.databases.first()
+
+    backup_retry_attempts = Configuration.get_by_name_as_int('backup_retry_attempts', default=3)
+
+    snapshot = Snapshot.create(
+        instance, group, target_volume or provider.volume,
+        environment=provider.environment, persistent=True if persist != 0 else False
+    )
+
+    snapshot_final_status = Snapshot.SUCCESS
+
+    locked = None
+    client = None
+    driver = infra.get_driver()
+    try:
+        client = driver.get_client(instance)
+        locked = lock_instance(driver, instance, client)
+        if not locked:
+            snapshot_final_status = Snapshot.WARNING
+
+        if 'MySQL' in type(driver).__name__:
+            mysql_binlog_save(client, instance)
+
+        has_snapshot = Snapshot.objects.filter(
+            status=Snapshot.WARNING, instance=instance, end_at__year=datetime.now().year,
+            end_at__month=datetime.now().month, end_at__day=datetime.now().day
+        )
+        backup_hour_list = Configuration.get_by_name_as_list('make_database_backup_hour')
+        if not snapshot_final_status == Snapshot.WARNING and not has_snapshot:
+            cont = 0
+            for _ in range(backup_retry_attempts):
+                cont += 1
+                try:
+                    code = 201
+                    response, data = provider.new_take_snapshot(persist=persist)
+
+                    if response.status_code < 400:
+                        break
+
+                    if cont >= 3:
+                        raise IndexError
+
+                except IndexError as e:
+                    response, content = e
+                    if response.status_code == 503:
+                        errormsg = "{} - 503 error creating snapshot for instance: {}. It will try again in 30 seconds. ".format(
+                            strftime("%d/%m/%Y %H:%M:%S"), instance
+                        )
+                        LOG.error(errormsg)
+                        if task:
+                            task.add_detail(errormsg)
+                        sleep(30)
+                    else:
+                        raise e
+
+            if response.status_code < 400:
+                while code != 200:
+                    sleep(20)
+                    snap_response, snap_status = provider.take_snapshot_status(data['identifier'])
+                    if snap_response.status_code in [200, 202]:
+                        unlock_instance(driver, instance, client)
+                    if snap_response.status_code == 200:
+                        break
+                    if snap_response.status_code >= 400:
+                        raise error
+                    code = snap_response.status_code
+
+                snapshot.done(snap_status)
+                snapshot.save()
+            else:
+                errormsg = response['message']
+                set_backup_error(infra, snapshot, errormsg)
+        else:
+            if str(current_hour) in backup_hour_list:
+                raise Exception("Backup with WARNING already created today.")
+
+    except Exception as e:
+        errormsg = "Error creating snapshot: {}".format(e)
+        error['errormsg'] = errormsg
+        set_backup_error(infra, snapshot, errormsg)
+        return snapshot
+    finally:
+        unlock_instance(driver, instance, client)
+
+    if not snapshot.size:
+        command = "du -sb /data/.snapshot/%s | awk '{print $1}'" % (
+            snapshot.snapshot_name
+        )
+        try:
+            output = instance.hostname.ssh.run_script(command)
+            size = int(output['stdout'][0])
+            snapshot.size = size
+        except Exception as e:
+            snapshot.size = 0
+            LOG.error("Error exec remote command {}".format(e))
+
+    backup_path = database.backup_path
+    if backup_path:
+        now = datetime.now()
+        target_path = "{}/{}/{}/{}/{}".format(
+            backup_path,
+            now.strftime("%Y_%m_%d"),
+            instance.hostname.hostname.split('.')[0],
+            now.strftime("%Y%m%d%H%M%S"),
+            infra.name
+        )
+        snapshot_path = "/data/.snapshot/{}/data/".format(
+            snapshot.snapshot_name
+        )
+        command = """
+        if [ -d "{backup_path}" ]
+        then
+            rm -rf {backup_path}/20[0-9][0-9]_[0-1][0-9]_[0-3][0-9] &
+            mkdir -p {target_path}
+            cp -r {snapshot_path} {target_path} &
+        fi
+        """.format(backup_path=backup_path,
+                   target_path=target_path,
+                   snapshot_path=snapshot_path)
+        try:
+            instance.hostname.ssh.run_script(command)
+        except Exception as e:
+            LOG.error("Error exec remote command {}".format(e))
+
+    snapshot.status = snapshot_final_status
+    snapshot.end_at = datetime.now()
+    snapshot.save()
+    register_backup_dbmonitor(infra, snapshot)
+
+    return snapshot
+
+
+def make_instance_snapshot_backup(instance, error, group,
+                                  provider_class=VolumeProviderSnapshot,
+                                  target_volume=None,
+                                  current_hour=None,
+                                  task=None,
+                                  persist=0):
+    infra = instance.databaseinfra
+    env = infra.environment
+    if env.name == 'prod':
+        return make_instance_dccm_snapshot_backup(instance, error, group,
+                                                  provider_class=provider_class,
+                                                  target_volume=target_volume,
+                                                  current_hour=current_hour,
+                                                  task=task,
+                                                  persist=persist)
+    else:
+        return make_instance_gcp_snapshot_backup(instance, error, group,
+                                                 provider_class=provider_class,
+                                                 target_volume=target_volume,
+                                                 current_hour=current_hour,
+                                                 task=task,
+                                                 persist=persist)
+
+
 def make_instance_snapshot_backup_upgrade_disk(instance, error, group, provider_class=VolumeProviderSnapshot,
                                                target_volume=None,
                                                current_hour=None):
@@ -742,6 +905,7 @@ def _create_database_backup(instance, task, group, current_hour, persist):
 
     error = {}
     try:
+        LOG.info('Starting make database snapshot')
         snapshot = make_instance_snapshot_backup(
             instance=instance,
             error=error,

dbaas/dbaas_services/analyzing/views.py

@@ -20,6 +20,9 @@
 import logging
 
 
+LOG = logging.getLogger(__name__)
+
+
 class SubUsedResourceReport(ListView):
     def get(self, request, *args, **kwargs):
         reports = (models.AnalyzeRepository.objects.all()
@@ -87,18 +90,18 @@ def vm_by_line_database_report(self):
                     database.attention_description,
                     instance.hostname.hostname.encode("utf-8"),
                     database.environment,
-                    database.team,
-                    database.team.name,
-                    database.team.team_area,
-                    database.team.email,
-                    database.team.contacts,
-                    database.team.organization.name,
+                    self._check_values(database, 'team'),
+                    self._check_values(database, 'team_name'),
+                    self._check_values(database, 'team_area'),
+                    self._check_values(database, 'team_email'),
+                    self._check_values(database, 'team_contacts'),
+                    self._check_values(database, 'team_organization'),
                     database.created_at,
                     database.is_in_quarantine,
                     database.apps_bind_name,
-                    database.infra.offering.cpus,
-                    database.infra.offering.memory_size_mb,
-                    database.infra.disk_offering.size_gb(),
+                    self._check_values(database, 'cpu'),
+                    self._check_values(database, 'memory_size'),
+                    self._check_values(database, 'disk_size'),
                     database.engine_type
                 ]
                 writer.writerow(data)
@@ -131,20 +134,75 @@ def default_database_report(self):
                 database.attention_description,
                 hostname,
                 database.environment,
-                database.team,
-                database.team.name,
-                database.team.team_area,
-                database.team.email,
-                database.team.contacts,
-                database.team.organization.name,
+                self._check_values(database, 'team'),
+                self._check_values(database, 'team_name'),
+                self._check_values(database, 'team_area'),
+                self._check_values(database, 'team_email'),
+                self._check_values(database, 'team_contacts'),
+                self._check_values(database, 'team_organization'),
                 database.created_at,
                 database.is_in_quarantine,
                 database.apps_bind_name,
-                database.infra.offering.cpus,
-                database.infra.offering.memory_size_mb,
-                database.infra.disk_offering.size_gb(),
+                self._check_values(database, 'cpu'),
+                self._check_values(database, 'memory_size'),
+                self._check_values(database, 'disk_size'),
                 database.engine_type
             ]
             writer.writerow(data)
 
-        return response
+        return response
+
+    def _check_values(self, database, attr):
+        if attr == 'team':
+            try:
+                return database.team
+            except:
+                return ''
+
+        if attr == 'team_name':
+            try:
+                return database.team.name
+            except:
+                return ''
+
+        if attr == 'team_area':
+            try:
+                return database.team.team_area
+            except:
+                return ''
+
+        if attr == 'team_email':
+            try:
+                return database.team.email
+            except:
+                return ''
+
+        if attr == 'team_contacts':
+            try:
+                return database.team.contacts
+            except:
+                return ''
+
+        if attr == 'team_organization':
+            try:
+                return database.team.organization.name
+            except:
+                return ''
+
+        if attr == 'cpu':
+            try:
+                return database.infra.offering.cpus
+            except:
+                return 0
+
+        if attr == 'memory_size':
+            try:
+                return database.infra.offering.memory_size_mb
+            except:
+                return 0
+
+        if attr == 'disk_size':
+            try:
+                return database.infra.disk_offering.size_gb()
+            except:
+                return 0

dbaas/drivers/replication_topologies/base.py

@@ -732,6 +732,9 @@ def get_stop_database_vm_steps(self):
                 'workflow.steps.util.host_provider.StopIfRunning',
             )
         }]
+    
+    def get_auto_upgrade_database_vm_offering(self):
+        raise NotImplementedError('Not implemented for topology')
 
     def get_start_database_vm_steps(self):
         return [{

dbaas/drivers/replication_topologies/mysql.py

@@ -1065,22 +1065,30 @@ def get_update_ssl_steps(self):
             'Disable monitoring and alarms': (
                 'workflow.steps.util.zabbix.DisableAlarms',
                 'workflow.steps.util.db_monitor.DisableMonitoring',
-                'workflow.steps.util.ssl.UpdateExpireAtDateRollback',
-                'workflow.steps.util.ssl.BackupSSLFolder',
             ),
         }] + [{
             'Disable SSL': (
                 'workflow.steps.util.ssl.UnSetReplicationUserRequireSSL',
             ),
         }] + [{
             'Configure SSL': (
-                'workflow.steps.util.ssl.UpdateSSLForInfra',
-                'workflow.steps.util.ssl.UpdateSSLForInstance',
+                'workflow.steps.util.ssl.UpdateExpireAtDateRollback',
+                'workflow.steps.util.ssl.MoveSSLFolder',
+                'workflow.steps.util.ssl.UpdateOpenSSlLib',
+                'workflow.steps.util.ssl.CreateSSLFolder',
+                'workflow.steps.util.ssl.CreateSSLConfForInfraEndPoint',
+                'workflow.steps.util.ssl.CreateSSLConfForInstanceIP',
+                'workflow.steps.util.ssl.RequestSSLForInfra',
+                'workflow.steps.util.ssl.RequestSSLForInstance',
                 'workflow.steps.util.ssl.CreateJsonRequestFileInfra',
                 'workflow.steps.util.ssl.CreateJsonRequestFileInstance',
                 'workflow.steps.util.ssl.CreateCertificateInfra',
                 'workflow.steps.util.ssl.CreateCertificateInstance',
                 'workflow.steps.util.ssl.SetSSLFilesAccessMySQL',
+                'workflow.steps.util.ssl.SetInfraConfiguredSSL',
+                'workflow.steps.util.plan.Configure',
+                'workflow.steps.util.plan.ConfigureLog',
+                'workflow.steps.util.metric_collector.ConfigureTelegraf',
                 'workflow.steps.util.ssl.UpdateExpireAtDate',
             ),
         }] + [{
@@ -1089,7 +1097,7 @@ def get_update_ssl_steps(self):
                 'workflow.steps.util.vm.ChangeMaster',
                 'workflow.steps.util.database.CheckIfSwitchMaster',
                 'workflow.steps.util.database.Stop',
-                'workflow.steps.util.ssl.RestoreSSLFolder4Rollback',
+                # 'workflow.steps.util.ssl.RestoreSSLFolder4Rollback',
                 'workflow.steps.util.database.Start',
                 'workflow.steps.util.metric_collector.RestartTelegraf',
                 'workflow.steps.util.database.CheckIfSwitchMasterRollback',
@@ -1103,6 +1111,7 @@ def get_update_ssl_steps(self):
             ),
         }] + [{
             'Enabling monitoring and alarms': (
+                'workflow.steps.util.db_monitor.UpdateInfraSSLMonitor',
                 'workflow.steps.util.db_monitor.EnableMonitoring',
                 'workflow.steps.util.zabbix.EnableAlarms',
             ),