Backup fixes

Author: GitTimeraider

app.py

@@ -24,8 +24,11 @@
 )
 logger = logging.getLogger(__name__)
 
-# Set APScheduler logging level to DEBUG for better debugging
-logging.getLogger('apscheduler').setLevel(logging.DEBUG)
+# Set APScheduler logging level to WARNING to reduce noise
+logging.getLogger('apscheduler').setLevel(logging.WARNING)
+
+# Set timezone
+LOCAL_TZ = pytz.timezone(os.environ.get('TZ', 'UTC'))
 
 app = Flask(__name__)
 app.config['SECRET_KEY'] = os.environ.get('SECRET_KEY', 'dev-secret-key-change-in-production')
@@ -138,8 +141,28 @@ def format_local_time_filter(utc_dt, format_str='%Y-%m-%d %H:%M'):
 # Initialize backup service
 backup_service = BackupService()
 
-# Initialize scheduler
-scheduler = BackgroundScheduler()
+# Initialize scheduler with job store to prevent duplicates
+from apscheduler.jobstores.memory import MemoryJobStore
+from apscheduler.executors.pool import ThreadPoolExecutor
+
+jobstores = {
+    'default': MemoryJobStore()
+}
+executors = {
+    'default': ThreadPoolExecutor(max_workers=2)  # Limit concurrent backups
+}
+job_defaults = {
+    'coalesce': True,  # Combine multiple pending executions of the same job
+    'max_instances': 1,  # Only one instance of a job can run at a time
+    'misfire_grace_time': 60  # 1 minute grace time for missed jobs
+}
+
+scheduler = BackgroundScheduler(
+    jobstores=jobstores,
+    executors=executors,
+    job_defaults=job_defaults,
+    timezone=LOCAL_TZ
+)
 scheduler.start()
 atexit.register(lambda: scheduler.shutdown())
 
@@ -579,10 +602,21 @@ def backup_with_context():
             try:
                 # Refresh the repository object to ensure it's bound to the current session
                 repo = Repository.query.get(repository.id)
-                if repo and repo.is_active:
-                    backup_service.backup_repository(repo)
-                else:
+                if not repo or not repo.is_active:
                     logger.warning(f"Repository {repository.id} not found or inactive, skipping backup")
+                    return
+                
+                # Check if there's already a running backup for this repository
+                running_job = BackupJob.query.filter_by(
+                    repository_id=repository.id, 
+                    status='running'
+                ).first()
+                
+                if running_job:
+                    logger.warning(f"Backup already running for repository {repo.name}, skipping")
+                    return
+                
+                backup_service.backup_repository(repo)
             except Exception as e:
                 logger.error(f"Error in scheduled backup for repository {repository.id}: {e}")
 
@@ -654,12 +688,17 @@ def backup_with_context():
         return  # Manual only
 
     scheduler.add_job(
-        func=backup_with_context,  # Use the wrapper function instead
+        func=backup_with_context,
         trigger=trigger,
         id=job_id,
         name=f'Backup {repository.name}',
+        replace_existing=True,
+        misfire_grace_time=300,  # 5 minutes grace time
+        coalesce=True,  # Combine multiple pending executions
         replace_existing=True
     )
+    
+    logger.info(f"Scheduled backup job for {repository.name} with trigger: {trigger}")
 
 if __name__ == '__main__':
     app.run(host='0.0.0.0', port=8080, debug=False)

backup_service.py

@@ -31,6 +31,7 @@ def backup_repository(self, repository):
         db.session.add(backup_job)
         db.session.commit()
 
+        temp_clone_dir = None
         try:
             # Create user-specific backup directory
             user_backup_dir = self.backup_base_dir / f"user_{repository.user_id}"
@@ -44,39 +45,39 @@ def backup_repository(self, repository):
             timestamp = datetime.utcnow().strftime('%Y%m%d_%H%M%S')
             backup_name = f"{repository.name}_{timestamp}"
 
-            # Clone repository to temporary location
+            # Create unique temporary directory and ensure it's clean
             temp_clone_dir = repo_backup_dir / f"temp_{timestamp}"
+            
+            # Remove temp directory if it already exists
+            if temp_clone_dir.exists():
+                logger.warning(f"Temp directory already exists, removing: {temp_clone_dir}")
+                shutil.rmtree(temp_clone_dir)
+            
             temp_clone_dir.mkdir(exist_ok=True)
 
-            try:
-                self._clone_repository(repository, temp_clone_dir)
-                
-                # Create backup in specified format
-                backup_path = self._create_backup(
-                    temp_clone_dir, 
-                    repo_backup_dir, 
-                    backup_name, 
-                    repository.backup_format
-                )
-                
-                # Clean up old backups based on retention policy
-                self._cleanup_old_backups(repo_backup_dir, repository.retention_count, repository.backup_format)
-                
-                # Update backup job record
-                backup_job.status = 'completed'
-                backup_job.backup_path = str(backup_path)
-                backup_job.file_size = self._get_file_size(backup_path)
-                backup_job.completed_at = datetime.utcnow()
-                
-                # Update repository last backup time
-                repository.last_backup = datetime.utcnow()
-                
-                logger.info(f"Backup completed successfully: {backup_path}")
-                
-            finally:
-                # Always clean up temporary directory
-                if temp_clone_dir.exists():
-                    shutil.rmtree(temp_clone_dir)
+            self._clone_repository(repository, temp_clone_dir)
+            
+            # Create backup in specified format
+            backup_path = self._create_backup(
+                temp_clone_dir, 
+                repo_backup_dir, 
+                backup_name, 
+                repository.backup_format
+            )
+            
+            # Clean up old backups based on retention policy
+            self._cleanup_old_backups(repo_backup_dir, repository.retention_count, repository.backup_format)
+            
+            # Update backup job record
+            backup_job.status = 'completed'
+            backup_job.backup_path = str(backup_path)
+            backup_job.file_size = self._get_file_size(backup_path)
+            backup_job.completed_at = datetime.utcnow()
+            
+            # Update repository last backup time
+            repository.last_backup = datetime.utcnow()
+            
+            logger.info(f"Backup completed successfully: {backup_path}")
 
         except Exception as e:
             logger.error(f"Backup failed for repository {repository.name}: {str(e)}")
@@ -85,6 +86,14 @@ def backup_repository(self, repository):
             backup_job.completed_at = datetime.utcnow()
 
         finally:
+            # Always clean up temporary directory
+            if temp_clone_dir and temp_clone_dir.exists():
+                try:
+                    logger.info(f"Cleaning up temporary directory: {temp_clone_dir}")
+                    shutil.rmtree(temp_clone_dir)
+                except Exception as cleanup_error:
+                    logger.error(f"Failed to cleanup temp directory {temp_clone_dir}: {cleanup_error}")
+            
             db.session.commit()
 
     def _clone_repository(self, repository, clone_dir):
@@ -97,9 +106,33 @@ def _clone_repository(self, repository, clone_dir):
                 # Convert https://github.com/user/repo to https://[email protected]/user/repo
                 clone_url = clone_url.replace('https://github.com/', f'https://{repository.github_token}@github.com/')
 
-        # Clone the repository
-        git.Repo.clone_from(clone_url, clone_dir, depth=1)
-        logger.info(f"Repository cloned to: {clone_dir}")
+        # Clean up any existing temp directories for this repository first
+        self._cleanup_temp_directories(clone_dir.parent)
+        
+        # Clone the repository with error handling
+        try:
+            git.Repo.clone_from(clone_url, clone_dir, depth=1)
+            logger.info(f"Repository cloned to: {clone_dir}")
+        except git.GitCommandError as e:
+            if "already exists and is not an empty directory" in str(e):
+                logger.warning(f"Directory exists, cleaning and retrying: {clone_dir}")
+                shutil.rmtree(clone_dir)
+                clone_dir.mkdir(exist_ok=True)
+                git.Repo.clone_from(clone_url, clone_dir, depth=1)
+            else:
+                raise e
+    
+    def _cleanup_temp_directories(self, repo_backup_dir):
+        """Clean up old temporary directories that might be left behind"""
+        try:
+            temp_dirs = [d for d in repo_backup_dir.iterdir() if d.is_dir() and d.name.startswith('temp_')]
+            for temp_dir in temp_dirs:
+                # Remove temp directories older than 1 hour
+                if datetime.utcnow().timestamp() - temp_dir.stat().st_mtime > 3600:
+                    logger.info(f"Cleaning up old temp directory: {temp_dir}")
+                    shutil.rmtree(temp_dir)
+        except Exception as e:
+            logger.warning(f"Failed to cleanup temp directories: {e}")
 
     def _create_backup(self, source_dir, backup_dir, backup_name, backup_format):
         """Create backup in the specified format"""