""" Version Manager - Handles project versioning operations Supports upload, download, restore of project snapshots """ import os import zipfile import shutil from pathlib import Path from datetime import datetime from typing import Dict, Any, Optional, List from loguru import logger from core.database import Database # Centralizzazione versions: tutte le versions vanno in questa directory VERSIONS_BASE_PATH = Path("I:/SOFTWARE/PYTHON/DEVELOPMENT/versions") class VersionManager: """Manages project version uploads, downloads, and restores""" def __init__(self, db: Database): self.db = db def upload_version(self, project_id: int, version_name: str, description: str = "", progress_callback=None) -> Dict[str, Any]: """ Upload new version: 1. Get project config 2. Create ZIP snapshot from source 3. Extract ZIP to server active location (for direct execution with Python embedded) 4. Save metadata Args: project_id: Project configuration ID version_name: Version identifier (e.g., "v1.2.0") description: Optional version description Returns: Dict with stats: { 'success': bool, 'files_count': int, 'size_bytes': int, 'zip_path': str, 'server_path': str, 'message': str } """ try: # Get project config project = self.db.get_project_config(project_id) if not project: return {'success': False, 'message': f"Project {project_id} not found"} source_path = Path(project['source_path']) server_base = Path(project['server_base_path']) exclude_patterns = project['exclude_patterns'].split('\n') if project['exclude_patterns'] else [] # Validate source exists if not source_path.exists(): return {'success': False, 'message': f"Source path not found: {source_path}"} # Prepare server paths versions_dir = VERSIONS_BASE_PATH / project['name'] server_active = server_base / project['name'] versions_dir.mkdir(parents=True, exist_ok=True) # Check if version name already exists existing_versions = self.db.get_project_versions(project_id) if any(v['version_name'] == version_name for v in existing_versions): return {'success': False, 'message': f"Version '{version_name}' already exists"} logger.info(f"Starting upload for project '{project['name']}' version '{version_name}'") # Track total time import time start_time = time.time() # Step 1: Copy files directly from source to server active location logger.info(f"Copying files from {source_path} to {server_active}") # Ensure destination directory exists (Robocopy will handle cleanup) if not server_active.exists(): server_active.mkdir(parents=True, exist_ok=True) logger.info(f"Created new directory: {server_active}") else: logger.info(f"Using existing directory: {server_active}") # Note: Robocopy with /MIR will sync and remove obsolete files automatically # Wrap progress callback for copy phase def copy_progress(current, total, filename): if progress_callback: progress_callback('copy', current, total, filename) # Direct copy with progress tracking copy_stats = self._copy_tree(source_path, server_active, exclude_patterns, copy_progress) logger.info(f"Copied {copy_stats['files_count']} files to server") # Step 2: Create ZIP locally from source (fast - no network I/O) logger.info(f"Creating local ZIP snapshot") # Create ZIP in temp location first import tempfile timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') zip_filename = f"{version_name}_{timestamp}.zip" zip_path = versions_dir / zip_filename temp_dir = Path(tempfile.gettempdir()) / "scheduler_zips" temp_dir.mkdir(exist_ok=True) temp_zip = temp_dir / zip_filename # Wrap progress callback for ZIP phase def zip_progress(current, total, filename): if progress_callback: progress_callback('zip', current, total, filename) zip_stats = self._create_zip(source_path, temp_zip, exclude_patterns, zip_progress) logger.info(f"ZIP created locally: {zip_stats['size_bytes'] / (1024*1024):.2f} MB") # Step 3: Copy ZIP to server versions directory logger.info(f"Copying ZIP to server: {zip_path}") shutil.copy2(temp_zip, zip_path) temp_zip.unlink() # Clean up temp file logger.info(f"Upload complete: {copy_stats['files_count']} files") # Step 4: Save version metadata to database version_id = self.db.create_project_version( project_id=project_id, version_name=version_name, zip_filename=zip_filename, description=description, size_bytes=zip_stats['size_bytes'], files_count=zip_stats['files_count'] ) # Calculate total time elapsed_time = time.time() - start_time elapsed_minutes = int(elapsed_time // 60) elapsed_seconds = int(elapsed_time % 60) time_str = f"{elapsed_minutes}m {elapsed_seconds}s" if elapsed_minutes > 0 else f"{elapsed_seconds}s" logger.info(f"Version {version_id} created successfully: {zip_stats['files_count']} files, {zip_stats['size_bytes'] / (1024*1024):.2f} MB in {time_str}") return { 'success': True, 'version_id': version_id, 'files_count': zip_stats['files_count'], 'size_bytes': zip_stats['size_bytes'], 'zip_path': str(zip_path), 'server_path': str(server_active), 'elapsed_time': elapsed_time, 'message': f"Version '{version_name}' uploaded and extracted to server" } except Exception as e: logger.error(f"Error uploading version: {e}") return {'success': False, 'message': f"Upload failed: {str(e)}"} def download_version(self, version_id: int, destination: str, progress_callback=None) -> Dict[str, Any]: """ Download version to local path Args: version_id: Version record ID destination: Local destination path Returns: Dict with stats: { 'success': bool, 'files_count': int, 'destination': str, 'message': str } """ try: # Get version metadata version = self.db.get_project_version(version_id) if not version: return {'success': False, 'message': f"Version {version_id} not found"} # Get project config project = self.db.get_project_config(version['project_id']) if not project: return {'success': False, 'message': f"Project {version['project_id']} not found"} # Locate ZIP file versions_dir = VERSIONS_BASE_PATH / project['name'] zip_path = versions_dir / version['zip_filename'] if not zip_path.exists(): return {'success': False, 'message': f"ZIP file not found: {zip_path}"} # Create destination directory dest_path = Path(destination) dest_path.mkdir(parents=True, exist_ok=True) logger.info(f"Downloading version '{version['version_name']}' to {dest_path}") # Extract ZIP with progress def extract_progress(current, total, filename): if progress_callback: progress_callback('extract', current, total, filename) extract_stats = self._extract_zip(zip_path, dest_path, extract_progress) logger.info(f"Version downloaded: {extract_stats['files_count']} files") return { 'success': True, 'files_count': extract_stats['files_count'], 'destination': str(dest_path), 'message': f"Version '{version['version_name']}' downloaded successfully" } except Exception as e: logger.error(f"Error downloading version: {e}") return {'success': False, 'message': f"Download failed: {str(e)}"} def restore_version(self, version_id: int, progress_callback=None) -> Dict[str, Any]: """ Restore version to server active location Creates automatic backup first Args: version_id: Version record ID Returns: Dict with stats: { 'success': bool, 'files_count': int, 'backup_version_id': int, 'message': str } """ try: # Get version metadata version = self.db.get_project_version(version_id) if not version: return {'success': False, 'message': f"Version {version_id} not found"} # Get project config project = self.db.get_project_config(version['project_id']) if not project: return {'success': False, 'message': f"Project {version['project_id']} not found"} # Prepare paths server_base = Path(project['server_base_path']) server_active = server_base / project['name'] versions_dir = VERSIONS_BASE_PATH / project['name'] zip_path = versions_dir / version['zip_filename'] if not zip_path.exists(): return {'success': False, 'message': f"ZIP file not found: {zip_path}"} logger.info(f"Restoring version '{version['version_name']}' for project '{project['name']}'") # Step 1: Create automatic backup of current state (if exists) backup_version_id = None backup_version_name = None if server_active.exists() and any(server_active.iterdir()): timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') backup_version_name = f"pre-restore-backup-{timestamp}" backup_description = f"Auto-backup before restoring {version['version_name']}" logger.info("Creating pre-restore backup...") exclude_patterns = project['exclude_patterns'].split('\n') if project['exclude_patterns'] else [] # Create backup ZIP backup_zip_filename = f"{backup_version_name}.zip" backup_zip_path = versions_dir / backup_zip_filename # Backup progress def backup_progress(current, total, filename): if progress_callback: progress_callback('backup', current, total, filename) backup_stats = self._create_zip(server_active, backup_zip_path, exclude_patterns, backup_progress) # Save backup metadata backup_version_id = self.db.create_project_version( project_id=project['project_id'], version_name=backup_version_name, zip_filename=backup_zip_filename, description=backup_description, size_bytes=backup_stats['size_bytes'], files_count=backup_stats['files_count'] ) logger.info(f"Pre-restore backup created: version ID {backup_version_id}") else: logger.info("No existing files to backup (first restore)") # Step 2: Clear current server active location if server_active.exists(): logger.info(f"Clearing existing files in {server_active}") for item in server_active.iterdir(): if item.is_dir(): shutil.rmtree(item) else: item.unlink() # Step 3: Extract version ZIP to server active location logger.info(f"Extracting {version['version_name']} to {server_active}") server_active.mkdir(parents=True, exist_ok=True) # Extract progress def extract_progress(current, total, filename): if progress_callback: progress_callback('extract', current, total, filename) extract_stats = self._extract_zip(zip_path, server_active, extract_progress) logger.info(f"Version restored: {extract_stats['files_count']} files") # Build success message message = f"Version '{version['version_name']}' restored successfully" if backup_version_name: message += f". Backup created as '{backup_version_name}'" else: message += " (first restore, no backup needed)" return { 'success': True, 'files_count': extract_stats['files_count'], 'backup_version_id': backup_version_id, 'backup_version_name': backup_version_name, 'message': message } except Exception as e: logger.error(f"Error restoring version: {e}") return {'success': False, 'message': f"Restore failed: {str(e)}"} def delete_version(self, version_id: int): """ Delete version ZIP and database record Args: version_id: Version record ID Returns: Dict with result: { 'success': bool, 'message': str } """ try: # Get version metadata version = self.db.get_project_version(version_id) if not version: return {'success': False, 'message': f"Version {version_id} not found"} # Get project config project = self.db.get_project_config(version['project_id']) if not project: return {'success': False, 'message': f"Project {version['project_id']} not found"} # Locate ZIP file versions_dir = VERSIONS_BASE_PATH / project['name'] zip_path = versions_dir / version['zip_filename'] # Delete ZIP file if exists if zip_path.exists(): zip_path.unlink() logger.info(f"Deleted ZIP file: {zip_path}") else: logger.warning(f"ZIP file not found: {zip_path}") # Delete database record self.db.delete_project_version(version_id) return { 'success': True, 'message': f"Version '{version['version_name']}' deleted successfully" } except Exception as e: logger.error(f"Error deleting version: {e}") return {'success': False, 'message': f"Delete failed: {str(e)}"} def _create_zip(self, source_dir: Path, zip_path: Path, exclude_patterns: List[str], progress_callback=None) -> Dict[str, Any]: """ Helper: Create ZIP with exclusions Args: source_dir: Directory to compress zip_path: Output ZIP file path exclude_patterns: Patterns to exclude progress_callback: Optional function(current, total, filename) for progress updates Returns: Dict with stats: { 'files_count': int, 'size_bytes': int } """ import fnmatch # First pass: count total files to compress total_files = 0 if progress_callback: for root, dirs, files in os.walk(source_dir): root_path = Path(root) rel_root = root_path.relative_to(source_dir) # Filter directories dirs[:] = [d for d in dirs if not self._should_exclude(str(rel_root / d), exclude_patterns)] for file in files: file_path = root_path / file rel_path = file_path.relative_to(source_dir) if not self._should_exclude(str(rel_path), exclude_patterns): total_files += 1 files_count = 0 size_bytes = 0 with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf: for root, dirs, files in os.walk(source_dir): # Convert to Path for easier manipulation root_path = Path(root) rel_root = root_path.relative_to(source_dir) # Filter directories (modifies dirs in-place to prevent recursion) dirs[:] = [d for d in dirs if not self._should_exclude(str(rel_root / d), exclude_patterns)] for file in files: file_path = root_path / file rel_path = file_path.relative_to(source_dir) # Check if file should be excluded if self._should_exclude(str(rel_path), exclude_patterns): continue try: zipf.write(file_path, rel_path) files_count += 1 size_bytes += file_path.stat().st_size # Report progress if progress_callback: progress_callback(files_count, total_files, str(rel_path)) except Exception as e: logger.warning(f"Failed to add {rel_path} to ZIP: {e}") # Get final ZIP size zip_size = zip_path.stat().st_size return { 'files_count': files_count, 'size_bytes': zip_size, 'uncompressed_size': size_bytes } def _extract_zip(self, zip_path: Path, destination: Path, progress_callback=None) -> Dict[str, Any]: """ Helper: Extract ZIP to destination Args: zip_path: Source ZIP file destination: Extraction destination progress_callback: Optional function(current, total, filename) for progress updates Returns: Dict with stats: { 'files_count': int } """ files_count = 0 with zipfile.ZipFile(zip_path, 'r') as zipf: members = zipf.namelist() total_files = len(members) for index, member in enumerate(members, start=1): zipf.extract(member, destination) files_count += 1 # Report progress if progress_callback: progress_callback(index, total_files, member) return { 'files_count': files_count } def _is_windows_device_name(self, filename: str) -> bool: """ Check if filename is a Windows reserved device name (nul, con, prn, aux, com1-9, lpt1-9) """ reserved_names = { 'nul', 'con', 'prn', 'aux', 'com1', 'com2', 'com3', 'com4', 'com5', 'com6', 'com7', 'com8', 'com9', 'lpt1', 'lpt2', 'lpt3', 'lpt4', 'lpt5', 'lpt6', 'lpt7', 'lpt8', 'lpt9' } # Check base name (without extension) name_lower = filename.lower() base_name = name_lower.split('.')[0] return base_name in reserved_names def _copy_tree(self, source_dir: Path, dest_dir: Path, exclude_patterns: List[str], progress_callback=None) -> Dict[str, Any]: """ Helper: Copy directory tree using Robocopy + Python cleanup Strategy: 1. Use Robocopy for fast bulk copy (all files, multi-threaded) 2. Use Python to remove excluded files/directories This is much faster than file-by-file copy, especially on network drives. With Python embedded, exclusions are minimal (~1% of files), making this approach optimal: Robocopy copies ~99%, Python cleanup removes ~1%. Args: source_dir: Source directory to copy from dest_dir: Destination directory to copy to exclude_patterns: Patterns to exclude progress_callback: Optional function(current, total, filename) for progress updates Returns: Dict with stats: { 'files_count': int, 'bytes_copied': int } """ import subprocess # Step 1: Robocopy bulk copy with standard exclusions (very fast) logger.info(f"Robocopy: {source_dir} -> {dest_dir}") # Robocopy command with optimization flags # Note: We let Robocopy copy everything fast, then Python removes exclusions # (Robocopy /XD only excludes from root, not recursively, so not useful for __pycache__) robocopy_cmd = [ 'robocopy', str(source_dir), str(dest_dir), '/MIR', # Mirror mode: Copy all + remove obsolete files in destination '/MT:16', # Multi-threaded (16 threads for speed) '/R:2', # Retry 2 times on failure '/W:1', # Wait 1 second between retries '/NFL', # No file list (less output) '/NDL', # No directory list '/NP', # No progress (we'll track with Python) '/BYTES', # Show sizes in bytes '/NC', # No class (file size info) '/NS', # No size '/NJH', # No job header '/NJS', # No job summary # Only exclude Windows special device files (these can't be copied anyway) '/XF', 'nul', 'con', 'prn', 'aux', 'nul.*', 'con.*', 'prn.*', 'aux.*', 'com1', 'com2', 'com3', 'com4', 'com5', 'com6', 'com7', 'com8', 'com9', 'lpt1', 'lpt2', 'lpt3', 'lpt4', 'lpt5', 'lpt6', 'lpt7', 'lpt8', 'lpt9' ] logger.info("Robocopy will copy all files (no pre-counting for maximum speed)") # Report indeterminate progress during Robocopy if progress_callback: progress_callback(0, 0, "Multi-threaded copy in progress...") # Execute Robocopy # Note: Robocopy exit codes: 0-7 are success, 8+ are errors logger.info("Starting Robocopy execution...") try: result = subprocess.run( robocopy_cmd, capture_output=True, text=True, encoding='cp1252', # Windows console encoding timeout=900 # 15 minute timeout to prevent infinite hangs ) except subprocess.TimeoutExpired: error_msg = "Robocopy timed out after 15 minutes" logger.error(error_msg) raise Exception(error_msg) # Check for errors (exit code 8 or higher means error) if result.returncode >= 8: error_msg = f"Robocopy failed with exit code {result.returncode}" logger.error(error_msg) logger.error(f"Robocopy stdout: {result.stdout}") logger.error(f"Robocopy stderr: {result.stderr}") raise Exception(error_msg) logger.info(f"Robocopy completed successfully with exit code {result.returncode}") # Step 2: No cleanup, no counting - Robocopy done! # The accurate file count will come from ZIP creation logger.info("Server copy complete - ZIP will be created from local source") # Report progress (completion) - use 0 since we don't have accurate count if progress_callback: progress_callback(1, 1, "Server copy complete") # Return placeholder values (accurate count will come from ZIP) return { 'files_count': 0, # Will be updated from ZIP stats 'bytes_copied': 0 # Will be updated from ZIP stats } def _should_exclude(self, path_str: str, exclude_patterns: List[str]) -> bool: """ Check if path matches any exclusion pattern Args: path_str: Path string to check exclude_patterns: List of patterns (supports wildcards) Returns: True if should be excluded """ import fnmatch # Normalize path separators path_str = path_str.replace('\\', '/') for pattern in exclude_patterns: pattern = pattern.strip() # Skip empty lines and comments if not pattern or pattern.startswith('#'): continue # Normalize pattern pattern = pattern.replace('\\', '/') # Check directory pattern (ends with /) if pattern.endswith('/'): # Match directory at any level if fnmatch.fnmatch(path_str + '/', '*/' + pattern) or fnmatch.fnmatch(path_str + '/', pattern): return True # Also check if any parent directory matches parts = path_str.split('/') for i in range(len(parts)): if fnmatch.fnmatch(parts[i] + '/', pattern): return True else: # File pattern - match against full path or filename if fnmatch.fnmatch(path_str, '*/' + pattern) or fnmatch.fnmatch(path_str, pattern): return True # Also check just the filename filename = path_str.split('/')[-1] if fnmatch.fnmatch(filename, pattern): return True return False