param( [Parameter(Mandatory=$true)] [string]$SqlInstance, [Parameter(Mandatory=$false)] [string]$Directories, [Parameter(Mandatory=$false)] [int]$Jobs = 2, [Parameter(Mandatory=$false)] [switch]$Force ) # # backupmult.ps1 - Parallel database backup script using Ola Hallengren's DatabasesInParallel feature # # Uses Ola H's built-in parallel processing by starting multiple concurrent backup jobs # Each job will automatically share the database load using DatabasesInParallel=Y # # Import SQL Server PowerShell module try { if (Get-Module -ListAvailable -Name SqlServer) { Import-Module SqlServer -ErrorAction Stop Write-Host "INFO: SqlServer PowerShell module loaded successfully." } elseif (Get-Module -ListAvailable -Name SQLPS) { Import-Module SQLPS -ErrorAction Stop Write-Host "INFO: SQLPS PowerShell module loaded successfully." } else { throw "No SQL Server PowerShell module found" } if (-not (Get-Command Invoke-Sqlcmd -ErrorAction SilentlyContinue)) { throw "Invoke-Sqlcmd command not available" } } catch { Write-Host "ERROR: Failed to import SQL Server PowerShell module. Please install it using: Install-Module -Name SqlServer -AllowClobber" Write-Host "ERROR: $($_.Exception.Message)" exit 1 } $instanceName = $SqlInstance.Split('\')[1] # Use provided directories or default to comma-separated multi-directory setup if ($Directories) { $directoryParam = $Directories Write-Host "INFO: Using provided directories: $directoryParam" } else { $directoryParam = "C:\Rubrik\$instanceName\Dir1, C:\Rubrik\$instanceName\Dir2, C:\Rubrik\$instanceName\Dir3, C:\Rubrik\$instanceName\Dir4" Write-Host "INFO: Using default multi-directory setup: $directoryParam" } $fullBackupDay = 'Thursday' $fullBackupOverdueDays = 7 $logFile = "C:\Rubrik\backup-multi-$instanceName.log" # Validate job count if ($Jobs -lt 1 -or $Jobs -gt 8) { Write-Host "ERROR: Jobs parameter must be between 1 and 8. Provided: $Jobs" exit 1 } Write-Host "INFO: Starting $Jobs parallel backup jobs" $today = (Get-Date).Date function Write-Log($message, $jobId = "") { $timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss" $jobPrefix = if ($jobId) { "[JOB-$jobId] " } else { "" } $logEntry = "$timestamp $jobPrefix$message" # Use mutex for thread-safe logging to main log file $mutex = $null try { $mutex = [System.Threading.Mutex]::new($false, "BackupLogMutex") if ($mutex.WaitOne(5000)) { # 5 second timeout Add-Content -Path $logFile -Value $logEntry -Encoding UTF8 } else { Write-Warning "Could not acquire log mutex, writing to console only" } } catch { Write-Warning "Logging error: $($_.Exception.Message)" } finally { if ($mutex) { $mutex.ReleaseMutex() $mutex.Dispose() } } Write-Host $logEntry } function Get-BackupType($directoryParam) { # Use first directory to check flags (assuming shared flag logic across all directories) $firstDir = ($directoryParam -split ',')[0].Trim() $fullFlag = Join-Path $firstDir "last_full.flag" $diffFlag = Join-Path $firstDir "last_diff.flag" # Check if full backup is overdue $isFullBackupOverdue = $false if (Test-Path $fullFlag) { try { $lastFullDate = [DateTime]::ParseExact((Get-Content $fullFlag).Trim(), "yyyy-MM-dd", $null) $daysSinceLastFull = ($today - $lastFullDate).Days $isFullBackupOverdue = $daysSinceLastFull -gt $fullBackupOverdueDays Write-Log "INFO: Last full backup was $daysSinceLastFull days ago. Overdue threshold: $fullBackupOverdueDays days." } catch { $isFullBackupOverdue = $true Write-Log "WARNING: Could not parse last full backup date. Treating as overdue." } } else { $isFullBackupOverdue = $true Write-Log "WARNING: No last full backup date found. Treating as overdue." } # Determine backup type if ((Get-Date).DayOfWeek -eq $fullBackupDay -or $isFullBackupOverdue) { if (-not (Test-Path $fullFlag) -or (Get-Content $fullFlag).Trim() -ne $today.ToString("yyyy-MM-dd")) { # Create flag directory if it doesn't exist $flagDir = Split-Path $fullFlag -Parent if (-not (Test-Path $flagDir)) { New-Item -ItemType Directory -Path $flagDir -Force | Out-Null } Set-Content $fullFlag $today.ToString("yyyy-MM-dd") -Encoding UTF8 $reason = if($isFullBackupOverdue) { "overdue" } else { "scheduled" } return @{ Type = "FULL"; CleanupTime = 168; Reason = $reason } } else { return @{ Type = "LOG"; CleanupTime = 24; Reason = "full already taken today" } } } else { if (-not (Test-Path $diffFlag) -or (Get-Content $diffFlag).Trim() -ne $today.ToString("yyyy-MM-dd")) { # Create flag directory if it doesn't exist $flagDir = Split-Path $diffFlag -Parent if (-not (Test-Path $flagDir)) { New-Item -ItemType Directory -Path $flagDir -Force | Out-Null } Set-Content $diffFlag $today.ToString("yyyy-MM-dd") -Encoding UTF8 return @{ Type = "DIFF"; CleanupTime = 168; Reason = "differential scheduled" } } else { return @{ Type = "LOG"; CleanupTime = 24; Reason = "diff already taken today" } } } } # Determine backup type $backupInfo = Get-BackupType $directoryParam Write-Log "Selected $($backupInfo.Type) backup ($($backupInfo.Reason))" # Build the Ola H query with DatabasesInParallel enabled $query = @" EXECUTE [dbo].[DatabaseBackup] @Databases = 'ALL_DATABASES', @Directory = '$directoryParam', @BackupType = '$($backupInfo.Type)', @Verify = 'N', @CleanupTime = $($backupInfo.CleanupTime), @CheckSum = 'Y', @LogToTable = 'Y', @DatabasesInParallel = 'Y' "@ Write-Log "SQL Query: $query" # Function to execute backup job with message capture function Start-BackupJob($jobId, $sqlInstance, $query, $baseLogFile) { $scriptBlock = { param($JobId, $SqlInstance, $Query, $BaseLogFile) # Create job-specific log file path $jobLogFile = $BaseLogFile -replace '\.log$', "-job$JobId.log" function Write-JobLog($message) { $timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss" $logEntry = "$timestamp [JOB-$JobId] $message" if ($jobLogFile -and $jobLogFile.Trim() -ne "") { Add-Content -Path $jobLogFile -Value $logEntry -Encoding UTF8 } Write-Output $logEntry } try { Write-JobLog "Starting backup job" # Create SQL connection with message capture $connection = New-Object System.Data.SqlClient.SqlConnection $connection.ConnectionString = "Server=$SqlInstance;Integrated Security=true;Connection Timeout=30" $infoMessages = @() # Event handler for informational messages (PRINT statements) $connection.add_InfoMessage({ param($sqlSender, $e) $message = $e.Message if ($message -and $message.Trim() -ne "") { $script:infoMessages += $message Write-JobLog "SQL INFO: $message" } }) try { $connection.Open() Write-JobLog "Connected to SQL Server" $command = New-Object System.Data.SqlClient.SqlCommand $command.Connection = $connection $command.CommandText = $Query $command.CommandTimeout = 0 # No timeout for backup operations Write-JobLog "Executing backup command..." # Execute and capture any result sets $reader = $command.ExecuteReader() # Process any result sets while ($reader.Read()) { $rowData = @() for ($i = 0; $i -lt $reader.FieldCount; $i++) { $rowData += "$($reader.GetName($i)): $($reader.GetValue($i))" } if ($rowData.Count -gt 0) { Write-JobLog "SQL RESULT: $($rowData -join ', ')" } } $reader.Close() Write-JobLog "Backup completed successfully. Captured $($infoMessages.Count) messages." return @{ Success = $true; JobId = $JobId } } finally { if ($connection.State -eq [System.Data.ConnectionState]::Open) { $connection.Close() } $connection.Dispose() } } catch { Write-JobLog "ERROR: Backup failed - $($_.Exception.Message)" # Log SQL Server specific errors if ($_.Exception -is [System.Data.SqlClient.SqlException]) { Write-JobLog "ERROR: SQL Server Error Details:" foreach ($sqlError in $_.Exception.Errors) { Write-JobLog "ERROR: Severity: $($sqlError.Class), State: $($sqlError.State), Number: $($sqlError.Number)" Write-JobLog "ERROR: Message: $($sqlError.Message)" if ($sqlError.Procedure) { Write-JobLog "ERROR: Procedure: $($sqlError.Procedure), Line: $($sqlError.LineNumber)" } } } return @{ Success = $false; JobId = $JobId; ErrorMessage = $_.Exception.Message } } } return Start-Job -ScriptBlock $scriptBlock -ArgumentList $jobId, $sqlInstance, $query, $baseLogFile } # Start parallel backup jobs Write-Log "Starting $Jobs parallel backup jobs using DatabasesInParallel feature" [System.Collections.ArrayList]$jobList = @() for ($i = 1; $i -le $Jobs; $i++) { $job = Start-BackupJob -jobId $i -sqlInstance $SqlInstance -query $query -logFile $logFile $null = $jobList.Add($job) Write-Log "Started backup job $i (Job ID: $($job.Id))" Start-Sleep -Milliseconds 100 # Small delay to stagger job starts } # Monitor jobs and capture output Write-Log "Monitoring $($jobList.Count) backup jobs..." $allJobsCompleted = $false [System.Collections.ArrayList]$completedJobs = @() while (-not $allJobsCompleted) { Start-Sleep -Seconds 5 foreach ($job in $jobList) { if ($job.Id -notin $completedJobs -and $job.State -ne "Running") { $null = $completedJobs.Add($job.Id) if ($job.State -eq "Completed") { $result = Receive-Job -Job $job if ($result) { foreach ($line in $result) { Write-Host $line } } Write-Log "Job $($job.Id) completed successfully" } else { $jobError = Receive-Job -Job $job Write-Log "ERROR: Job $($job.Id) failed with state: $($job.State)" if ($jobError) { foreach ($line in $jobError) { Write-Log "ERROR: $line" } } } Remove-Job -Job $job } } $allJobsCompleted = $completedJobs.Count -eq $jobList.Count # Progress update $runningCount = ($jobList | Where-Object { $_.State -eq "Running" }).Count if ($runningCount -gt 0) { Write-Log "Progress: $($completedJobs.Count)/$($jobList.Count) jobs completed, $runningCount still running..." } } Write-Log "All backup jobs completed" # Consolidate job logs into main log file Write-Log "Consolidating job logs..." for ($i = 1; $i -le $Jobs; $i++) { $jobLogFile = $logFile -replace '\.log$', "-job$i.log" if (Test-Path $jobLogFile) { try { $jobContent = Get-Content $jobLogFile -ErrorAction Stop foreach ($line in $jobContent) { Add-Content -Path $logFile -Value $line -Encoding UTF8 } Remove-Item $jobLogFile -Force Write-Log "Consolidated log from job $i" } catch { Write-Log "WARNING: Could not consolidate log from job $i : $($_.Exception.Message)" } } } # Final status check $failedJobs = $jobList | Where-Object { $_.State -ne "Completed" } if ($failedJobs.Count -gt 0) { Write-Log "ERROR: $($failedJobs.Count) jobs failed" exit 1 } else { Write-Log "SUCCESS: All $($jobList.Count) backup jobs completed successfully" }