First parallel script

backup.ps1

@@ -1,19 +1,61 @@
 param(
     [Parameter(Mandatory=$true)]
-    [string]$SqlInstance
+    [string]$SqlInstance,
+    
+    [Parameter(Mandatory=$false)]
+    [string]$Directory,
+    
+    [Parameter(Mandatory=$false)]
+    [switch]$Force
 )
 
-#
 # backup.ps1
 #
-# TODO: Update cleanup time based on backup type
+# TODO: Parallelize backups for multiple DBs in the instance
+
+# Import SQL Server PowerShell module
+try {
+    # Try to import the newer SqlServer module first
+    if (Get-Module -ListAvailable -Name SqlServer) {
+        Import-Module SqlServer -ErrorAction Stop
+        Write-Host "INFO: SqlServer PowerShell module loaded successfully."
+    } 
+    # Fall back to older SQLPS module if available
+    elseif (Get-Module -ListAvailable -Name SQLPS) {
+        Import-Module SQLPS -ErrorAction Stop
+        Write-Host "INFO: SQLPS PowerShell module loaded successfully."
+    }
+    else {
+        throw "No SQL Server PowerShell module found"
+    }
+    
+    # Verify Invoke-Sqlcmd is available
+    if (-not (Get-Command Invoke-Sqlcmd -ErrorAction SilentlyContinue)) {
+        throw "Invoke-Sqlcmd command not available"
+    }
+}
+catch {
+    Write-Host "ERROR: Failed to import SQL Server PowerShell module. Please install it using: Install-Module -Name SqlServer -AllowClobber"
+    Write-Host "ERROR: $($_.Exception.Message)"
+    exit 1
+}
 
 $instanceName = $SqlInstance.Split('\')[1]
 
-$directory = "C:\Rubrik\$instanceName"
+# Use provided directory parameter or default to instance-based path
+if ($Directory) {
+    $directory = $Directory
+    Write-Host "INFO: Using provided directory: $directory"
+} else {
+    $directory = "C:\Rubrik\$instanceName"
+    Write-Host "INFO: Using default directory: $directory"
+}
+
 $fullBackupDay = 'Thursday'
+$fullBackupOverdueDays = 7  # Force full backup if last full backup is older than this many days
 $checkCluster = $false
-$logFile = "C:\Rubrik\backup-$instanceName.log"
+#$logFile = "C:\Rubrik\backup-$instanceName.log"
+$logFile = "H:\Backup\backup-$instanceName.log"
 
 $fullFlag = $directory + "\last_full.flag"
 $diffFlag = $directory + "\last_diff.flag"
@@ -27,6 +69,35 @@ function FlagTakenToday($flagPath) {
     return $false
 }
 
+function GetLastFullBackupDate($flagPath) {
+    if (Test-Path $flagPath) {
+        $flagDate = (Get-Content $flagPath | Out-String).Trim()
+        try {
+            return [DateTime]::ParseExact($flagDate, "yyyy-MM-dd", $null)
+        }
+        catch {
+            Write-Log "WARNING: Could not parse last full backup date from flag file: $flagDate"
+            return $null
+        }
+    }
+    return $null
+}
+
+function IsFullBackupOverdue($flagPath, $overdueDays) {
+    $lastFullDate = GetLastFullBackupDate $flagPath
+    if ($null -eq $lastFullDate) {
+        Write-Log "WARNING: No last full backup date found. Full backup is considered overdue."
+        return $true
+    }
+    
+    $daysSinceLastFull = ($today - $lastFullDate).Days
+    $isOverdue = $daysSinceLastFull -gt $overdueDays
+    
+    Write-Log "INFO: Last full backup was $daysSinceLastFull days ago on $($lastFullDate.ToString('yyyy-MM-dd')). Overdue threshold: $overdueDays days."
+    
+    return $isOverdue
+}
+
 function Write-Log($message) {
     $timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss"
     $logEntry = "$timestamp $message"
@@ -34,20 +105,23 @@ function Write-Log($message) {
     Write-Host $logEntry
 }
 
-# Check if directory exists and is a symbolic link
+# Check if directory exists and is a symbolic link (unless -Force is specified)
 if (-not (Test-Path $directory)) {
     Write-Log "ERROR: Directory '$directory' does not exist. Exiting script."
     exit 1
 }
 
-$directoryInfo = Get-Item $directory
-if (-not ($directoryInfo.Attributes -band [System.IO.FileAttributes]::ReparsePoint)) {
+if (-not $Force) {
+    $directoryInfo = Get-Item $directory
+    if (-not ($directoryInfo.Attributes -band [System.IO.FileAttributes]::ReparsePoint)) {
         Write-Log "ERROR: Directory '$directory' is not a symbolic link. Exiting script."
         exit 1
+    }
+    Write-Log "INFO: Directory '$directory' exists and is a symbolic link. Target: $($directoryInfo.Target). Proceeding."
+} else {
+    Write-Log "INFO: Force parameter specified. Skipping symbolic link check for directory '$directory'."
 }
 
-Write-Log "INFO: Directory '$directory' exists and is a symbolic link. Target: $($directoryInfo.Target). Proceeding."
-
 if ($checkCluster) {
     # Check if SQL instance is running locally
     $localNode = $env:COMPUTERNAME
@@ -69,17 +143,31 @@ if ($checkCluster) {
     Write-Log "INFO: Cluster check is disabled. Proceeding without verification."
 }
 
+# Check if full backup is overdue regardless of the day
+$isFullBackupOverdue = IsFullBackupOverdue $fullFlag $fullBackupOverdueDays
+
 if ((Get-Date).DayOfWeek -eq $fullBackupDay) {
     if (-not (FlagTakenToday $fullFlag)) {
         $backupType = "FULL"
         $cleanupTime = 168
         Set-Content $fullFlag $today.ToString("yyyy-MM-dd")
-        Write-Log "Selected FULL backup. Flag updated."
+        Write-Log "Selected FULL backup (scheduled day). Flag updated."
     } else {
         $backupType = "LOG"
         $cleanupTime = 24
         Write-Log "FULL backup already taken today. Selected LOG backup."
     }
+} elseif ($isFullBackupOverdue) {
+    if (-not (FlagTakenToday $fullFlag)) {
+        $backupType = "FULL"
+        $cleanupTime = 168
+        Set-Content $fullFlag $today.ToString("yyyy-MM-dd")
+        Write-Log "Selected FULL backup (overdue - forcing full backup). Flag updated."
+    } else {
+        $backupType = "LOG"
+        $cleanupTime = 24
+        Write-Log "FULL backup already taken today (was overdue). Selected LOG backup."
+    }
 } else {
     if (-not (FlagTakenToday $diffFlag)) {
         $backupType = "DIFF"
@@ -95,16 +183,89 @@ if ((Get-Date).DayOfWeek -eq $fullBackupDay) {
 
 $query = "EXECUTE [dbo].[DatabaseBackup] @Databases = 'ALL_DATABASES', @Directory = '$directory', @BackupType = '$backupType', @Verify = 'N', @CleanupTime = $cleanupTime, @CheckSum = 'Y', @LogToTable = 'Y'"
 Write-Log "Executing backup type: $backupType"
+Write-Log "SQL Query: $query"
 
-$sqlcmdOutput = & sqlcmd -S $SqlInstance -Q $query 2>&1
-$sqlcmdExitCode = $LASTEXITCODE
+try {
+    # Execute the backup using PowerShell SQL module with better error handling
+    # Capture verbose output from Ola H scripts
+    $infoMessages = @()
     
-if ($sqlcmdExitCode -eq 0) {
-    foreach ($line in $sqlcmdOutput) {
-        Write-Log $line
+    # Create event handlers to capture SQL Server messages
+    $connection = New-Object System.Data.SqlClient.SqlConnection
+    $connection.ConnectionString = "Server=$SqlInstance;Integrated Security=true;Connection Timeout=30"
+    
+    # Event handler for informational messages (PRINT statements)
+    $connection.add_InfoMessage({
+        param($sqlSender, $e)
+        $message = $e.Message
+        if ($message -and $message.Trim() -ne "") {
+            $script:infoMessages += $message
+            Write-Log "SQL INFO: $message"
         }
-    Write-Log "$backupType Backup execution completed."
-} else {
-    Write-Log "ERROR: Backup execution failed. Exit code: $sqlcmdExitCode. Output: $sqlcmdOutput"
+    })
+    
+    try {
+        $connection.Open()
+        
+        $command = New-Object System.Data.SqlClient.SqlCommand
+        $command.Connection = $connection
+        $command.CommandText = $query
+        $command.CommandTimeout = 0  # No timeout for backup operations
+        
+        Write-Log "Executing SQL command with message capture..."
+        
+        # Execute and capture any result sets
+        $reader = $command.ExecuteReader()
+        
+        # Process any result sets
+        while ($reader.Read()) {
+            $rowData = @()
+            for ($i = 0; $i -lt $reader.FieldCount; $i++) {
+                $rowData += "$($reader.GetName($i)): $($reader.GetValue($i))"
+            }
+            if ($rowData.Count -gt 0) {
+                Write-Log "SQL RESULT: $($rowData -join ', ')"
+            }
+        }
+        
+        $reader.Close()
+    }
+    finally {
+        if ($connection.State -eq [System.Data.ConnectionState]::Open) {
+            $connection.Close()
+        }
+        $connection.Dispose()
+    }
+    
+    Write-Log "$backupType Backup execution completed successfully."
+    Write-Log "Total informational messages captured: $($infoMessages.Count)"
+}
+catch {
+    Write-Log "ERROR: Backup execution failed with exception: $($_.Exception.Message)"
+    
+    # Log additional SQL Server error details if available
+    if ($_.Exception.InnerException) {
+        Write-Log "ERROR: Inner Exception: $($_.Exception.InnerException.Message)"
+    }
+    
+    # Check for SQL Server specific errors
+    if ($_.Exception -is [System.Data.SqlClient.SqlException]) {
+        Write-Log "ERROR: SQL Server Error Details:"
+        foreach ($sqlError in $_.Exception.Errors) {
+            Write-Log "ERROR: Severity: $($sqlError.Class), State: $($sqlError.State), Number: $($sqlError.Number)"
+            Write-Log "ERROR: Message: $($sqlError.Message)"
+            if ($sqlError.Procedure) {
+                Write-Log "ERROR: Procedure: $($sqlError.Procedure), Line: $($sqlError.LineNumber)"
+            }
+        }
+    }
+    
+    # Clean up connection if it exists
+    if ($connection -and $connection.State -eq [System.Data.ConnectionState]::Open) {
+        $connection.Close()
+        $connection.Dispose()
+    }
+    
+    exit 1
 }
 

backupmult.ps1

@@ -0,0 +1,344 @@
+param(
+    [Parameter(Mandatory=$true)]
+    [string]$SqlInstance,
+    
+    [Parameter(Mandatory=$false)]
+    [string]$Directories,
+    
+    [Parameter(Mandatory=$false)]
+    [int]$Jobs = 2,
+    
+    [Parameter(Mandatory=$false)]
+    [switch]$Force
+)
+
+#
+# backupmult.ps1 - Parallel database backup script using Ola Hallengren's DatabasesInParallel feature
+#
+# Uses Ola H's built-in parallel processing by starting multiple concurrent backup jobs
+# Each job will automatically share the database load using DatabasesInParallel=Y
+#
+
+# Import SQL Server PowerShell module
+try {
+    if (Get-Module -ListAvailable -Name SqlServer) {
+        Import-Module SqlServer -ErrorAction Stop
+        Write-Host "INFO: SqlServer PowerShell module loaded successfully."
+    } 
+    elseif (Get-Module -ListAvailable -Name SQLPS) {
+        Import-Module SQLPS -ErrorAction Stop
+        Write-Host "INFO: SQLPS PowerShell module loaded successfully."
+    }
+    else {
+        throw "No SQL Server PowerShell module found"
+    }
+    
+    if (-not (Get-Command Invoke-Sqlcmd -ErrorAction SilentlyContinue)) {
+        throw "Invoke-Sqlcmd command not available"
+    }
+}
+catch {
+    Write-Host "ERROR: Failed to import SQL Server PowerShell module. Please install it using: Install-Module -Name SqlServer -AllowClobber"
+    Write-Host "ERROR: $($_.Exception.Message)"
+    exit 1
+}
+
+$instanceName = $SqlInstance.Split('\')[1]
+
+# Use provided directories or default to comma-separated multi-directory setup
+if ($Directories) {
+    $directoryParam = $Directories
+    Write-Host "INFO: Using provided directories: $directoryParam"
+} else {
+    $directoryParam = "C:\Rubrik\$instanceName\Dir1, C:\Rubrik\$instanceName\Dir2, C:\Rubrik\$instanceName\Dir3, C:\Rubrik\$instanceName\Dir4"
+    Write-Host "INFO: Using default multi-directory setup: $directoryParam"
+}
+
+$fullBackupDay = 'Thursday'
+$fullBackupOverdueDays = 7
+$logFile = "C:\Rubrik\backup-multi-$instanceName.log"
+
+# Validate job count
+if ($Jobs -lt 1 -or $Jobs -gt 8) {
+    Write-Host "ERROR: Jobs parameter must be between 1 and 8. Provided: $Jobs"
+    exit 1
+}
+
+Write-Host "INFO: Starting $Jobs parallel backup jobs"
+
+$today = (Get-Date).Date
+
+function Write-Log($message, $jobId = "") {
+    $timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss"
+    $jobPrefix = if ($jobId) { "[JOB-$jobId] " } else { "" }
+    $logEntry = "$timestamp $jobPrefix$message"
+    
+    # Use mutex for thread-safe logging to main log file
+    $mutex = $null
+    try {
+        $mutex = [System.Threading.Mutex]::new($false, "BackupLogMutex")
+        if ($mutex.WaitOne(5000)) {  # 5 second timeout
+            Add-Content -Path $logFile -Value $logEntry -Encoding UTF8
+        } else {
+            Write-Warning "Could not acquire log mutex, writing to console only"
+        }
+    } catch {
+        Write-Warning "Logging error: $($_.Exception.Message)"
+    } finally {
+        if ($mutex) {
+            $mutex.ReleaseMutex()
+            $mutex.Dispose()
+        }
+    }
+    
+    Write-Host $logEntry
+}
+
+function Get-BackupType($directoryParam) {
+    # Use first directory to check flags (assuming shared flag logic across all directories)
+    $firstDir = ($directoryParam -split ',')[0].Trim()
+    $fullFlag = Join-Path $firstDir "last_full.flag"
+    $diffFlag = Join-Path $firstDir "last_diff.flag"
+    
+    # Check if full backup is overdue
+    $isFullBackupOverdue = $false
+    if (Test-Path $fullFlag) {
+        try {
+            $lastFullDate = [DateTime]::ParseExact((Get-Content $fullFlag).Trim(), "yyyy-MM-dd", $null)
+            $daysSinceLastFull = ($today - $lastFullDate).Days
+            $isFullBackupOverdue = $daysSinceLastFull -gt $fullBackupOverdueDays
+            Write-Log "INFO: Last full backup was $daysSinceLastFull days ago. Overdue threshold: $fullBackupOverdueDays days."
+        }
+        catch {
+            $isFullBackupOverdue = $true
+            Write-Log "WARNING: Could not parse last full backup date. Treating as overdue."
+        }
+    } else {
+        $isFullBackupOverdue = $true
+        Write-Log "WARNING: No last full backup date found. Treating as overdue."
+    }
+    
+    # Determine backup type
+    if ((Get-Date).DayOfWeek -eq $fullBackupDay -or $isFullBackupOverdue) {
+        if (-not (Test-Path $fullFlag) -or (Get-Content $fullFlag).Trim() -ne $today.ToString("yyyy-MM-dd")) {
+            # Create flag directory if it doesn't exist
+            $flagDir = Split-Path $fullFlag -Parent
+            if (-not (Test-Path $flagDir)) {
+                New-Item -ItemType Directory -Path $flagDir -Force | Out-Null
+            }
+            Set-Content $fullFlag $today.ToString("yyyy-MM-dd") -Encoding UTF8
+            $reason = if($isFullBackupOverdue) { "overdue" } else { "scheduled" }
+            return @{ Type = "FULL"; CleanupTime = 168; Reason = $reason }
+        } else {
+            return @{ Type = "LOG"; CleanupTime = 24; Reason = "full already taken today" }
+        }
+    } else {
+        if (-not (Test-Path $diffFlag) -or (Get-Content $diffFlag).Trim() -ne $today.ToString("yyyy-MM-dd")) {
+            # Create flag directory if it doesn't exist
+            $flagDir = Split-Path $diffFlag -Parent
+            if (-not (Test-Path $flagDir)) {
+                New-Item -ItemType Directory -Path $flagDir -Force | Out-Null
+            }
+            Set-Content $diffFlag $today.ToString("yyyy-MM-dd") -Encoding UTF8
+            return @{ Type = "DIFF"; CleanupTime = 168; Reason = "differential scheduled" }
+        } else {
+            return @{ Type = "LOG"; CleanupTime = 24; Reason = "diff already taken today" }
+        }
+    }
+}
+
+# Determine backup type
+$backupInfo = Get-BackupType $directoryParam
+Write-Log "Selected $($backupInfo.Type) backup ($($backupInfo.Reason))"
+
+# Build the Ola H query with DatabasesInParallel enabled
+$query = @"
+EXECUTE [dbo].[DatabaseBackup] 
+    @Databases = 'ALL_DATABASES',
+    @Directory = '$directoryParam',
+    @BackupType = '$($backupInfo.Type)',
+    @Verify = 'N',
+    @CleanupTime = $($backupInfo.CleanupTime),
+    @CheckSum = 'Y',
+    @LogToTable = 'Y',
+    @DatabasesInParallel = 'Y'
+"@
+
+Write-Log "SQL Query: $query"
+
+# Function to execute backup job with message capture
+function Start-BackupJob($jobId, $sqlInstance, $query, $baseLogFile) {    
+    $scriptBlock = {
+        param($JobId, $SqlInstance, $Query, $BaseLogFile)
+        
+        # Create job-specific log file path
+        $jobLogFile = $BaseLogFile -replace '\.log$', "-job$JobId.log"
+        
+        function Write-JobLog($message) {
+            $timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss"
+            $logEntry = "$timestamp [JOB-$JobId] $message"
+            if ($jobLogFile -and $jobLogFile.Trim() -ne "") {
+                Add-Content -Path $jobLogFile -Value $logEntry -Encoding UTF8
+            }
+            Write-Output $logEntry
+        }
+        
+        try {
+            Write-JobLog "Starting backup job"
+            
+            # Create SQL connection with message capture
+            $connection = New-Object System.Data.SqlClient.SqlConnection
+            $connection.ConnectionString = "Server=$SqlInstance;Integrated Security=true;Connection Timeout=30"
+            
+            $infoMessages = @()
+            
+            # Event handler for informational messages (PRINT statements)
+            $connection.add_InfoMessage({
+                param($sqlSender, $e)
+                $message = $e.Message
+                if ($message -and $message.Trim() -ne "") {
+                    $script:infoMessages += $message
+                    Write-JobLog "SQL INFO: $message"
+                }
+            })
+            
+            try {
+                $connection.Open()
+                Write-JobLog "Connected to SQL Server"
+                
+                $command = New-Object System.Data.SqlClient.SqlCommand
+                $command.Connection = $connection
+                $command.CommandText = $Query
+                $command.CommandTimeout = 0  # No timeout for backup operations
+                
+                Write-JobLog "Executing backup command..."
+                
+                # Execute and capture any result sets
+                $reader = $command.ExecuteReader()
+                
+                # Process any result sets
+                while ($reader.Read()) {
+                    $rowData = @()
+                    for ($i = 0; $i -lt $reader.FieldCount; $i++) {
+                        $rowData += "$($reader.GetName($i)): $($reader.GetValue($i))"
+                    }
+                    if ($rowData.Count -gt 0) {
+                        Write-JobLog "SQL RESULT: $($rowData -join ', ')"
+                    }
+                }
+                
+                $reader.Close()
+                Write-JobLog "Backup completed successfully. Captured $($infoMessages.Count) messages."
+                return @{ Success = $true; JobId = $JobId }
+            }
+            finally {
+                if ($connection.State -eq [System.Data.ConnectionState]::Open) {
+                    $connection.Close()
+                }
+                $connection.Dispose()
+            }
+        }
+        catch {
+            Write-JobLog "ERROR: Backup failed - $($_.Exception.Message)"
+            
+            # Log SQL Server specific errors
+            if ($_.Exception -is [System.Data.SqlClient.SqlException]) {
+                Write-JobLog "ERROR: SQL Server Error Details:"
+                foreach ($sqlError in $_.Exception.Errors) {
+                    Write-JobLog "ERROR: Severity: $($sqlError.Class), State: $($sqlError.State), Number: $($sqlError.Number)"
+                    Write-JobLog "ERROR: Message: $($sqlError.Message)"
+                    if ($sqlError.Procedure) {
+                        Write-JobLog "ERROR: Procedure: $($sqlError.Procedure), Line: $($sqlError.LineNumber)"
+                    }
+                }
+            }
+            
+            return @{ Success = $false; JobId = $JobId; ErrorMessage = $_.Exception.Message }
+        }
+    }
+    
+    return Start-Job -ScriptBlock $scriptBlock -ArgumentList $jobId, $sqlInstance, $query, $baseLogFile
+}
+
+# Start parallel backup jobs
+Write-Log "Starting $Jobs parallel backup jobs using DatabasesInParallel feature"
+[System.Collections.ArrayList]$jobList = @()
+
+for ($i = 1; $i -le $Jobs; $i++) {
+    $job = Start-BackupJob -jobId $i -sqlInstance $SqlInstance -query $query -logFile $logFile
+    $null = $jobList.Add($job)
+    Write-Log "Started backup job $i (Job ID: $($job.Id))"
+    Start-Sleep -Milliseconds 100  # Small delay to stagger job starts
+}
+
+# Monitor jobs and capture output
+Write-Log "Monitoring $($jobList.Count) backup jobs..."
+$allJobsCompleted = $false
+[System.Collections.ArrayList]$completedJobs = @()
+
+while (-not $allJobsCompleted) {
+    Start-Sleep -Seconds 5
+    
+    foreach ($job in $jobList) {
+        if ($job.Id -notin $completedJobs -and $job.State -ne "Running") {
+            $null = $completedJobs.Add($job.Id)
+            
+            if ($job.State -eq "Completed") {
+                $result = Receive-Job -Job $job
+                if ($result) {
+                    foreach ($line in $result) {
+                        Write-Host $line
+                    }
+                }
+                Write-Log "Job $($job.Id) completed successfully"
+            } else {
+                $jobError = Receive-Job -Job $job
+                Write-Log "ERROR: Job $($job.Id) failed with state: $($job.State)"
+                if ($jobError) {
+                    foreach ($line in $jobError) {
+                        Write-Log "ERROR: $line"
+                    }
+                }
+            }
+            
+            Remove-Job -Job $job
+        }
+    }
+    
+    $allJobsCompleted = $completedJobs.Count -eq $jobList.Count
+    
+    # Progress update
+    $runningCount = ($jobList | Where-Object { $_.State -eq "Running" }).Count
+    if ($runningCount -gt 0) {
+        Write-Log "Progress: $($completedJobs.Count)/$($jobList.Count) jobs completed, $runningCount still running..."
+    }
+}
+
+Write-Log "All backup jobs completed"
+
+# Consolidate job logs into main log file
+Write-Log "Consolidating job logs..."
+for ($i = 1; $i -le $Jobs; $i++) {
+    $jobLogFile = $logFile -replace '\.log$', "-job$i.log"
+    if (Test-Path $jobLogFile) {
+        try {
+            $jobContent = Get-Content $jobLogFile -ErrorAction Stop
+            foreach ($line in $jobContent) {
+                Add-Content -Path $logFile -Value $line -Encoding UTF8
+            }
+            Remove-Item $jobLogFile -Force
+            Write-Log "Consolidated log from job $i"
+        } catch {
+            Write-Log "WARNING: Could not consolidate log from job $i : $($_.Exception.Message)"
+        }
+    }
+}
+
+# Final status check
+$failedJobs = $jobList | Where-Object { $_.State -ne "Completed" }
+if ($failedJobs.Count -gt 0) {
+    Write-Log "ERROR: $($failedJobs.Count) jobs failed"
+    exit 1
+} else {
+    Write-Log "SUCCESS: All $($jobList.Count) backup jobs completed successfully"
+}