Error checking

backupmult.ps1

@@ -171,14 +171,32 @@ function Start-BackupJob($jobId, $sqlInstance, $query, $baseLogFile) {
     $scriptBlock = {
         param($JobId, $SqlInstance, $Query, $BaseLogFile)
         
-        # Create job-specific log file path
+        # Debug the base log file parameter
-        $jobLogFile = $BaseLogFile -replace '\.log$', "-job$JobId.log"
+        Write-Output "DEBUG: BaseLogFile parameter = '$BaseLogFile'"
+        
+        # Create job-specific log file path with fallback
+        if ($BaseLogFile -and $BaseLogFile.Trim() -ne "") {
+            $jobLogFile = $BaseLogFile -replace '\.log$', "-job$JobId.log"
+        } else {
+            # Fallback log file path
+            $jobLogFile = "C:\Rubrik\backup-multi-job$JobId.log"
+        }
+        
+        Write-Output "DEBUG: Job log file will be: '$jobLogFile'"
         
         function Write-JobLog($message) {
             $timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss"
             $logEntry = "$timestamp [JOB-$JobId] $message"
             if ($jobLogFile -and $jobLogFile.Trim() -ne "") {
-                Add-Content -Path $jobLogFile -Value $logEntry -Encoding UTF8
+                try {
+                    Add-Content -Path $jobLogFile -Value $logEntry -Encoding UTF8
+                    # Output to console for debugging
+                    Write-Output "LOGGED TO $jobLogFile : $logEntry"
+                } catch {
+                    Write-Output "LOG ERROR: $($_.Exception.Message) - File: $jobLogFile"
+                }
+            } else {
+                Write-Output "NO LOG FILE: jobLogFile is empty or null"
             }
             Write-Output $logEntry
         }
@@ -199,12 +217,14 @@ function Start-BackupJob($jobId, $sqlInstance, $query, $baseLogFile) {
                 if ($message -and $message.Trim() -ne "") {
                     $script:infoMessages += $message
                     Write-JobLog "SQL INFO: $message"
+                    Write-Output "SQL INFO: $message"  # Also output for Receive-Job
                 }
             })
             
             try {
+                Write-JobLog "Attempting to connect to SQL Server: $SqlInstance"
                 $connection.Open()
-                Write-JobLog "Connected to SQL Server"
+                Write-JobLog "Connected to SQL Server successfully"
                 
                 $command = New-Object System.Data.SqlClient.SqlCommand
                 $command.Connection = $connection
@@ -223,13 +243,26 @@ function Start-BackupJob($jobId, $sqlInstance, $query, $baseLogFile) {
                         $rowData += "$($reader.GetName($i)): $($reader.GetValue($i))"
                     }
                     if ($rowData.Count -gt 0) {
-                        Write-JobLog "SQL RESULT: $($rowData -join ', ')"
+                        $resultLine = "SQL RESULT: $($rowData -join ', ')"
+                        Write-JobLog $resultLine
+                        Write-Output $resultLine  # Also output for Receive-Job
                     }
                 }
                 
                 $reader.Close()
-                Write-JobLog "Backup completed successfully. Captured $($infoMessages.Count) messages."
+                $summaryMessage = "Backup completed successfully. Captured $($infoMessages.Count) messages."
-                return @{ Success = $true; JobId = $JobId }
+                Write-JobLog $summaryMessage
+                Write-Output $summaryMessage  # Also output for Receive-Job
+                
+                # Output all captured SQL messages for debugging
+                Write-Output "=== SQL MESSAGES START ==="
+                foreach ($msg in $infoMessages) {
+                    Write-Output "SQL: $msg"
+                }
+                Write-Output "=== SQL MESSAGES END ==="
+                
+                # Don't return hashtable - just output success message
+                Write-Output "JOB-${JobId}: SUCCESS"
             }
             finally {
                 if ($connection.State -eq [System.Data.ConnectionState]::Open) {
@@ -239,21 +272,53 @@ function Start-BackupJob($jobId, $sqlInstance, $query, $baseLogFile) {
             }
         }
         catch {
-            Write-JobLog "ERROR: Backup failed - $($_.Exception.Message)"
+            $errorMessage = "ERROR: Backup failed - $($_.Exception.Message)"
+            Write-JobLog $errorMessage
+            Write-Output $errorMessage  # Also output for Receive-Job
+            
+            # Check for specific connection errors
+            if ($_.Exception.Message -like "*server*not found*" -or 
+                $_.Exception.Message -like "*network-related*" -or
+                $_.Exception.Message -like "*instance*" -or
+                $_.Exception.Message -like "*login*failed*") {
+                $connError = "ERROR: CONNECTION FAILURE - Check SQL Server instance name and connectivity"
+                Write-JobLog $connError
+                Write-Output $connError
+            }
             
             # Log SQL Server specific errors
             if ($_.Exception -is [System.Data.SqlClient.SqlException]) {
                 Write-JobLog "ERROR: SQL Server Error Details:"
+                Write-Output "ERROR: SQL Server Error Details:"
                 foreach ($sqlError in $_.Exception.Errors) {
-                    Write-JobLog "ERROR: Severity: $($sqlError.Class), State: $($sqlError.State), Number: $($sqlError.Number)"
+                    $errorDetail = "ERROR: Severity: $($sqlError.Class), State: $($sqlError.State), Number: $($sqlError.Number)"
-                    Write-JobLog "ERROR: Message: $($sqlError.Message)"
+                    Write-JobLog $errorDetail
+                    Write-Output $errorDetail
+                    
+                    $errorMsg = "ERROR: Message: $($sqlError.Message)"
+                    Write-JobLog $errorMsg
+                    Write-Output $errorMsg
+                    
                     if ($sqlError.Procedure) {
-                        Write-JobLog "ERROR: Procedure: $($sqlError.Procedure), Line: $($sqlError.LineNumber)"
+                        $procError = "ERROR: Procedure: $($sqlError.Procedure), Line: $($sqlError.LineNumber)"
+                        Write-JobLog $procError
+                        Write-Output $procError
                     }
                 }
             }
             
-            return @{ Success = $false; JobId = $JobId; ErrorMessage = $_.Exception.Message }
+            # Log full exception details for debugging
+            $fullError = "ERROR: Full Exception Type: $($_.Exception.GetType().Name)"
+            Write-JobLog $fullError
+            Write-Output $fullError
+            
+            if ($_.Exception.InnerException) {
+                $innerError = "ERROR: Inner Exception: $($_.Exception.InnerException.Message)"
+                Write-JobLog $innerError
+                Write-Output $innerError
+            }
+            
+            Write-Output "JOB-${JobId}: FAILED"
         }
     }
     
@@ -280,28 +345,36 @@ while (-not $allJobsCompleted) {
     Start-Sleep -Seconds 5
     
     foreach ($job in $jobList) {
-        if ($job.Id -notin $completedJobs -and $job.State -ne "Running") {
+        if ($job.Id -notin $completedJobs) {
-            $null = $completedJobs.Add($job.Id)
+            # Check if job is no longer running
-            
+            if ($job.State -eq "Completed" -or $job.State -eq "Failed" -or $job.State -eq "Stopped") {
-            if ($job.State -eq "Completed") {
+                $null = $completedJobs.Add($job.Id)
-                $result = Receive-Job -Job $job
+                
-                if ($result) {
+                # Get all job output
-                    foreach ($line in $result) {
+                $jobOutput = Receive-Job -Job $job -Keep  # Use -Keep to preserve output
-                        Write-Host $line
+                
+                if ($job.State -eq "Completed") {
+                    Write-Log "Job $($job.Id) completed successfully"
+                    
+                    # Log all job output to main log
+                    if ($jobOutput) {
+                        Write-Log "=== Job $($job.Id) Output ==="
+                        foreach ($line in $jobOutput) {
+                            Write-Log "$line"
+                        }
+                        Write-Log "=== End Job $($job.Id) Output ==="
                     }
-                }
+                } else {
-                Write-Log "Job $($job.Id) completed successfully"
+                    Write-Log "ERROR: Job $($job.Id) failed with state: $($job.State)"
-            } else {
+                    if ($jobOutput) {
-                $jobError = Receive-Job -Job $job
+                        Write-Log "=== Job $($job.Id) Error Output ==="
-                Write-Log "ERROR: Job $($job.Id) failed with state: $($job.State)"
+                        foreach ($line in $jobOutput) {
-                if ($jobError) {
+                            Write-Log "ERROR: $line"
-                    foreach ($line in $jobError) {
+                        }
-                        Write-Log "ERROR: $line"
+                        Write-Log "=== End Job $($job.Id) Error Output ==="
                     }
                 }
             }
-            
-            Remove-Job -Job $job
         }
     }
     
@@ -316,13 +389,53 @@ while (-not $allJobsCompleted) {
 
 Write-Log "All backup jobs completed"
 
+# Collect job states and outputs before cleanup for final status check
+$jobResults = @{}
+foreach ($job in $jobList) {
+    $jobOutput = Receive-Job -Job $job -Keep -ErrorAction SilentlyContinue
+    $hasFailed = $false
+    
+    # Check if job output contains failure indicator
+    if ($jobOutput) {
+        foreach ($line in $jobOutput) {
+            if ($line -like "*JOB-*: FAILED") {
+                $hasFailed = $true
+                break
+            }
+        }
+    }
+    
+    $jobResults[$job.Id] = @{
+        State = $job.State
+        Failed = $hasFailed
+    }
+}
+
+# Clean up jobs
+Write-Log "Cleaning up completed jobs..."
+foreach ($job in $jobList) {
+    try {
+        if ($job.State -eq "Running") {
+            Write-Log "WARNING: Job $($job.Id) still running, stopping it..."
+            Stop-Job -Job $job -Force
+            Start-Sleep -Seconds 2
+        }
+        Remove-Job -Job $job -Force -ErrorAction SilentlyContinue
+        Write-Log "Cleaned up job $($job.Id)"
+    } catch {
+        Write-Log "WARNING: Could not clean up job $($job.Id): $($_.Exception.Message)"
+    }
+}
+
 # Consolidate job logs into main log file
 Write-Log "Consolidating job logs..."
 for ($i = 1; $i -le $Jobs; $i++) {
     $jobLogFile = $logFile -replace '\.log$', "-job$i.log"
+    Write-Log "Checking for job log file: $jobLogFile"
     if (Test-Path $jobLogFile) {
         try {
             $jobContent = Get-Content $jobLogFile -ErrorAction Stop
+            Write-Log "Found $($jobContent.Count) lines in job $i log"
             foreach ($line in $jobContent) {
                 Add-Content -Path $logFile -Value $line -Encoding UTF8
             }
@@ -331,14 +444,21 @@ for ($i = 1; $i -le $Jobs; $i++) {
         } catch {
             Write-Log "WARNING: Could not consolidate log from job $i : $($_.Exception.Message)"
         }
+    } else {
+        Write-Log "WARNING: Job log file not found for job $i"
     }
 }
 
-# Final status check
+# Final status check using job output analysis
-$failedJobs = $jobList | Where-Object { $_.State -ne "Completed" }
+$failedJobIds = $jobResults.Keys | Where-Object { $jobResults[$_].Failed -eq $true }
-if ($failedJobs.Count -gt 0) {
+
-    Write-Log "ERROR: $($failedJobs.Count) jobs failed"
+if ($failedJobIds.Count -gt 0) {
+    Write-Log "ERROR: $($failedJobIds.Count) out of $($jobResults.Count) backup jobs failed"
+    foreach ($jobId in $failedJobIds) {
+        Write-Log "ERROR: Job ID $jobId failed"
+    }
+    Write-Log "CRITICAL: Backup operation failed - check errors above"
     exit 1
 } else {
-    Write-Log "SUCCESS: All $($jobList.Count) backup jobs completed successfully"
+    Write-Log "SUCCESS: All $($jobResults.Count) backup jobs completed successfully"
 }