From 88bdab08d265ad727905ff12b0c7d5819bc145a0 Mon Sep 17 00:00:00 2001 From: James Pattinson Date: Wed, 22 Oct 2025 17:31:50 +0100 Subject: [PATCH] Error checking --- backupmult.ps1 | 190 ++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 155 insertions(+), 35 deletions(-) diff --git a/backupmult.ps1 b/backupmult.ps1 index 5fdc481..051bffc 100644 --- a/backupmult.ps1 +++ b/backupmult.ps1 @@ -171,14 +171,32 @@ function Start-BackupJob($jobId, $sqlInstance, $query, $baseLogFile) { $scriptBlock = { param($JobId, $SqlInstance, $Query, $BaseLogFile) - # Create job-specific log file path - $jobLogFile = $BaseLogFile -replace '\.log$', "-job$JobId.log" + # Debug the base log file parameter + Write-Output "DEBUG: BaseLogFile parameter = '$BaseLogFile'" + + # Create job-specific log file path with fallback + if ($BaseLogFile -and $BaseLogFile.Trim() -ne "") { + $jobLogFile = $BaseLogFile -replace '\.log$', "-job$JobId.log" + } else { + # Fallback log file path + $jobLogFile = "C:\Rubrik\backup-multi-job$JobId.log" + } + + Write-Output "DEBUG: Job log file will be: '$jobLogFile'" function Write-JobLog($message) { $timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss" $logEntry = "$timestamp [JOB-$JobId] $message" if ($jobLogFile -and $jobLogFile.Trim() -ne "") { - Add-Content -Path $jobLogFile -Value $logEntry -Encoding UTF8 + try { + Add-Content -Path $jobLogFile -Value $logEntry -Encoding UTF8 + # Output to console for debugging + Write-Output "LOGGED TO $jobLogFile : $logEntry" + } catch { + Write-Output "LOG ERROR: $($_.Exception.Message) - File: $jobLogFile" + } + } else { + Write-Output "NO LOG FILE: jobLogFile is empty or null" } Write-Output $logEntry } @@ -199,12 +217,14 @@ function Start-BackupJob($jobId, $sqlInstance, $query, $baseLogFile) { if ($message -and $message.Trim() -ne "") { $script:infoMessages += $message Write-JobLog "SQL INFO: $message" + Write-Output "SQL INFO: $message" # Also output for Receive-Job } }) try { + Write-JobLog "Attempting to connect to SQL Server: $SqlInstance" $connection.Open() - Write-JobLog "Connected to SQL Server" + Write-JobLog "Connected to SQL Server successfully" $command = New-Object System.Data.SqlClient.SqlCommand $command.Connection = $connection @@ -223,13 +243,26 @@ function Start-BackupJob($jobId, $sqlInstance, $query, $baseLogFile) { $rowData += "$($reader.GetName($i)): $($reader.GetValue($i))" } if ($rowData.Count -gt 0) { - Write-JobLog "SQL RESULT: $($rowData -join ', ')" + $resultLine = "SQL RESULT: $($rowData -join ', ')" + Write-JobLog $resultLine + Write-Output $resultLine # Also output for Receive-Job } } $reader.Close() - Write-JobLog "Backup completed successfully. Captured $($infoMessages.Count) messages." - return @{ Success = $true; JobId = $JobId } + $summaryMessage = "Backup completed successfully. Captured $($infoMessages.Count) messages." + Write-JobLog $summaryMessage + Write-Output $summaryMessage # Also output for Receive-Job + + # Output all captured SQL messages for debugging + Write-Output "=== SQL MESSAGES START ===" + foreach ($msg in $infoMessages) { + Write-Output "SQL: $msg" + } + Write-Output "=== SQL MESSAGES END ===" + + # Don't return hashtable - just output success message + Write-Output "JOB-${JobId}: SUCCESS" } finally { if ($connection.State -eq [System.Data.ConnectionState]::Open) { @@ -239,21 +272,53 @@ function Start-BackupJob($jobId, $sqlInstance, $query, $baseLogFile) { } } catch { - Write-JobLog "ERROR: Backup failed - $($_.Exception.Message)" + $errorMessage = "ERROR: Backup failed - $($_.Exception.Message)" + Write-JobLog $errorMessage + Write-Output $errorMessage # Also output for Receive-Job + + # Check for specific connection errors + if ($_.Exception.Message -like "*server*not found*" -or + $_.Exception.Message -like "*network-related*" -or + $_.Exception.Message -like "*instance*" -or + $_.Exception.Message -like "*login*failed*") { + $connError = "ERROR: CONNECTION FAILURE - Check SQL Server instance name and connectivity" + Write-JobLog $connError + Write-Output $connError + } # Log SQL Server specific errors if ($_.Exception -is [System.Data.SqlClient.SqlException]) { Write-JobLog "ERROR: SQL Server Error Details:" + Write-Output "ERROR: SQL Server Error Details:" foreach ($sqlError in $_.Exception.Errors) { - Write-JobLog "ERROR: Severity: $($sqlError.Class), State: $($sqlError.State), Number: $($sqlError.Number)" - Write-JobLog "ERROR: Message: $($sqlError.Message)" + $errorDetail = "ERROR: Severity: $($sqlError.Class), State: $($sqlError.State), Number: $($sqlError.Number)" + Write-JobLog $errorDetail + Write-Output $errorDetail + + $errorMsg = "ERROR: Message: $($sqlError.Message)" + Write-JobLog $errorMsg + Write-Output $errorMsg + if ($sqlError.Procedure) { - Write-JobLog "ERROR: Procedure: $($sqlError.Procedure), Line: $($sqlError.LineNumber)" + $procError = "ERROR: Procedure: $($sqlError.Procedure), Line: $($sqlError.LineNumber)" + Write-JobLog $procError + Write-Output $procError } } } - return @{ Success = $false; JobId = $JobId; ErrorMessage = $_.Exception.Message } + # Log full exception details for debugging + $fullError = "ERROR: Full Exception Type: $($_.Exception.GetType().Name)" + Write-JobLog $fullError + Write-Output $fullError + + if ($_.Exception.InnerException) { + $innerError = "ERROR: Inner Exception: $($_.Exception.InnerException.Message)" + Write-JobLog $innerError + Write-Output $innerError + } + + Write-Output "JOB-${JobId}: FAILED" } } @@ -280,28 +345,36 @@ while (-not $allJobsCompleted) { Start-Sleep -Seconds 5 foreach ($job in $jobList) { - if ($job.Id -notin $completedJobs -and $job.State -ne "Running") { - $null = $completedJobs.Add($job.Id) - - if ($job.State -eq "Completed") { - $result = Receive-Job -Job $job - if ($result) { - foreach ($line in $result) { - Write-Host $line + if ($job.Id -notin $completedJobs) { + # Check if job is no longer running + if ($job.State -eq "Completed" -or $job.State -eq "Failed" -or $job.State -eq "Stopped") { + $null = $completedJobs.Add($job.Id) + + # Get all job output + $jobOutput = Receive-Job -Job $job -Keep # Use -Keep to preserve output + + if ($job.State -eq "Completed") { + Write-Log "Job $($job.Id) completed successfully" + + # Log all job output to main log + if ($jobOutput) { + Write-Log "=== Job $($job.Id) Output ===" + foreach ($line in $jobOutput) { + Write-Log "$line" + } + Write-Log "=== End Job $($job.Id) Output ===" } - } - Write-Log "Job $($job.Id) completed successfully" - } else { - $jobError = Receive-Job -Job $job - Write-Log "ERROR: Job $($job.Id) failed with state: $($job.State)" - if ($jobError) { - foreach ($line in $jobError) { - Write-Log "ERROR: $line" + } else { + Write-Log "ERROR: Job $($job.Id) failed with state: $($job.State)" + if ($jobOutput) { + Write-Log "=== Job $($job.Id) Error Output ===" + foreach ($line in $jobOutput) { + Write-Log "ERROR: $line" + } + Write-Log "=== End Job $($job.Id) Error Output ===" } } } - - Remove-Job -Job $job } } @@ -316,13 +389,53 @@ while (-not $allJobsCompleted) { Write-Log "All backup jobs completed" +# Collect job states and outputs before cleanup for final status check +$jobResults = @{} +foreach ($job in $jobList) { + $jobOutput = Receive-Job -Job $job -Keep -ErrorAction SilentlyContinue + $hasFailed = $false + + # Check if job output contains failure indicator + if ($jobOutput) { + foreach ($line in $jobOutput) { + if ($line -like "*JOB-*: FAILED") { + $hasFailed = $true + break + } + } + } + + $jobResults[$job.Id] = @{ + State = $job.State + Failed = $hasFailed + } +} + +# Clean up jobs +Write-Log "Cleaning up completed jobs..." +foreach ($job in $jobList) { + try { + if ($job.State -eq "Running") { + Write-Log "WARNING: Job $($job.Id) still running, stopping it..." + Stop-Job -Job $job -Force + Start-Sleep -Seconds 2 + } + Remove-Job -Job $job -Force -ErrorAction SilentlyContinue + Write-Log "Cleaned up job $($job.Id)" + } catch { + Write-Log "WARNING: Could not clean up job $($job.Id): $($_.Exception.Message)" + } +} + # Consolidate job logs into main log file Write-Log "Consolidating job logs..." for ($i = 1; $i -le $Jobs; $i++) { $jobLogFile = $logFile -replace '\.log$', "-job$i.log" + Write-Log "Checking for job log file: $jobLogFile" if (Test-Path $jobLogFile) { try { $jobContent = Get-Content $jobLogFile -ErrorAction Stop + Write-Log "Found $($jobContent.Count) lines in job $i log" foreach ($line in $jobContent) { Add-Content -Path $logFile -Value $line -Encoding UTF8 } @@ -331,14 +444,21 @@ for ($i = 1; $i -le $Jobs; $i++) { } catch { Write-Log "WARNING: Could not consolidate log from job $i : $($_.Exception.Message)" } + } else { + Write-Log "WARNING: Job log file not found for job $i" } } -# Final status check -$failedJobs = $jobList | Where-Object { $_.State -ne "Completed" } -if ($failedJobs.Count -gt 0) { - Write-Log "ERROR: $($failedJobs.Count) jobs failed" +# Final status check using job output analysis +$failedJobIds = $jobResults.Keys | Where-Object { $jobResults[$_].Failed -eq $true } + +if ($failedJobIds.Count -gt 0) { + Write-Log "ERROR: $($failedJobIds.Count) out of $($jobResults.Count) backup jobs failed" + foreach ($jobId in $failedJobIds) { + Write-Log "ERROR: Job ID $jobId failed" + } + Write-Log "CRITICAL: Backup operation failed - check errors above" exit 1 } else { - Write-Log "SUCCESS: All $($jobList.Count) backup jobs completed successfully" + Write-Log "SUCCESS: All $($jobResults.Count) backup jobs completed successfully" } \ No newline at end of file