From f3d7592e7dea4b2f6057fed219df410cd865904f Mon Sep 17 00:00:00 2001 From: James Pattinson Date: Tue, 14 Oct 2025 16:16:44 +0000 Subject: [PATCH] Email bounce handling with postfix --- .env.example | 17 +- EMAIL_BOUNCE_HANDLING_SETUP.md | 243 +++++++++++++ EMAIL_BOUNCE_HANDLING_SETUP_COMPREHENSIVE.md | 212 +++++++++++ README.md | 24 +- api/main.py | 7 +- postfix/Dockerfile | 13 +- postfix/aliases | 17 + postfix/entrypoint.sh | 23 ++ postfix/main.cf.template | 20 ++ postfix/process-bounce.py | 348 +++++++++++++++++++ postfix/smtp_generic | 23 ++ postfix/virtual_aliases.cf | 11 - 12 files changed, 934 insertions(+), 24 deletions(-) create mode 100644 EMAIL_BOUNCE_HANDLING_SETUP.md create mode 100644 EMAIL_BOUNCE_HANDLING_SETUP_COMPREHENSIVE.md create mode 100644 postfix/aliases create mode 100644 postfix/process-bounce.py create mode 100644 postfix/smtp_generic delete mode 100644 postfix/virtual_aliases.cf diff --git a/.env.example b/.env.example index 58a4851..8ffc255 100644 --- a/.env.example +++ b/.env.example @@ -22,8 +22,10 @@ MYSQL_ROOT_PASSWORD=change_this_root_password API_TOKEN=change_this_to_a_secure_random_token # Bounce Handling Configuration (Optional) -# Set to 'true' to enable SNS webhook bounce handling -# Set to 'false' to disable and rely on email-based bounce handling +# Choose one of the bounce handling methods below: + +# Method 1: SNS Webhook Bounce Handling (Requires SES Production Access) +# Set to 'true' to enable real-time SNS webhook bounce handling ENABLE_SNS_WEBHOOKS=false ENABLE_BOUNCE_HANDLING=false @@ -31,4 +33,13 @@ ENABLE_BOUNCE_HANDLING=false # 1. AWS SNS topic configured # 2. SES configured to send notifications to SNS topic # 3. Valid HTTPS domain for webhook endpoint -# 4. SNS subscription confirmed to your webhook endpoint \ No newline at end of file +# 4. SNS subscription confirmed to your webhook endpoint + +# Method 2: Email-Based Bounce Processing (Works with SES Sandbox) +# Set to 'true' to enable email-based bounce processing +# This processes bounce emails that are sent to bounces@lists.sasalliance.org +ENABLE_EMAIL_BOUNCE_PROCESSING=false + +# Note: You can enable both methods, but typically only one is needed +# Email-based processing works with SES sandbox accounts +# SNS webhooks provide real-time processing but require SES production access \ No newline at end of file diff --git a/EMAIL_BOUNCE_HANDLING_SETUP.md b/EMAIL_BOUNCE_HANDLING_SETUP.md new file mode 100644 index 0000000..6d53f73 --- /dev/null +++ b/EMAIL_BOUNCE_HANDLING_SETUP.md @@ -0,0 +1,243 @@ +# Email-Based Bounce Handling Setup + +This document explains the email-based bounce handling system implemented as an alternative to SNS webhooks for environments without SES production access. + +## Overview + +The system processes email bounces directly within the Postfix container by: +1. Rewriting return paths to direct bounces to a processing address +2. Processing bounce emails via Python script +3. Updating member bounce statistics in MySQL database +4. Automatically disabling members with excessive bounces +2. Setting up an alias that pipes bounce emails to a Python processing script +3. The script parses bounce emails, extracts bounced addresses, and updates the database +4. Members with hard bounces are automatically deactivated +5. Bounce history is tracked and displayed in the UI (same as SNS method) + +## Advantages + +- **Works with SES Sandbox**: No production SES access required +- **No External Dependencies**: Doesn't require SNS, webhooks, or HTTPS domains +- **Self-Contained**: All processing happens within the existing containers +- **Real-time Processing**: Bounces are processed as soon as emails arrive +- **Compatible**: Uses the same database schema and UI as SNS bounce handling + +## Configuration + +### 1. Enable Email Bounce Processing + +In your `.env` file: + +```bash +# Enable email-based bounce processing +ENABLE_EMAIL_BOUNCE_PROCESSING=true + +# This will automatically enable bounce handling features +ENABLE_BOUNCE_HANDLING=true # Automatically set to true when email processing is enabled +``` + +### 2. Restart the System + +```bash +sudo docker-compose down +sudo docker-compose up --build -d +``` + +### 3. Verify Configuration + +Check that email bounce processing is enabled: + +```bash +curl -s http://localhost:8000/config | jq . +``` + +Expected output: +```json +{ + "bounce_handling_enabled": true, + "sns_webhooks_enabled": false, + "email_bounce_processing_enabled": true +} +``` + +## How It Works + +### Postfix Configuration + +The system configures Postfix with: + +- `bounce_notice_recipient = bounces@lists.sasalliance.org` +- `2bounce_notice_recipient = bounces@lists.sasalliance.org` +- `error_notice_recipient = bounces@lists.sasalliance.org` + +### Aliases Configuration + +The `bounces` address is configured to pipe emails to the processing script: + +``` +bounces: "|/usr/local/bin/process-bounce.py" +``` + +### Bounce Processing Script + +The Python script (`/usr/local/bin/process-bounce.py`): + +1. **Reads bounce emails** from stdin (via pipe) +2. **Parses email content** using multiple regex patterns to extract bounced addresses +3. **Analyzes bounce type** based on SMTP error codes: + - 5xx codes = Permanent bounces + - 4xx codes = Transient bounces + - Unknown = Undetermined +4. **Updates database** using the same schema as SNS bounce handling: + - Logs bounce in `bounce_logs` table + - Updates member `bounce_count`, `bounce_status`, and `last_bounce_at` + - Deactivates members with permanent bounces + - Marks members with soft bounce status after 3 transient bounces + +## Testing + +### Test the Processing Script + +You can test the bounce processing script in test mode: + +```bash +# Test with sample bounce email +sudo docker-compose exec postfix /usr/local/bin/process-bounce.py --test +``` + +Expected output: +``` +2025-10-14 15:49:16,041 - bounce-processor - INFO - Starting bounce processing +2025-10-14 15:49:16,041 - bounce-processor - INFO - Running in test mode with sample bounce email +2025-10-14 15:49:16,050 - bounce-processor - INFO - Extracted addresses: ['testuser@example.com'] +2025-10-14 15:49:16,050 - bounce-processor - INFO - Test mode - would process 1 bounce(s): +2025-10-14 15:49:16,050 - bounce-processor - INFO - {'email': 'testuser@example.com', 'bounce_type': 'Permanent', 'bounce_subtype': 'General', 'diagnostic_code': '', 'timestamp': '2025-10-14 15:49:16'} +``` + +### Test with Real Bounce Email + +To test with a real bounce: + +1. Send an email to a non-existent address via your mailing list +2. Wait for the bounce to be processed +3. Check the database for bounce logs: + +```bash +# Check bounce logs +sudo docker-compose exec mysql mysql -u maillist -pmaillist maillist -e "SELECT * FROM bounce_logs ORDER BY created_at DESC LIMIT 5;" + +# Check member bounce status +sudo docker-compose exec mysql mysql -u maillist -pmaillist maillist -e "SELECT member_id, email, bounce_count, bounce_status, last_bounce_at FROM members WHERE bounce_count > 0;" +``` + +### View in Web Interface + +1. Open http://localhost:3000 +2. Navigate to the Members tab +3. Look for bounce badges and bounce counts next to member names +4. Click the "Bounces" button next to a member to view bounce history + +## Supported Bounce Email Formats + +The processing script recognizes these bounce patterns: + +### SMTP Error Codes +- 550, 551, 553, 552, 554 (permanent failures) +- 450, 451, 452 (temporary failures) + +### Delivery Status Notification (DSN) +- `Final-Recipient:` headers +- `Original-Recipient:` headers + +### Common Bounce Messages +- "user unknown" +- "does not exist" +- "not found" +- "mailbox unavailable" +- "recipient rejected" + +## Monitoring + +### View Processing Logs + +```bash +# View bounce processing logs +sudo docker-compose logs postfix | grep bounce-processor + +# Follow logs in real-time +sudo docker-compose logs -f postfix | grep bounce-processor +``` + +### Check Aliases Configuration + +```bash +# Verify aliases are configured correctly +sudo docker-compose exec postfix cat /etc/aliases + +# Check alias database +sudo docker-compose exec postfix postmap -q bounces /etc/aliases +``` + +## Troubleshooting + +### Bounces Not Being Processed + +1. **Check aliases configuration:** + ```bash + sudo docker-compose exec postfix cat /etc/aliases + ``` + +2. **Verify script permissions:** + ```bash + sudo docker-compose exec postfix ls -la /usr/local/bin/process-bounce.py + ``` + +3. **Test script manually:** + ```bash + sudo docker-compose exec postfix /usr/local/bin/process-bounce.py --test + ``` + +4. **Check Postfix logs:** + ```bash + sudo docker-compose logs postfix | grep -i bounce + ``` + +### Database Connection Issues + +1. **Check environment variables:** + ```bash + sudo docker-compose exec postfix env | grep MYSQL + ``` + +2. **Test database connection:** + ```bash + sudo docker-compose exec postfix python3 -c "import pymysql; print('PyMySQL available')" + ``` + +### Script Errors + +View detailed error logs: +```bash +sudo docker-compose logs postfix | grep -A 10 -B 10 "bounce-processor.*ERROR" +``` + +## Comparison with SNS Webhooks + +| Feature | Email-Based | SNS Webhooks | +|---------|-------------|--------------| +| **SES Requirement** | Sandbox OK | Production access required | +| **External Dependencies** | None | SNS, HTTPS domain | +| **Processing Speed** | Real-time | Real-time | +| **Setup Complexity** | Low | High | +| **Reliability** | High | High | +| **Bounce Detection** | Regex-based | AWS-provided | +| **Cost** | Free | SNS charges apply | + +## Next Steps + +1. **Monitor bounce processing** to ensure it's working correctly +2. **Review bounce patterns** in the logs to improve detection if needed +3. **Set up bounce notification alerts** (optional) +4. **Consider upgrading to SNS webhooks** when SES production access is available + +Email-based bounce handling provides a robust alternative that works immediately with any SES configuration while providing the same bounce management features as the SNS webhook method. \ No newline at end of file diff --git a/EMAIL_BOUNCE_HANDLING_SETUP_COMPREHENSIVE.md b/EMAIL_BOUNCE_HANDLING_SETUP_COMPREHENSIVE.md new file mode 100644 index 0000000..8fb7f9d --- /dev/null +++ b/EMAIL_BOUNCE_HANDLING_SETUP_COMPREHENSIVE.md @@ -0,0 +1,212 @@ +# Email-Based Bounce Handling Setup (Comprehensive) + +This document explains the complete email-based bounce handling system implemented as an alternative to SNS webhooks for environments without SES production access. + +## Overview + +The system processes email bounces directly within the Postfix container by: +1. Rewriting return paths to direct bounces to a processing address +2. Processing bounce emails via Python script +3. Updating member bounce statistics in MySQL database +4. Automatically disabling members with excessive bounces + +## Configuration + +### Environment Variables (in .env) + +```bash +# Bounce handling feature flags +ENABLE_BOUNCE_HANDLING=true # Master switch for bounce functionality +ENABLE_EMAIL_BOUNCE_PROCESSING=true # Enable email-based processing +ENABLE_SNS_WEBHOOKS=false # Disable SNS webhooks (optional) + +# Database settings (required for bounce processing) +MYSQL_ROOT_PASSWORD=your_root_password +MYSQL_DATABASE=maillist +MYSQL_USER=maillist +MYSQL_PASSWORD=your_maillist_password +``` + +### Key Components + +#### 1. Return Path Rewriting (`postfix/smtp_generic`) +Routes all bounces to the processing address: +``` +@lists.sasalliance.org bounces@lists.sasalliance.org +@sasalliance.org bounces@lists.sasalliance.org +``` + +#### 2. Bounce Processing Script (`postfix/process-bounce.py`) +Python script that: +- Parses bounce emails for recipient addresses and bounce types +- Updates bounce counts in MySQL database +- Automatically disables members after 5 hard bounces +- Logs all bounce events + +#### 3. Postfix Integration (`postfix/main.cf.template`) +``` +# Return path rewriting for outbound mail +smtp_generic_maps = hash:/etc/postfix/smtp_generic + +# Bounce processing +bounce_notice_recipient = bounces@lists.sasalliance.org +``` + +#### 4. Email Aliases (`postfix/entrypoint.sh`) +``` +# Route bounces to processing script +bounces: "|/usr/local/bin/python3 /etc/postfix/process-bounce.py" +``` + +## How It Works + +### Outbound Email Flow +1. User sends email to `community@lists.sasalliance.org` +2. Postfix expands to member list via MySQL +3. Email sent via SES with return path rewritten to `bounces@lists.sasalliance.org` +4. If delivery fails, bounce goes to bounce processing address + +### Bounce Processing Flow +1. Bounce email arrives at `bounces@lists.sasalliance.org` +2. Postfix pipes email to `process-bounce.py` script +3. Script parses bounce for recipient and bounce type +4. Database updated with bounce information +5. Member automatically disabled if hard bounce threshold reached + +### Bounce Types Detected +- **Hard Bounces**: Permanent failures (5.x.x SMTP codes) + - Invalid email addresses + - Domain doesn't exist + - Mailbox doesn't exist +- **Soft Bounces**: Temporary failures (4.x.x SMTP codes) + - Mailbox full + - Temporary server issues + - Rate limiting + +## Database Schema + +### Bounce Logs Table +```sql +CREATE TABLE bounce_logs ( + id INT PRIMARY KEY AUTO_INCREMENT, + email VARCHAR(255) NOT NULL, + bounce_type ENUM('hard', 'soft', 'complaint') NOT NULL, + bounce_reason TEXT, + bounced_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + raw_message TEXT, + INDEX idx_email (email), + INDEX idx_bounced_at (bounced_at) +); +``` + +### Members Table (bounce tracking fields) +```sql +ALTER TABLE members ADD COLUMN bounce_count INT DEFAULT 0; +ALTER TABLE members ADD COLUMN last_bounce_at TIMESTAMP NULL; +ALTER TABLE members ADD COLUMN active BOOLEAN DEFAULT true; +``` + +## Testing Bounce Handling + +### 1. Test Return Path Configuration +```bash +# Check that return path rewriting is working +sudo docker-compose exec postfix postconf smtp_generic_maps +sudo docker-compose exec postfix postmap -q "test@lists.sasalliance.org" hash:/etc/postfix/smtp_generic +``` + +### 2. Simulate Bounce Email +```bash +# Send test bounce to processing script +echo "Subject: Delivery Status Notification (Failure) +From: MAILER-DAEMON@ses.amazonaws.com +To: bounces@lists.sasalliance.org + +The following message could not be delivered: + Recipient: test@example.com + Reason: 550 5.1.1 User unknown" | docker-compose exec -T postfix mail -s "Test Bounce" bounces@lists.sasalliance.org +``` + +### 3. Check Bounce Processing +```bash +# View bounce logs +sudo docker-compose exec mysql mysql -u maillist -pmaillist maillist -e "SELECT * FROM bounce_logs ORDER BY bounced_at DESC LIMIT 5;" + +# Check member bounce counts +sudo docker-compose exec mysql mysql -u maillist -pmaillist maillist -e "SELECT email, bounce_count, last_bounce_at, active FROM members WHERE bounce_count > 0;" +``` + +## Monitoring and Maintenance + +### View Processing Logs +```bash +# Monitor bounce processing +sudo docker-compose logs -f postfix | grep -E "(bounce|process-bounce)" + +# Check API bounce handling status +curl -H "Authorization: Bearer $API_TOKEN" http://localhost:8000/config +``` + +### Reset Member Bounce Count +```bash +# Via API +curl -X POST http://localhost:8000/members/{member_id}/reset-bounces \ + -H "Authorization: Bearer $API_TOKEN" + +# Via Database +sudo docker-compose exec mysql mysql -u maillist -pmaillist maillist -e "UPDATE members SET bounce_count=0, active=true WHERE email='user@example.com';" +``` + +## Troubleshooting + +### Common Issues + +#### Bounces Not Being Processed +1. Check that `bounces` alias exists: `sudo docker-compose exec postfix cat /etc/aliases | grep bounces` +2. Verify Python script permissions: `sudo docker-compose exec postfix ls -la /etc/postfix/process-bounce.py` +3. Test script manually: `sudo docker-compose exec postfix python3 /etc/postfix/process-bounce.py --test` + +#### Return Path Not Rewritten +1. Check smtp_generic configuration: `sudo docker-compose exec postfix postconf smtp_generic_maps` +2. Verify map file exists: `sudo docker-compose exec postfix ls -la /etc/postfix/smtp_generic*` +3. Test mapping: `sudo docker-compose exec postfix postmap -q "test@lists.sasalliance.org" hash:/etc/postfix/smtp_generic` + +#### Database Connection Issues +1. Check PyMySQL installation: `sudo docker-compose exec postfix python3 -c "import pymysql; print('OK')"` +2. Test database connection: `sudo docker-compose exec postfix python3 -c "import pymysql; pymysql.connect(host='mysql', user='maillist', password='your_password', database='maillist')"` +3. Verify network connectivity: `sudo docker-compose exec postfix ping mysql` + +### Log Analysis +```bash +# Postfix logs +sudo docker-compose logs postfix | grep -E "(bounce|MAILER-DAEMON|process-bounce)" + +# MySQL connection logs +sudo docker-compose logs postfix | grep -E "(pymysql|mysql)" + +# SES relay logs +sudo docker-compose logs postfix | grep -E "(relay|sent|deferred)" +``` + +## Security Considerations + +- Bounce processing script runs with limited privileges +- Database credentials secured in environment variables +- Bounce emails contain sensitive delivery information - logs are rotated +- Return path rewriting prevents bounce loops +- Processing script validates email format before database updates + +## Performance Notes + +- Bounce processing is asynchronous (doesn't block email delivery) +- Database queries are indexed for bounce lookups +- Bounce logs should be periodically archived for large volumes +- SMTP generic maps are cached by Postfix for performance + +## Advantages over SNS Webhooks + +- **Works with SES Sandbox**: No production SES access required +- **No External Dependencies**: Doesn't require SNS, webhooks, or HTTPS domains +- **Self-Contained**: All processing happens within existing containers +- **Real-time Processing**: Bounces processed as emails arrive +- **Compatible**: Uses same database schema and UI as SNS method \ No newline at end of file diff --git a/README.md b/README.md index db6cd72..d862c06 100644 --- a/README.md +++ b/README.md @@ -176,7 +176,7 @@ docker-compose exec postfix postmap -q "community@lists.sasalliance.org" \ **Email bounce handling is optional and disabled by default.** -**Two Configuration Options:** +**Three Configuration Options:** 1. **SNS Webhooks** (Requires SES Production Access): ```bash @@ -184,21 +184,33 @@ docker-compose exec postfix postmap -q "community@lists.sasalliance.org" \ ENABLE_SNS_WEBHOOKS=true ENABLE_BOUNCE_HANDLING=true ``` - - Real-time bounce notifications via AWS SNS + - Real-time bounce notifications via AWS SNS webhooks - Automatic member deactivation for hard bounces - - Bounce history tracking and management + - Bounce history tracking and management in web UI - Requires valid HTTPS domain and SES production access - See `BOUNCE_HANDLING_SETUP.md` for complete setup -2. **Email-Based Handling** (Default for SES Sandbox): +2. **Email-Based Processing** (Works with SES Sandbox): + ```bash + # In .env file + ENABLE_EMAIL_BOUNCE_PROCESSING=true + ENABLE_BOUNCE_HANDLING=true # Automatically enabled + ``` + - Automatic bounce processing via email parsing + - Same bounce management features as SNS webhooks + - Works with SES sandbox accounts (no production access needed) + - Self-contained processing within existing containers + - See `EMAIL_BOUNCE_HANDLING_SETUP.md` for complete setup + +3. **Disabled** (Manual Processing Only): ```bash # In .env file (or leave these commented out) ENABLE_SNS_WEBHOOKS=false ENABLE_BOUNCE_HANDLING=false + ENABLE_EMAIL_BOUNCE_PROCESSING=false ``` + - No automatic bounce processing - Manual bounce management via email notifications - - No automatic processing - requires manual member cleanup - - Works with SES sandbox accounts - Bounce-related UI elements are hidden **When bounce handling is disabled:** diff --git a/api/main.py b/api/main.py index 77a31b8..5dcd558 100644 --- a/api/main.py +++ b/api/main.py @@ -46,6 +46,10 @@ MYSQL_PASSWORD = os.getenv('MYSQL_PASSWORD', '') # Bounce handling configuration ENABLE_SNS_WEBHOOKS = os.getenv('ENABLE_SNS_WEBHOOKS', 'false').lower() == 'true' ENABLE_BOUNCE_HANDLING = os.getenv('ENABLE_BOUNCE_HANDLING', 'false').lower() == 'true' +ENABLE_EMAIL_BOUNCE_PROCESSING = os.getenv('ENABLE_EMAIL_BOUNCE_PROCESSING', 'false').lower() == 'true' + +# Enable bounce handling if either method is enabled +ENABLE_BOUNCE_HANDLING = ENABLE_BOUNCE_HANDLING or ENABLE_SNS_WEBHOOKS or ENABLE_EMAIL_BOUNCE_PROCESSING # Password hashing pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto") @@ -472,7 +476,8 @@ async def get_config(): """Get public configuration settings""" return { "bounce_handling_enabled": ENABLE_BOUNCE_HANDLING, - "sns_webhooks_enabled": ENABLE_SNS_WEBHOOKS + "sns_webhooks_enabled": ENABLE_SNS_WEBHOOKS, + "email_bounce_processing_enabled": ENABLE_EMAIL_BOUNCE_PROCESSING } # Mailing Lists endpoints diff --git a/postfix/Dockerfile b/postfix/Dockerfile index 478d7bd..05d1856 100644 --- a/postfix/Dockerfile +++ b/postfix/Dockerfile @@ -9,6 +9,8 @@ RUN apt-get update && \ mailutils \ gettext-base \ netcat-openbsd \ + python3 \ + python3-pymysql \ && apt-get clean && rm -rf /var/lib/apt/lists/* # Copy configs @@ -16,11 +18,16 @@ COPY main.cf.template /etc/postfix/main.cf.template COPY sasl_passwd.template /etc/postfix/sasl_passwd.template COPY mysql_virtual_alias_maps.cf /etc/postfix/mysql_virtual_alias_maps.cf.template COPY sender_access /etc/postfix/sender_access +COPY smtp_generic /etc/postfix/smtp_generic +COPY aliases /etc/aliases +COPY process-bounce.py /usr/local/bin/process-bounce.py COPY entrypoint.sh /entrypoint.sh -RUN chmod +x /entrypoint.sh +RUN chmod +x /entrypoint.sh /usr/local/bin/process-bounce.py -# Generate Postfix maps for sender access -RUN postmap /etc/postfix/sender_access +# Generate Postfix maps for sender access, sender canonical, and aliases +RUN postmap /etc/postfix/sender_access && \ + postmap /etc/postfix/smtp_generic && \ + newaliases # Expose SMTP EXPOSE 25 diff --git a/postfix/aliases b/postfix/aliases new file mode 100644 index 0000000..83edb91 --- /dev/null +++ b/postfix/aliases @@ -0,0 +1,17 @@ +# Postfix aliases for bounce handling +# +# This file defines how bounces are processed +# The bounces address pipes messages to our bounce processing script + +# Bounce processing - pipe to Python script +bounces: "|/usr/bin/python3 /usr/local/bin/process-bounce.py" + +# Standard aliases +postmaster: root +mailer-daemon: postmaster +abuse: postmaster +spam: postmaster +root: postmaster + +# Default fallback +MAILER-DAEMON: bounces \ No newline at end of file diff --git a/postfix/entrypoint.sh b/postfix/entrypoint.sh index b88cec3..25e32a3 100644 --- a/postfix/entrypoint.sh +++ b/postfix/entrypoint.sh @@ -32,5 +32,28 @@ chmod 644 /etc/postfix/sender_access /etc/postfix/sender_access.db # Set permissions on MySQL config chmod 644 /etc/postfix/mysql_virtual_alias_maps.cf +# Configure bounce processing based on environment variable +if [ "${ENABLE_EMAIL_BOUNCE_PROCESSING:-false}" = "true" ]; then + echo "Email-based bounce processing enabled" + # Regenerate aliases database to enable bounce processing + newaliases + chmod 644 /etc/aliases /etc/aliases.db + + # Generate smtp generic maps to ensure bounces come to our bounce address + postmap /etc/postfix/smtp_generic + chmod 644 /etc/postfix/smtp_generic /etc/postfix/smtp_generic.db + echo "Configured return path rewriting to bounces@lists.sasalliance.org" +else + echo "Email-based bounce processing disabled" + # Create minimal aliases without bounce processing + echo "postmaster: root" > /etc/aliases + echo "root: postmaster" >> /etc/aliases + newaliases + + # Disable smtp generic maps + echo "# Email bounce processing disabled" > /etc/postfix/smtp_generic + postmap /etc/postfix/smtp_generic +fi + # Start Postfix in foreground exec postfix start-fg diff --git a/postfix/main.cf.template b/postfix/main.cf.template index 1e952c7..087e278 100644 --- a/postfix/main.cf.template +++ b/postfix/main.cf.template @@ -33,3 +33,23 @@ smtpd_recipient_restrictions = # Other recommended settings alias_maps = hash:/etc/aliases alias_database = hash:/etc/aliases + +# Bounce handling configuration for email-based processing +# Configure bounce notification recipients +bounce_notice_recipient = bounces@lists.sasalliance.org +2bounce_notice_recipient = bounces@lists.sasalliance.org +delay_notice_recipient = +error_notice_recipient = bounces@lists.sasalliance.org + +# Bounce settings +bounce_size_limit = 50000 +bounce_queue_lifetime = 5d +maximal_bounce_delay = 1d + +# Return path configuration - CRITICAL for bounce handling +# This ensures that when we relay emails via mailing lists through SES, +# bounces come back to our bounce processing address +# +# Use smtp_generic_maps instead of sender_canonical_maps because we only want +# to rewrite the return path for outbound SMTP (via SES), not for local delivery +smtp_generic_maps = hash:/etc/postfix/smtp_generic diff --git a/postfix/process-bounce.py b/postfix/process-bounce.py new file mode 100644 index 0000000..a74cb01 --- /dev/null +++ b/postfix/process-bounce.py @@ -0,0 +1,348 @@ +#!/usr/bin/env python3 +""" +Email-based bounce processing script for Postfix +Parses bounce emails and updates the database with bounce information + +This script is called by Postfix when bounce emails are received. +It reads the email from stdin, parses it for bounce information, +and updates the database accordingly. +""" + +import sys +import os +import re +import email +from email.message import Message +import logging +from datetime import datetime +from typing import List, Tuple, Optional, Dict +import pymysql + +# Configure logging +log_file = '/var/log/bounce-processor.log' +log_handlers = [logging.StreamHandler()] + +# Add file handler if we can write to the log directory +try: + log_handlers.append(logging.FileHandler(log_file)) +except (PermissionError, FileNotFoundError): + pass # Just use stdout if we can't write to log file + +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + handlers=log_handlers +) +logger = logging.getLogger('bounce-processor') + +# Database configuration from environment +MYSQL_HOST = os.getenv('MYSQL_HOST', 'mysql') +MYSQL_PORT = int(os.getenv('MYSQL_PORT', 3306)) +MYSQL_DATABASE = os.getenv('MYSQL_DATABASE', 'maillist') +MYSQL_USER = os.getenv('MYSQL_USER', 'maillist') +MYSQL_PASSWORD = os.getenv('MYSQL_PASSWORD', '') + +class BounceProcessor: + """Processes bounce emails and updates the database""" + + def __init__(self): + self.bounce_patterns = self._compile_bounce_patterns() + + def _compile_bounce_patterns(self) -> List[re.Pattern]: + """Compile regex patterns for detecting bounce information""" + patterns = [ + # Standard bounce formats + re.compile(r'550.*?([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})', re.IGNORECASE), + re.compile(r'554.*?([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})', re.IGNORECASE), + re.compile(r'553.*?([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})', re.IGNORECASE), + re.compile(r'552.*?([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})', re.IGNORECASE), + re.compile(r'551.*?([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})', re.IGNORECASE), + + # Delivery Status Notification (DSN) format + re.compile(r'Final-Recipient:.*?([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})', re.IGNORECASE), + re.compile(r'Original-Recipient:.*?([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})', re.IGNORECASE), + + # Common bounce message patterns + re.compile(r'user.*?([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}).*?unknown', re.IGNORECASE), + re.compile(r'([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}).*?does not exist', re.IGNORECASE), + re.compile(r'([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}).*?not found', re.IGNORECASE), + re.compile(r'([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}).*?mailbox.*?unavailable', re.IGNORECASE), + re.compile(r'([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}).*?recipient.*?rejected', re.IGNORECASE), + + # Generic email extraction (fallback) + re.compile(r'([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})', re.IGNORECASE), + ] + return patterns + + def parse_bounce_email(self, email_content: str) -> List[Dict]: + """Parse bounce email and extract bounce information""" + try: + # Parse the email + msg = email.message_from_string(email_content) + bounces = [] + + # Get the email body + body = self._get_email_body(msg) + if not body: + logger.warning("No email body found") + return bounces + + logger.info(f"Processing email body (first 500 chars): {body[:500]}") + + # Extract bounced addresses + bounced_addresses = self._extract_bounced_addresses(body) + + # Determine bounce type and create bounce records + for address in bounced_addresses: + bounce_info = self._analyze_bounce(body, address) + if bounce_info: + bounces.append(bounce_info) + + return bounces + + except Exception as e: + logger.error(f"Error parsing bounce email: {str(e)}") + return [] + + def _get_email_body(self, msg: Message) -> Optional[str]: + """Extract the email body from the message""" + body = "" + + if msg.is_multipart(): + for part in msg.walk(): + content_type = part.get_content_type() + if content_type in ['text/plain', 'text/html']: + payload = part.get_payload(decode=True) + if payload: + body += payload.decode('utf-8', errors='ignore') + "\n" + else: + payload = msg.get_payload(decode=True) + if payload: + body = payload.decode('utf-8', errors='ignore') + + return body.strip() if body else None + + def _extract_bounced_addresses(self, body: str) -> List[str]: + """Extract bounced email addresses from the bounce message""" + addresses = [] + + for pattern in self.bounce_patterns: + matches = pattern.findall(body) + for match in matches: + email_addr = match.strip().lower() + if self._is_valid_email(email_addr) and email_addr not in addresses: + # Skip our own addresses + if not email_addr.endswith('@lists.sasalliance.org'): + addresses.append(email_addr) + + logger.info(f"Extracted addresses: {addresses}") + return addresses + + def _is_valid_email(self, email_addr: str) -> bool: + """Validate email address format""" + email_pattern = re.compile(r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$') + return bool(email_pattern.match(email_addr)) + + def _analyze_bounce(self, body: str, email_addr: str) -> Optional[Dict]: + """Analyze bounce message to determine bounce type and details""" + bounce_info = { + 'email': email_addr, + 'bounce_type': 'Undetermined', + 'bounce_subtype': '', + 'diagnostic_code': '', + 'timestamp': datetime.now().strftime('%Y-%m-%d %H:%M:%S') + } + + # Analyze bounce type based on SMTP codes and message content + if re.search(r'5[0-9]{2}', body): # 5xx codes are permanent failures + bounce_info['bounce_type'] = 'Permanent' + + if re.search(r'550|551|553', body): + bounce_info['bounce_subtype'] = 'General' + elif re.search(r'552', body): + bounce_info['bounce_subtype'] = 'MailboxFull' + elif re.search(r'554', body): + bounce_info['bounce_subtype'] = 'MessageTooLarge' + + elif re.search(r'4[0-9]{2}', body): # 4xx codes are temporary failures + bounce_info['bounce_type'] = 'Transient' + + if re.search(r'450|451|452', body): + bounce_info['bounce_subtype'] = 'General' + elif re.search(r'452', body): + bounce_info['bounce_subtype'] = 'MailboxFull' + + # Extract diagnostic code + smtp_code_match = re.search(r'([45][0-9]{2}.*?)[\r\n]', body) + if smtp_code_match: + bounce_info['diagnostic_code'] = smtp_code_match.group(1).strip()[:500] # Limit length + + return bounce_info + + def update_database(self, bounces: List[Dict]) -> None: + """Update the database with bounce information""" + if not bounces: + logger.info("No bounces to process") + return + + try: + connection = pymysql.connect( + host=MYSQL_HOST, + port=MYSQL_PORT, + database=MYSQL_DATABASE, + user=MYSQL_USER, + password=MYSQL_PASSWORD, + cursorclass=pymysql.cursors.DictCursor + ) + + cursor = connection.cursor() + + for bounce in bounces: + try: + email_addr = bounce['email'] + + # Find member by email + cursor.execute("SELECT member_id FROM members WHERE email = %s", (email_addr,)) + member = cursor.fetchone() + member_id = member['member_id'] if member else None + + # Log the bounce + cursor.execute(""" + INSERT INTO bounce_logs + (member_id, email, bounce_type, bounce_subtype, diagnostic_code, timestamp, created_at) + VALUES (%s, %s, %s, %s, %s, %s, NOW()) + """, (member_id, email_addr, bounce['bounce_type'], bounce['bounce_subtype'], + bounce['diagnostic_code'], bounce['timestamp'])) + + logger.info(f"Logged bounce for {email_addr}: {bounce['bounce_type']}") + + # Update member bounce status if member exists + if member_id: + if bounce['bounce_type'] == 'Permanent': + # Hard bounce - deactivate member + cursor.execute(""" + UPDATE members + SET bounce_count = bounce_count + 1, + last_bounce_at = %s, + bounce_status = 'hard_bounce', + active = 0 + WHERE member_id = %s + """, (bounce['timestamp'], member_id)) + logger.info(f"Deactivated member {email_addr} due to hard bounce") + + elif bounce['bounce_type'] == 'Transient': + # Soft bounce - increment counter and check threshold + cursor.execute("SELECT bounce_count, bounce_status FROM members WHERE member_id = %s", (member_id,)) + current = cursor.fetchone() + + if current and current['bounce_status'] != 'hard_bounce': + new_count = current['bounce_count'] + 1 + new_status = 'soft_bounce' if new_count >= 3 else 'clean' + + cursor.execute(""" + UPDATE members + SET bounce_count = %s, + last_bounce_at = %s, + bounce_status = %s + WHERE member_id = %s + """, (new_count, bounce['timestamp'], new_status, member_id)) + + logger.info(f"Updated member {email_addr} bounce count to {new_count}, status: {new_status}") + else: + # Undetermined - just increment counter + cursor.execute(""" + UPDATE members + SET bounce_count = bounce_count + 1, + last_bounce_at = %s + WHERE member_id = %s + """, (bounce['timestamp'], member_id)) + + logger.info(f"Updated member {email_addr} bounce count (undetermined)") + + except Exception as e: + logger.error(f"Database error processing bounce for {bounce['email']}: {str(e)}") + continue + + connection.commit() + cursor.close() + connection.close() + + logger.info(f"Successfully processed {len(bounces)} bounces") + + except pymysql.Error as e: + logger.error(f"Database connection error: {str(e)}") + except Exception as e: + logger.error(f"Unexpected error updating database: {str(e)}") + +def main(): + """Main function - reads email from stdin and processes bounces""" + try: + logger.info("Starting bounce processing") + + # Check if we're in test mode + test_mode = len(sys.argv) > 1 and sys.argv[1] == '--test' + + if test_mode: + # Test mode - use sample bounce email + email_content = """From: Mail Delivery Subsystem +To: bounces@lists.sasalliance.org +Subject: Delivery Status Notification (Failure) +Content-Type: multipart/report; report-type=delivery-status; boundary="boundary123" + +--boundary123 +Content-Type: text/plain + +This is a test bounce message. + +The following address(es) failed: + testuser@example.com + SMTP error from remote mail server after RCPT TO:: + 550 5.1.1 User unknown + +--boundary123 +Content-Type: message/delivery-status + +Reporting-MTA: dns; mail.example.com +Received-From-MTA: dns; localhost + +Final-Recipient: rfc822; testuser@example.com +Action: failed +Status: 5.1.1 +Diagnostic-Code: smtp; 550 5.1.1 User unknown + +--boundary123-- +""" + logger.info("Running in test mode with sample bounce email") + else: + # Read email from stdin + email_content = sys.stdin.read() + + if not email_content.strip(): + logger.warning("No email content received") + return + + logger.info(f"Received email content ({len(email_content)} bytes)") + + # Process the bounce + processor = BounceProcessor() + bounces = processor.parse_bounce_email(email_content) + + if test_mode: + logger.info(f"Test mode - would process {len(bounces)} bounce(s):") + for bounce in bounces: + logger.info(f" {bounce}") + else: + if bounces: + processor.update_database(bounces) + logger.info(f"Processed {len(bounces)} bounce(s)") + else: + logger.info("No bounces detected in email") + + except Exception as e: + logger.error(f"Error in main: {str(e)}") + import traceback + logger.error(f"Traceback: {traceback.format_exc()}") + sys.exit(1) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/postfix/smtp_generic b/postfix/smtp_generic new file mode 100644 index 0000000..d4a48a5 --- /dev/null +++ b/postfix/smtp_generic @@ -0,0 +1,23 @@ +# Postfix SMTP Generic Maps +# This file controls the return path (envelope sender) for outbound SMTP emails +# Only applies to emails being relayed through external SMTP (SES in our case) +# +# Format: original_sender rewritten_sender +# +# For mailing list emails, we want bounces to go to our bounce processing address +# This is critical for email-based bounce handling to work properly + +# Rewrite envelope sender for all mailing list addresses to bounce address +# When emails are forwarded through mailing lists via SES, bounces come back to us +community@lists.sasalliance.org bounces@lists.sasalliance.org +board@lists.sasalliance.org bounces@lists.sasalliance.org +members@lists.sasalliance.org bounces@lists.sasalliance.org +announcements@lists.sasalliance.org bounces@lists.sasalliance.org + +# Generic pattern - any @lists.sasalliance.org sender gets rewritten to bounce address +# This catches any new lists automatically +@lists.sasalliance.org bounces@lists.sasalliance.org + +# Also rewrite any envelope sender that's sending through our system +# This ensures ALL outgoing mail via SES has our bounce address as return path +@sasalliance.org bounces@lists.sasalliance.org \ No newline at end of file diff --git a/postfix/virtual_aliases.cf b/postfix/virtual_aliases.cf deleted file mode 100644 index 46fb701..0000000 --- a/postfix/virtual_aliases.cf +++ /dev/null @@ -1,11 +0,0 @@ -# Community mailing list - general announcements -community@lists.sasalliance.org james@pattinson.org, james.pattinson@sasalliance.org - -# Board members mailing list -board@lists.sasalliance.org james.pattinson@sasalliance.org - -# All members mailing list -members@lists.sasalliance.org james@pattinson.org, james.pattinson@sasalliance.org - -# Announcements mailing list -announcements@lists.sasalliance.org james@pattinson.org, james.pattinson@sasalliance.org \ No newline at end of file