diff --git a/SES_BOUNCE_TESTING_GUIDE.md b/SES_BOUNCE_TESTING_GUIDE.md new file mode 100644 index 0000000..7f8e799 --- /dev/null +++ b/SES_BOUNCE_TESTING_GUIDE.md @@ -0,0 +1,398 @@ +# Testing Bounces from SES Sandbox + +AWS SES provides **built-in bounce simulator addresses** that work even in sandbox mode. This guide shows you how to use them to test your bounce handling. + +## Quick Answer + +Send email to these special AWS addresses to simulate different bounce types: + +### Hard Bounce (Permanent - Invalid Address) +```bash +bounce@simulator.amazonses.com +``` + +### Soft Bounce (Temporary - Mailbox Full) +```bash +ooto@simulator.amazonses.com +``` + +### Complaint (Spam Report) +```bash +complaint@simulator.amazonses.com +``` + +### Successful Delivery (No Bounce) +```bash +success@simulator.amazonses.com +``` + +## Step-by-Step Testing Guide + +### Option 1: Using Your Mailing Lists (Recommended) + +This tests the complete flow: Postfix → SES → SNS → API + +1. **Add the simulator address as a member:** + ```bash + # Using the API + curl -X POST http://localhost:8000/members \ + -H "Authorization: Bearer $API_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "name": "Bounce Test", + "email": "bounce@simulator.amazonses.com", + "active": true + }' + ``` + + Or use the Web UI: + - Go to http://localhost:3000 + - Members tab → Add Member + - Name: `Bounce Test` + - Email: `bounce@simulator.amazonses.com` + +2. **Subscribe to a test list:** + - Click "Subscriptions" button for the test member + - Toggle on one of your mailing lists + +3. **Send email to the list:** + ```bash + # From inside Postfix container + sudo docker compose exec postfix bash + echo "This will bounce" | mail -s "Test Bounce" community@lists.sasalliance.org + exit + ``` + + Replace `community@lists.sasalliance.org` with your actual list email. + +4. **Wait 30-60 seconds** for: + - Email to be sent via SES + - SES to process the bounce + - SNS to send notification to your webhook + +5. **Check the results:** + + **Watch API logs in real-time:** + ```bash + sudo docker compose logs api -f + ``` + + You should see: + ``` + ============================================================ + SNS Webhook Request Received + ============================================================ + Content-Type: text/plain; charset=UTF-8 + User-Agent: Amazon Simple Notification Service Agent + + ✓ Notification Received + Notification Type: Bounce + + ✓ Processing Bounce + Bounce Type: Permanent + Recipients: ['bounce@simulator.amazonses.com'] + ✓ Bounce processed successfully + ============================================================ + ``` + + **Check the database:** + ```bash + # View bounce logs + sudo docker compose exec mysql mysql -u maillist -pmaillist maillist \ + -e "SELECT * FROM bounce_logs ORDER BY created_at DESC LIMIT 5;" + + # Check member status + sudo docker compose exec mysql mysql -u maillist -pmaillist maillist \ + -e "SELECT email, active, bounce_count, bounce_status FROM members WHERE email='bounce@simulator.amazonses.com';" + ``` + + **View in Web UI:** + - Open http://localhost:3000 + - Go to Members tab + - Find "Bounce Test" member + - Should show: ❌ Inactive (red), bounce badge, last bounce timestamp + - Click "Bounces" button to see detailed history + +### Option 2: Direct Email via Postfix (Simpler) + +Send directly to the simulator without going through your list: + +```bash +# Enter Postfix container +sudo docker compose exec postfix bash + +# Send test email +echo "Testing hard bounce" | mail -s "Hard Bounce Test" bounce@simulator.amazonses.com + +# Or soft bounce +echo "Testing soft bounce" | mail -s "Soft Bounce Test" ooto@simulator.amazonses.com + +# Exit container +exit +``` + +### Option 3: Using AWS SES Console (For Non-Sandbox Testing) + +If you have SES production access: + +1. Go to AWS SES Console +2. Click "Send test email" +3. To: `bounce@simulator.amazonses.com` +4. From: Your verified email/domain +5. Subject: "Bounce test" +6. Body: "Testing bounce handling" +7. Click "Send test email" + +## Testing Different Bounce Types + +### 1. Hard Bounce (Permanent Failure) +```bash +echo "Test" | mail -s "Test" bounce@simulator.amazonses.com +``` + +**Expected Result:** +- Member marked as `hard_bounce` +- Member deactivated (`active = 0`) +- `bounce_count` incremented +- Entry in `bounce_logs` table + +### 2. Soft Bounce (Transient Failure) +```bash +echo "Test" | mail -s "Test" ooto@simulator.amazonses.com +``` + +**Expected Result:** +- Member marked as `clean` (first time) +- After 3 soft bounces → `soft_bounce` status +- `bounce_count` incremented +- Member stays active + +### 3. Complaint (Spam Report) +```bash +echo "Test" | mail -s "Test" complaint@simulator.amazonses.com +``` + +**Expected Result:** +- API receives complaint notification +- Currently logged but not processed (you can extend the handler) + +## Monitoring the Test + +### Real-Time Monitoring (Recommended) + +Open 3 terminal windows: + +**Terminal 1 - API Logs:** +```bash +sudo docker compose logs api -f +``` + +**Terminal 2 - Postfix Logs:** +```bash +sudo docker compose logs postfix -f +``` + +**Terminal 3 - Send Test Email:** +```bash +sudo docker compose exec postfix bash +echo "Test" | mail -s "Bounce Test" bounce@simulator.amazonses.com +``` + +### Timeline + +Here's what happens and when: + +- **T+0s**: Email sent to Postfix +- **T+1-3s**: Postfix relays to SES +- **T+5-10s**: SES processes and generates bounce +- **T+10-30s**: SNS sends notification to your webhook +- **T+30-60s**: API processes bounce and updates database + +## Verifying the Complete Flow + +### 1. Check Postfix Logs +```bash +sudo docker compose logs postfix | grep bounce@simulator +``` + +Should show: +``` +postfix/smtp[xxx]: ... to=, relay=email-smtp.eu-west-2.amazonaws.com[...], ... status=sent +``` + +### 2. Check SNS Subscription Status +- Go to AWS SNS Console +- Find your topic +- Check "Subscriptions" tab +- Status should be "Confirmed" +- Messages delivered should be > 0 + +### 3. Check API Logs +```bash +sudo docker compose logs api | grep -A 20 "SNS Webhook" +``` + +Should show successful processing. + +### 4. Check Database +```bash +sudo docker compose exec mysql mysql -u maillist -pmaillist maillist < 0; +EOF +``` + +## Troubleshooting + +### "Email not sent" or "Relay access denied" + +**Problem**: Postfix not configured to relay via SES + +**Check**: +```bash +sudo docker compose exec postfix postconf relayhost +sudo docker compose exec postfix postconf smtp_sasl_auth_enable +``` + +Should show: +``` +relayhost = [email-smtp.eu-west-2.amazonaws.com]:587 +smtp_sasl_auth_enable = yes +``` + +### "No bounce received after 5 minutes" + +**Possible causes**: + +1. **SNS subscription not confirmed** + - Check AWS SNS console + - Status should be "Confirmed", not "Pending" + +2. **SNS topic not configured in SES** + - Check SES → Configuration Sets or Verified Identities → Notifications + - Bounce notifications should point to your SNS topic + +3. **Webhook endpoint not accessible** + - SNS requires HTTPS + - Test: `curl https://your-domain.com:8000/health` + +4. **API container not running** + ```bash + sudo docker compose ps api + ``` + +### "Bounce received but not in database" + +**Check API logs for errors**: +```bash +sudo docker compose logs api | grep -i error +``` + +**Check database tables exist**: +```bash +sudo docker compose exec mysql mysql -u maillist -pmaillist maillist -e "SHOW TABLES;" +``` + +Should include: `bounce_logs`, `members` + +## Testing Multiple Bounces + +To test the "3 soft bounces = soft_bounce status" logic: + +```bash +sudo docker compose exec postfix bash + +# Send 3 emails to soft bounce simulator +for i in {1..3}; do + echo "Soft bounce test $i" | mail -s "Test $i" ooto@simulator.amazonses.com + sleep 70 # Wait between sends for SNS processing +done +``` + +After the 3rd bounce: +- Member's `bounce_status` should change from `clean` to `soft_bounce` +- `bounce_count` should be 3 + +## Cleanup After Testing + +Remove test bounce data: + +```bash +sudo docker compose exec mysql mysql -u maillist -pmaillist maillist < 0;" +``` + +## Common SNS Message Types + +### 1. SubscriptionConfirmation +First message when you create the subscription. API auto-confirms by calling SubscribeURL. + +### 2. Notification (Bounce) +```json +{ + "Type": "Notification", + "Message": "{\"notificationType\":\"Bounce\",\"bounce\":{...}}" +} +``` + +### 3. Notification (Complaint) +```json +{ + "Type": "Notification", + "Message": "{\"notificationType\":\"Complaint\",\"complaint\":{...}}" +} +``` + +## Next Steps + +1. **Monitor logs while SNS sends**: `sudo docker compose logs api -f` +2. **Trigger a real bounce**: Send to `bounce@simulator.amazonses.com` +3. **Check the detailed logs** to see exactly what SNS is sending +4. **Verify database updates** to confirm bounces are being recorded + +## If You Still See Errors + +The enhanced logging will now show you: +- What headers SNS is sending +- What body content is arriving (or if it's empty) +- Where in the processing pipeline the error occurs +- Full stack traces for any exceptions + +Copy the relevant log section and we can diagnose the exact issue! diff --git a/api/main.py b/api/main.py index 2fabb1a..105b9a4 100644 --- a/api/main.py +++ b/api/main.py @@ -826,7 +826,16 @@ async def verify_sns_signature(request: Request) -> dict: """Verify SNS message signature""" try: body = await request.body() + print(f"SNS webhook received body: {body}") + print(f"SNS webhook body length: {len(body)}") + print(f"SNS webhook headers: {dict(request.headers)}") + + if not body: + print("ERROR: Empty body received") + raise HTTPException(status_code=400, detail="Empty request body") + message = json.loads(body.decode('utf-8')) + print(f"SNS webhook parsed message type: {message.get('Type')}") # For SubscriptionConfirmation and UnsubscribeConfirmation, we don't validate signature # AWS will send a URL to confirm @@ -901,9 +910,15 @@ async def process_bounce(bounce_data: dict): try: bounce_type = bounce_data.get('bounceType') # Permanent, Transient, Undetermined bounce_subtype = bounce_data.get('bounceSubType', '') - timestamp = bounce_data.get('timestamp') + timestamp_str = bounce_data.get('timestamp') feedback_id = bounce_data.get('feedbackId', '') + # Convert ISO 8601 timestamp to MySQL datetime format + # SES sends: '2025-10-13T16:22:40.359Z' + # MySQL needs: '2025-10-13 16:22:40' + from datetime import datetime as dt + timestamp = dt.fromisoformat(timestamp_str.replace('Z', '+00:00')).strftime('%Y-%m-%d %H:%M:%S') + bounced_recipients = bounce_data.get('bouncedRecipients', []) with get_db() as conn: @@ -972,48 +987,75 @@ async def process_bounce(bounce_data: dict): cursor.close() except Exception as e: - print(f"Error processing bounce: {str(e)}") + print(f"✗ Error processing bounce: {str(e)}") + print(f"Error type: {type(e).__name__}") + print(f"Bounce data: {bounce_data}") + import traceback + traceback.print_exc() raise @app.post("/webhooks/sns", response_class=PlainTextResponse) async def sns_webhook(request: Request): """Handle SNS notifications for bounces and complaints""" try: + print(f"=== SNS Webhook Request ===") + print(f"Headers: {dict(request.headers)}") + print(f"Content-Type: {request.headers.get('content-type')}") + print(f"User-Agent: {request.headers.get('user-agent')}") + # Verify SNS signature message = await verify_sns_signature(request) + print(f"Message Type: {message.get('Type')}") + print(f"Message Keys: {list(message.keys())}") + message_type = message.get('Type') # Handle subscription confirmation if message_type == 'SubscriptionConfirmation': subscribe_url = message.get('SubscribeURL') + print(f"Subscription confirmation received, URL: {subscribe_url}") if subscribe_url: # Confirm subscription async with httpx.AsyncClient() as client: - await client.get(subscribe_url) + response = await client.get(subscribe_url) + print(f"Subscription confirmation response: {response.status_code}") return "Subscription confirmed" # Handle notification elif message_type == 'Notification': # Parse the message - notification = json.loads(message.get('Message', '{}')) - notification_type = notification.get('notificationType') + inner_message = message.get('Message', '{}') + print(f"Inner message (first 500 chars): {inner_message[:500]}") + notification = json.loads(inner_message) + + # SES can send either 'notificationType' or 'eventType' depending on configuration + notification_type = notification.get('notificationType') or notification.get('eventType') + print(f"Notification type: {notification_type}") if notification_type == 'Bounce': bounce = notification.get('bounce', {}) + print(f"\n✓ Processing Bounce") + print(f" Bounce Type: {bounce.get('bounceType')}") + print(f" Recipients: {[r.get('emailAddress') for r in bounce.get('bouncedRecipients', [])]}") await process_bounce(bounce) + print(f" ✓ Bounce processed successfully") return "Bounce processed" elif notification_type == 'Complaint': # We could also track complaints similarly to bounces + print(f"\n✓ Complaint received") return "Complaint received" + print(f"=== End SNS Webhook Request ===") return "OK" except HTTPException: raise except Exception as e: print(f"SNS webhook error: {str(e)}") + import traceback + traceback.print_exc() raise HTTPException(status_code=500, detail=str(e)) # Bounce management endpoints diff --git a/test_bounce_quick.sh b/test_bounce_quick.sh new file mode 100644 index 0000000..879279a --- /dev/null +++ b/test_bounce_quick.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Quick SES Bounce Test - One-liner to test bounce handling + +echo "🚀 Testing SES Bounce Handling" +echo "================================" +echo "" + +# Check if API and Postfix are running +if ! sudo docker compose ps | grep -q "maillist-api.*Up"; then + echo "❌ API container not running. Start with: sudo docker compose up -d" + exit 1 +fi + +if ! sudo docker compose ps | grep -q "maillist-postfix.*Up"; then + echo "❌ Postfix container not running. Start with: sudo docker compose up -d" + exit 1 +fi + +echo "✅ Containers are running" +echo "" + +# Start watching logs in background +echo "📋 Opening log viewer (press Ctrl+C to stop)..." +echo "" +sleep 2 + +# Show the command they can run +echo "Run this command to send a test bounce:" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "echo 'Test bounce' | sudo docker compose exec -T postfix mail -s 'Bounce Test' bounce@simulator.amazonses.com" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "" +echo "Then wait 30-60 seconds and check:" +echo "" +echo "1. API logs (in this window)" +echo "2. Database: sudo docker compose exec mysql mysql -u maillist -pmaillist maillist -e 'SELECT * FROM bounce_logs ORDER BY created_at DESC LIMIT 5;'" +echo "3. Web UI: http://localhost:3000 → Members tab" +echo "" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "Watching API logs now..." +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "" + +# Follow logs +sudo docker compose logs api -f --tail 20 diff --git a/web/static/js/api.js b/web/static/js/api.js index 66ddb23..e965c1a 100644 --- a/web/static/js/api.js +++ b/web/static/js/api.js @@ -25,8 +25,8 @@ class APIClient { return `${protocol}//${hostname}:8000`; } - // If running in production, assume API is on port 8000 - return `${protocol}//${hostname}:8000`; + // If running in production behind a reverse proxy, use /api path + return `${protocol}//${hostname}/api`; } /** @@ -104,16 +104,18 @@ class APIClient { return this.request('/'); } - // Authentication API async login(username, password) { + // Don't include Authorization header for login + const tempHeaders = { ...this.headers }; + delete tempHeaders['Authorization']; + const response = await this.request('/auth/login', { method: 'POST', - body: JSON.stringify({ - username: username, - password: password - }) + headers: tempHeaders, + body: JSON.stringify({ username, password }) }); + // Set the token from the response if (response.access_token) { this.setToken(response.access_token); } @@ -126,41 +128,16 @@ class APIClient { await this.request('/auth/logout', { method: 'POST' }); - } catch (error) { - // Ignore logout errors, we'll clear the token anyway + } finally { + // Clear token even if logout fails + this.clearToken(); } - this.clearToken(); } async getCurrentUser() { return this.request('/auth/me'); } - // User management API - async getUsers() { - return this.request('/users'); - } - - async createUser(userData) { - return this.request('/users', { - method: 'POST', - body: JSON.stringify(userData) - }); - } - - async updateUser(userId, userData) { - return this.request(`/users/${userId}`, { - method: 'PATCH', - body: JSON.stringify(userData) - }); - } - - async deleteUser(userId) { - return this.request(`/users/${userId}`, { - method: 'DELETE' - }); - } - // Mailing Lists API async getLists() { return this.request('/lists'); @@ -255,7 +232,32 @@ class APIClient { }); } - // Bounce management API + // User Management API + async getUsers() { + return this.request('/users'); + } + + async createUser(userData) { + return this.request('/users', { + method: 'POST', + body: JSON.stringify(userData) + }); + } + + async updateUser(userId, userData) { + return this.request(`/users/${userId}`, { + method: 'PATCH', + body: JSON.stringify(userData) + }); + } + + async deleteUser(userId) { + return this.request(`/users/${userId}`, { + method: 'DELETE' + }); + } + + // Bounce Management API async getMemberBounces(memberId) { return this.request(`/members/${memberId}/bounces`); }