From 94a9a1479b7531a03fe9115c7ecbe1297ef06a7d Mon Sep 17 00:00:00 2001
From: Son <son@son>
Date: Wed, 13 Oct 2021 10:18:46 +0200
Subject: [PATCH 01/20] add more logging

---
 email_handler.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/email_handler.py b/email_handler.py
index 1f6dda19..40910366 100644
--- a/email_handler.py
+++ b/email_handler.py
@@ -1889,6 +1889,12 @@ async def handle(envelope: Envelope) -> str:
                 return status.E207
             else:
                 return status.E522
+        else:
+            LOG.i(
+                "The rate limit is now removed for mail_from:%s,rcpt_tos:%s ",
+                mail_from,
+                rcpt_tos,
+            )
 
     # Handle "out of office" auto notice. An automatic response is sent for every forwarded email
     # todo: remove logging

From becf789d5e1102dab62187680d0e459dbbed3ee9 Mon Sep 17 00:00:00 2001
From: Son <son@son>
Date: Wed, 13 Oct 2021 10:27:46 +0200
Subject: [PATCH 02/20] Revert "add more logging"

This reverts commit 94a9a1479b7531a03fe9115c7ecbe1297ef06a7d.
---
 email_handler.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/email_handler.py b/email_handler.py
index 40910366..1f6dda19 100644
--- a/email_handler.py
+++ b/email_handler.py
@@ -1889,12 +1889,6 @@ async def handle(envelope: Envelope) -> str:
                 return status.E207
             else:
                 return status.E522
-        else:
-            LOG.i(
-                "The rate limit is now removed for mail_from:%s,rcpt_tos:%s ",
-                mail_from,
-                rcpt_tos,
-            )
 
     # Handle "out of office" auto notice. An automatic response is sent for every forwarded email
     # todo: remove logging

From 0ec4a3971c83671b5f08e801c18bb0cca3e9f444 Mon Sep 17 00:00:00 2001
From: Son <son@son>
Date: Wed, 13 Oct 2021 10:27:54 +0200
Subject: [PATCH 03/20] Revert "sleep for 60s when rate limit is hit for the
 first time"

This reverts commit 2524c8ab98c660e2654652a6ad67e2b76333cb86.
---
 email_handler.py | 45 ++++++++++++++++-----------------------------
 1 file changed, 16 insertions(+), 29 deletions(-)

diff --git a/email_handler.py b/email_handler.py
index 1f6dda19..fea0769f 100644
--- a/email_handler.py
+++ b/email_handler.py
@@ -1858,37 +1858,24 @@ async def handle(envelope: Envelope) -> str:
                 return status.E523
 
     if rate_limited(mail_from, rcpt_tos):
-        LOG.w(
-            "Rate Limiting applied for mail_from:%s rcpt_tos:%s, retry in 60s",
-            mail_from,
-            rcpt_tos,
-        )
-        # slow down the rate a bit
-        await asyncio.sleep(60)
+        LOG.w("Rate Limiting applied for mail_from:%s rcpt_tos:%s", mail_from, rcpt_tos)
 
-        # rate limit is still applied
-        if rate_limited(mail_from, rcpt_tos):
-            LOG.w(
-                "Rate Limiting (no retry) applied for mail_from:%s rcpt_tos:%s",
-                mail_from,
-                rcpt_tos,
-            )
-            # add more logging info. TODO: remove
-            if len(rcpt_tos) == 1:
-                alias = Alias.get_by(email=rcpt_tos[0])
-                if alias:
-                    LOG.w(
-                        "total number email log on %s, %s is %s, %s",
-                        alias,
-                        alias.user,
-                        EmailLog.filter(EmailLog.alias_id == alias.id).count(),
-                        EmailLog.filter(EmailLog.user_id == alias.user_id).count(),
-                    )
+        # add more logging info. TODO: remove
+        if len(rcpt_tos) == 1:
+            alias = Alias.get_by(email=rcpt_tos[0])
+            if alias:
+                LOG.w(
+                    "total number email log on %s, %s is %s, %s",
+                    alias,
+                    alias.user,
+                    EmailLog.filter(EmailLog.alias_id == alias.id).count(),
+                    EmailLog.filter(EmailLog.user_id == alias.user_id).count(),
+                )
 
-            if should_ignore_bounce(envelope.mail_from):
-                return status.E207
-            else:
-                return status.E522
+        if should_ignore_bounce(envelope.mail_from):
+            return status.E207
+        else:
+            return status.E522
 
     # Handle "out of office" auto notice. An automatic response is sent for every forwarded email
     # todo: remove logging

From 68cf54b2d9e57e11c6d89a28df3746606a9635b7 Mon Sep 17 00:00:00 2001
From: Son <son@son>
Date: Wed, 13 Oct 2021 10:27:59 +0200
Subject: [PATCH 04/20] Revert "use async in email handler"

This reverts commit 4d7cd09847641ed1593a12680c05470755878448.
---
 email_handler.py | 39 ++++++++++++++++++++-------------------
 1 file changed, 20 insertions(+), 19 deletions(-)

diff --git a/email_handler.py b/email_handler.py
index fea0769f..39c7a17a 100644
--- a/email_handler.py
+++ b/email_handler.py
@@ -31,7 +31,6 @@ It should contain the following info:
 
 """
 import argparse
-import asyncio
 import email
 import os
 import time
@@ -1712,7 +1711,7 @@ def should_ignore(mail_from: str, rcpt_tos: List[str]) -> bool:
     return False
 
 
-async def handle(envelope: Envelope) -> str:
+def handle(envelope: Envelope) -> str:
     """Return SMTP status"""
 
     # sanitize mail_from, rcpt_tos
@@ -1935,7 +1934,7 @@ async def handle(envelope: Envelope) -> str:
 class MailHandler:
     async def handle_DATA(self, server, session, envelope: Envelope):
         try:
-            ret = await self._handle(envelope)
+            ret = self._handle(envelope)
             return ret
         except Exception:
             LOG.e(
@@ -1946,7 +1945,7 @@ class MailHandler:
             return status.E404
 
     @newrelic.agent.background_task(application=newrelic_app)
-    async def _handle(self, envelope: Envelope):
+    def _handle(self, envelope: Envelope):
         start = time.time()
 
         # generate a different message_id to keep track of an email lifecycle
@@ -1959,21 +1958,23 @@ class MailHandler:
             envelope.rcpt_tos,
         )
 
-        ret = await handle(envelope)
-        elapsed = time.time() - start
-        LOG.i(
-            "Finish mail from %s, rctp tos %s, takes %s seconds <<===",
-            envelope.mail_from,
-            envelope.rcpt_tos,
-            elapsed,
-        )
-        newrelic.agent.record_custom_metric(
-            "Custom/email_handler_time", elapsed, newrelic_app
-        )
-        newrelic.agent.record_custom_metric(
-            "Custom/number_incoming_email", 1, newrelic_app
-        )
-        return ret
+        app = new_app()
+        with app.app_context():
+            ret = handle(envelope)
+            elapsed = time.time() - start
+            LOG.i(
+                "Finish mail from %s, rctp tos %s, takes %s seconds <<===",
+                envelope.mail_from,
+                envelope.rcpt_tos,
+                elapsed,
+            )
+            newrelic.agent.record_custom_metric(
+                "Custom/email_handler_time", elapsed, newrelic_app
+            )
+            newrelic.agent.record_custom_metric(
+                "Custom/number_incoming_email", 1, newrelic_app
+            )
+            return ret
 
 
 def main(port: int):

From e13a974e53a787cf6fda2617ac0739306dff228d Mon Sep 17 00:00:00 2001
From: Son <son@son>
Date: Wed, 13 Oct 2021 10:30:04 +0200
Subject: [PATCH 05/20] disable rate limiting

---
 app/email/rate_limit.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/app/email/rate_limit.py b/app/email/rate_limit.py
index 7e78022e..918db145 100644
--- a/app/email/rate_limit.py
+++ b/app/email/rate_limit.py
@@ -93,6 +93,9 @@ def rate_limited_reply_phase(reply_email: str) -> bool:
 
 
 def rate_limited(mail_from: str, rcpt_tos: [str]) -> bool:
+    # todo: re-enable rate limiting
+    return False
+
     for rcpt_to in rcpt_tos:
         if is_reply_email(rcpt_to):
             if rate_limited_reply_phase(rcpt_to):

From 3a0b125323ed694e0e2cd1fab50d8917b4b6152d Mon Sep 17 00:00:00 2001
From: Son <son@son>
Date: Wed, 13 Oct 2021 10:52:41 +0200
Subject: [PATCH 06/20] fix table name

---
 app/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/models.py b/app/models.py
index f1e09f0b..5e6bc5df 100644
--- a/app/models.py
+++ b/app/models.py
@@ -2557,7 +2557,7 @@ class IgnoredEmail(Base, ModelMixin):
 class IgnoreBounceSender(Base, ModelMixin):
     """Ignore sender that doesn't correctly handle bounces, for example noreply@github.com"""
 
-    __tablename__ = "ignored_bounce_sender"
+    __tablename__ = "ignore_bounce_sender"
 
     mail_from = sa.Column(sa.String(512), nullable=False, unique=True)
 

From fbabe6fb444ad0820c8f07ef3836c23aef829478 Mon Sep 17 00:00:00 2001
From: Son <son@son>
Date: Wed, 13 Oct 2021 11:40:15 +0200
Subject: [PATCH 07/20] use job system for deleting mailbox

---
 app/api/views/mailbox.py       | 15 ++++++++++++---
 app/config.py                  |  1 +
 app/dashboard/views/mailbox.py | 18 ++++++++++++------
 job_runner.py                  | 26 +++++++++++++++++++++++++-
 4 files changed, 50 insertions(+), 10 deletions(-)

diff --git a/app/api/views/mailbox.py b/app/api/views/mailbox.py
index 87f9dc79..d235d390 100644
--- a/app/api/views/mailbox.py
+++ b/app/api/views/mailbox.py
@@ -1,10 +1,12 @@
 from smtplib import SMTPRecipientsRefused
 
+import arrow
 from flask import g
 from flask import jsonify
 from flask import request
 
 from app.api.base import api_bp, require_api_auth
+from app.config import JOB_DELETE_MAILBOX
 from app.dashboard.views.mailbox import send_verification_email
 from app.dashboard.views.mailbox_detail import verify_mailbox_change
 from app.db import Session
@@ -13,7 +15,8 @@ from app.email_utils import (
     email_can_be_used_as_mailbox,
     is_valid_email,
 )
-from app.models import Mailbox
+from app.log import LOG
+from app.models import Mailbox, Job
 from app.utils import sanitize_email
 
 
@@ -88,8 +91,14 @@ def delete_mailbox(mailbox_id):
     if mailbox.id == user.default_mailbox_id:
         return jsonify(error="You cannot delete the default mailbox"), 400
 
-    Mailbox.delete(mailbox_id)
-    Session.commit()
+    # Schedule delete account job
+    LOG.w("schedule delete mailbox job for %s", mailbox)
+    Job.create(
+        name=JOB_DELETE_MAILBOX,
+        payload={"mailbox_id": mailbox.id},
+        run_at=arrow.now(),
+        commit=True,
+    )
 
     return jsonify(deleted=True), 200
 
diff --git a/app/config.py b/app/config.py
index 2553e3f0..69e8ec1c 100644
--- a/app/config.py
+++ b/app/config.py
@@ -255,6 +255,7 @@ JOB_ONBOARDING_3 = "onboarding-3"
 JOB_ONBOARDING_4 = "onboarding-4"
 JOB_BATCH_IMPORT = "batch-import"
 JOB_DELETE_ACCOUNT = "delete-account"
+JOB_DELETE_MAILBOX = "delete-mailbox"
 
 # for pagination
 PAGE_LIMIT = 20
diff --git a/app/dashboard/views/mailbox.py b/app/dashboard/views/mailbox.py
index c69d9a55..ef5bf42d 100644
--- a/app/dashboard/views/mailbox.py
+++ b/app/dashboard/views/mailbox.py
@@ -1,5 +1,4 @@
-from threading import Thread
-
+import arrow
 from flask import render_template, request, redirect, url_for, flash
 from flask_login import login_required, current_user
 from flask_wtf import FlaskForm
@@ -7,7 +6,7 @@ from itsdangerous import Signer
 from wtforms import validators
 from wtforms.fields.html5 import EmailField
 
-from app.config import MAILBOX_SECRET, URL
+from app.config import MAILBOX_SECRET, URL, JOB_DELETE_MAILBOX
 from app.dashboard.base import dashboard_bp
 from app.db import Session
 from app.email_utils import (
@@ -18,7 +17,7 @@ from app.email_utils import (
     is_valid_email,
 )
 from app.log import LOG
-from app.models import Mailbox
+from app.models import Mailbox, Job
 
 
 class NewMailboxForm(FlaskForm):
@@ -51,8 +50,15 @@ def mailbox_route():
                 flash("You cannot delete default mailbox", "error")
                 return redirect(url_for("dashboard.mailbox_route"))
 
-            LOG.d("Schedule deleting %s", mailbox)
-            Thread(target=delete_mailbox, args=(mailbox.id,)).start()
+            # Schedule delete account job
+            LOG.w("schedule delete mailbox job for %s", mailbox)
+            Job.create(
+                name=JOB_DELETE_MAILBOX,
+                payload={"mailbox_id": mailbox.id},
+                run_at=arrow.now(),
+                commit=True,
+            )
+
             flash(
                 f"Mailbox {mailbox.email} scheduled for deletion."
                 f"You will receive a confirmation email when the deletion is finished",
diff --git a/job_runner.py b/job_runner.py
index f0a9fa7e..b32ebe28 100644
--- a/job_runner.py
+++ b/job_runner.py
@@ -12,6 +12,7 @@ from app.config import (
     JOB_ONBOARDING_4,
     JOB_BATCH_IMPORT,
     JOB_DELETE_ACCOUNT,
+    JOB_DELETE_MAILBOX,
 )
 from app.db import Session
 from app.email_utils import (
@@ -20,7 +21,7 @@ from app.email_utils import (
 )
 from app.import_utils import handle_batch_import
 from app.log import LOG
-from app.models import User, Job, BatchImport
+from app.models import User, Job, BatchImport, Mailbox
 from server import create_light_app
 
 
@@ -163,6 +164,29 @@ if __name__ == "__main__":
                     render("transactional/account-delete.txt"),
                     render("transactional/account-delete.html"),
                 )
+            elif job.name == JOB_DELETE_MAILBOX:
+                mailbox_id = job.payload.get("mailbox_id")
+                mailbox = Mailbox.get(mailbox_id)
+                if not mailbox:
+                    continue
+
+                mailbox_email = mailbox.email
+                user = mailbox.user
+
+                Mailbox.delete(mailbox_id)
+                Session.commit()
+                LOG.d("Mailbox %s %s deleted", mailbox_id, mailbox_email)
+
+                send_email(
+                    user.email,
+                    f"Your mailbox {mailbox_email} has been deleted",
+                    f"""Mailbox {mailbox_email} along with its aliases are deleted successfully.
+                
+                Regards,
+                SimpleLogin team.
+                        """,
+                )
+
             else:
                 LOG.e("Unknown job name %s", job.name)
 

From cfe88b5df2c477f9d19db1349a2c0a528bf9b6b2 Mon Sep 17 00:00:00 2001
From: Son <son@son>
Date: Wed, 13 Oct 2021 11:43:44 +0200
Subject: [PATCH 08/20] use job system to delete domain

---
 app/config.py                        |  1 +
 app/dashboard/views/domain_detail.py | 45 ++++++++--------------------
 job_runner.py                        | 27 ++++++++++++++++-
 3 files changed, 40 insertions(+), 33 deletions(-)

diff --git a/app/config.py b/app/config.py
index 69e8ec1c..7ad3a8ac 100644
--- a/app/config.py
+++ b/app/config.py
@@ -256,6 +256,7 @@ JOB_ONBOARDING_4 = "onboarding-4"
 JOB_BATCH_IMPORT = "batch-import"
 JOB_DELETE_ACCOUNT = "delete-account"
 JOB_DELETE_MAILBOX = "delete-mailbox"
+JOB_DELETE_DOMAIN = "delete-domain"
 
 # for pagination
 PAGE_LIMIT = 20
diff --git a/app/dashboard/views/domain_detail.py b/app/dashboard/views/domain_detail.py
index 0bdc45da..a14d5103 100644
--- a/app/dashboard/views/domain_detail.py
+++ b/app/dashboard/views/domain_detail.py
@@ -1,12 +1,11 @@
-from threading import Thread
-
 import re2 as re
+from arrow import arrow
 from flask import render_template, request, redirect, url_for, flash
 from flask_login import login_required, current_user
 from flask_wtf import FlaskForm
 from wtforms import StringField, validators, IntegerField
 
-from app.config import EMAIL_SERVERS_WITH_PRIORITY, EMAIL_DOMAIN
+from app.config import EMAIL_SERVERS_WITH_PRIORITY, EMAIL_DOMAIN, JOB_DELETE_DOMAIN
 from app.dashboard.base import dashboard_bp
 from app.db import Session
 from app.dns_utils import (
@@ -15,7 +14,6 @@ from app.dns_utils import (
     get_txt_record,
     get_cname_record,
 )
-from app.email_utils import send_email
 from app.log import LOG
 from app.models import (
     CustomDomain,
@@ -25,6 +23,7 @@ from app.models import (
     DomainMailbox,
     AutoCreateRule,
     AutoCreateRuleMailbox,
+    Job,
 )
 from app.utils import random_string
 
@@ -276,7 +275,16 @@ def domain_detail(custom_domain_id):
         elif request.form.get("form-name") == "delete":
             name = custom_domain.domain
             LOG.d("Schedule deleting %s", custom_domain)
-            Thread(target=delete_domain, args=(custom_domain_id,)).start()
+
+            # Schedule delete domain job
+            LOG.w("schedule delete domain job for %s", custom_domain)
+            Job.create(
+                name=JOB_DELETE_DOMAIN,
+                payload={"custom_domain_id": custom_domain.id},
+                run_at=arrow.now(),
+                commit=True,
+            )
+
             flash(
                 f"{name} scheduled for deletion."
                 f"You will receive a confirmation email when the deletion is finished",
@@ -290,33 +298,6 @@ def domain_detail(custom_domain_id):
     return render_template("dashboard/domain_detail/info.html", **locals())
 
 
-def delete_domain(custom_domain_id: int):
-    from server import create_light_app
-
-    with create_light_app().app_context():
-        custom_domain = CustomDomain.get(custom_domain_id)
-        if not custom_domain:
-            return
-
-        domain_name = custom_domain.domain
-        user = custom_domain.user
-
-        CustomDomain.delete(custom_domain.id)
-        Session.commit()
-
-        LOG.d("Domain %s deleted", domain_name)
-
-        send_email(
-            user.email,
-            f"Your domain {domain_name} has been deleted",
-            f"""Domain {domain_name} along with its aliases are deleted successfully.
-
-Regards,
-SimpleLogin team.
-        """,
-        )
-
-
 @dashboard_bp.route("/domains/<int:custom_domain_id>/trash", methods=["GET", "POST"])
 @login_required
 def domain_detail_trash(custom_domain_id):
diff --git a/job_runner.py b/job_runner.py
index b32ebe28..3ec6f48d 100644
--- a/job_runner.py
+++ b/job_runner.py
@@ -13,6 +13,7 @@ from app.config import (
     JOB_BATCH_IMPORT,
     JOB_DELETE_ACCOUNT,
     JOB_DELETE_MAILBOX,
+    JOB_DELETE_DOMAIN,
 )
 from app.db import Session
 from app.email_utils import (
@@ -21,7 +22,7 @@ from app.email_utils import (
 )
 from app.import_utils import handle_batch_import
 from app.log import LOG
-from app.models import User, Job, BatchImport, Mailbox
+from app.models import User, Job, BatchImport, Mailbox, CustomDomain
 from server import create_light_app
 
 
@@ -187,6 +188,30 @@ if __name__ == "__main__":
                         """,
                 )
 
+            elif job.name == JOB_DELETE_DOMAIN:
+                custom_domain_id = job.payload.get("custom_domain_id")
+                custom_domain = CustomDomain.get(custom_domain_id)
+                if not custom_domain:
+                    continue
+
+                domain_name = custom_domain.domain
+                user = custom_domain.user
+
+                CustomDomain.delete(custom_domain.id)
+                Session.commit()
+
+                LOG.d("Domain %s deleted", domain_name)
+
+                send_email(
+                    user.email,
+                    f"Your domain {domain_name} has been deleted",
+                    f"""Domain {domain_name} along with its aliases are deleted successfully.
+                
+                Regards,
+                SimpleLogin team.
+                        """,
+                )
+
             else:
                 LOG.e("Unknown job name %s", job.name)
 

From ced31edda2bf2700f4f195b8456184a984ccac22 Mon Sep 17 00:00:00 2001
From: Son <son@son>
Date: Wed, 13 Oct 2021 11:52:41 +0200
Subject: [PATCH 09/20] flake8

---
 job_runner.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/job_runner.py b/job_runner.py
index 3ec6f48d..a5427d9b 100644
--- a/job_runner.py
+++ b/job_runner.py
@@ -182,7 +182,6 @@ if __name__ == "__main__":
                     user.email,
                     f"Your mailbox {mailbox_email} has been deleted",
                     f"""Mailbox {mailbox_email} along with its aliases are deleted successfully.
-                
                 Regards,
                 SimpleLogin team.
                         """,
@@ -206,7 +205,6 @@ if __name__ == "__main__":
                     user.email,
                     f"Your domain {domain_name} has been deleted",
                     f"""Domain {domain_name} along with its aliases are deleted successfully.
-                
                 Regards,
                 SimpleLogin team.
                         """,

From 05cf0855112c9537379e1651932afec01027c4a4 Mon Sep 17 00:00:00 2001
From: Son <son@son>
Date: Wed, 13 Oct 2021 13:32:27 +0200
Subject: [PATCH 10/20] fix

---
 app/dashboard/views/domain_detail.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/dashboard/views/domain_detail.py b/app/dashboard/views/domain_detail.py
index a14d5103..7ff4a3f5 100644
--- a/app/dashboard/views/domain_detail.py
+++ b/app/dashboard/views/domain_detail.py
@@ -1,5 +1,5 @@
+import arrow
 import re2 as re
-from arrow import arrow
 from flask import render_template, request, redirect, url_for, flash
 from flask_login import login_required, current_user
 from flask_wtf import FlaskForm

From e5a7aeb3fb0db89707212c75315dd7c8401f386b Mon Sep 17 00:00:00 2001
From: Son <son@son>
Date: Wed, 13 Oct 2021 19:16:20 +0200
Subject: [PATCH 11/20] add sl-job-runner to self host instruction

---
 README.md       | 15 +++++++++++++++
 docs/upgrade.md | 15 ++++++++++++++-
 2 files changed, 29 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 7d0c6d99..faa530ab 100644
--- a/README.md
+++ b/README.md
@@ -473,6 +473,21 @@ docker run -d \
     simplelogin/app:3.4.0 python email_handler.py
 ```
 
+And finally the `job runner`
+
+```bash
+docker run -d \
+    --name sl-job-runner \
+    -v $(pwd)/sl:/sl \
+    -v $(pwd)/sl/upload:/code/static/upload \
+    -v $(pwd)/simplelogin.env:/code/.env \
+    -v $(pwd)/dkim.key:/dkim.key \
+    -v $(pwd)/dkim.pub.key:/dkim.pub.key \
+    --restart always \
+    --network="sl-network" \
+    simplelogin/app:3.4.0 python job_runner.py
+```
+
 ### Nginx
 
 Install Nginx and make sure to replace `mydomain.com` by your domain
diff --git a/docs/upgrade.md b/docs/upgrade.md
index 904250d7..4e579417 100644
--- a/docs/upgrade.md
+++ b/docs/upgrade.md
@@ -126,7 +126,7 @@ for user in User.query.all():
 sudo docker pull simplelogin/app:3.4.0
 
 # Stop SimpleLogin containers
-sudo docker stop sl-email sl-migration sl-app sl-db
+sudo docker stop sl-email sl-migration sl-app sl-db sl-job-runner
 
 # Make sure to remove these containers to avoid conflict
 sudo docker rm -f sl-email sl-migration sl-app sl-db
@@ -193,5 +193,18 @@ sudo docker run -d \
     --restart always \
     --network="sl-network" \
     simplelogin/app:3.4.0 python email_handler.py
+    
+# Run the job runner
+docker run -d \
+    --name sl-job-runner \
+    -v $(pwd)/sl:/sl \
+    -v $(pwd)/sl/upload:/code/static/upload \
+    -v $(pwd)/simplelogin.env:/code/.env \
+    -v $(pwd)/dkim.key:/dkim.key \
+    -v $(pwd)/dkim.pub.key:/dkim.pub.key \
+    --restart always \
+    --network="sl-network" \
+    simplelogin/app:3.4.0 python job_runner.py
+    
 ```
 

From 3bdeda3e04635ee7bfef59e31f024972a15e0e58 Mon Sep 17 00:00:00 2001
From: Son <son@son>
Date: Thu, 14 Oct 2021 15:10:16 +0200
Subject: [PATCH 12/20] add get_mailbox_bounce_info()

---
 app/email_utils.py                |  45 +++++++++++--
 local_data/email_tests/bounce.eml | 102 ++++++++++++++++++++++++++++++
 tests/test_email_utils.py         |  23 ++++++-
 3 files changed, 162 insertions(+), 8 deletions(-)
 create mode 100644 local_data/email_tests/bounce.eml

diff --git a/app/email_utils.py b/app/email_utils.py
index c80aabc0..1904fd3b 100644
--- a/app/email_utils.py
+++ b/app/email_utils.py
@@ -598,22 +598,53 @@ def mailbox_already_used(email: str, user) -> bool:
     return False
 
 
-def get_orig_message_from_bounce(msg: Message) -> Message:
+def get_orig_message_from_bounce(bounce_report: Message) -> Optional[Message]:
     """parse the original email from Bounce"""
     i = 0
-    for part in msg.walk():
+    for part in bounce_report.walk():
         i += 1
 
-        # the original message is the 4th part
-        # 1st part is the root part,  multipart/report
-        # 2nd is text/plain, Postfix log
+        # 1st part is the container (bounce report)
+        # 2nd part is the report from our own Postfix
+        # 3rd is report from other mailbox
+        # 4th is the container of the original message
         # ...
         # 7th is original message
         if i == 7:
             return part
 
 
-def get_orig_message_from_hotmail_complaint(msg: Message) -> Message:
+def get_mailbox_bounce_info(bounce_report: Message) -> Optional[Message]:
+    """
+    Return the bounce info from the bounce report
+    An example of bounce info:
+
+    Final-Recipient: rfc822; not-existing@gmail.com
+    Original-Recipient: rfc822;not-existing@gmail.com
+    Action: failed
+    Status: 5.1.1
+    Remote-MTA: dns; gmail-smtp-in.l.google.com
+    Diagnostic-Code: smtp;
+     550-5.1.1 The email account that you tried to reach does
+        not exist. Please try 550-5.1.1 double-checking the recipient's email
+        address for typos or 550-5.1.1 unnecessary spaces. Learn more at 550 5.1.1
+        https://support.google.com/mail/?p=NoSuchUser z127si6173191wmc.132 - gsmtp
+
+    """
+    i = 0
+    for part in bounce_report.walk():
+        i += 1
+
+        # 1st part is the container (bounce report)
+        # 2nd part is the report from our own Postfix
+        # 3rd is report from other mailbox
+        # 4th is the container of the original message
+        # 5th is a child of 3rd that contains more info about the bounce
+        if i == 5:
+            return part
+
+
+def get_orig_message_from_hotmail_complaint(msg: Message) -> Optional[Message]:
     i = 0
     for part in msg.walk():
         i += 1
@@ -625,7 +656,7 @@ def get_orig_message_from_hotmail_complaint(msg: Message) -> Message:
             return part
 
 
-def get_orig_message_from_yahoo_complaint(msg: Message) -> Message:
+def get_orig_message_from_yahoo_complaint(msg: Message) -> Optional[Message]:
     i = 0
     for part in msg.walk():
         i += 1
diff --git a/local_data/email_tests/bounce.eml b/local_data/email_tests/bounce.eml
new file mode 100644
index 00000000..4498aa27
--- /dev/null
+++ b/local_data/email_tests/bounce.eml
@@ -0,0 +1,102 @@
+Received: by mx1.sl.co (Postfix)
+	id F09806333D; Thu, 14 Oct 2021 09:14:44 +0000 (UTC)
+Date: Thu, 14 Oct 2021 09:14:44 +0000 (UTC)
+From: mailer-daemon@bounce.sl.co (Mail Delivery System)
+Subject: Undelivered Mail Returned to Sender
+To: bounce+5352+@sl.co
+Auto-Submitted: auto-replied
+MIME-Version: 1.0
+Content-Type: multipart/report; report-type=delivery-status;
+	boundary="8A32A6333B.1634202884/mx1.sl.co"
+Content-Transfer-Encoding: 8bit
+Message-Id: <20211014091444.F09806333D@mx1.sl.co>
+
+This is a MIME-encapsulated message.
+
+--8A32A6333B.1634202884/mx1.sl.co
+Content-Description: Notification
+Content-Type: text/plain; charset=utf-8
+Content-Transfer-Encoding: 8bit
+
+This is the mail system at host mx1.sl.co.
+
+I'm sorry to have to inform you that your message could not
+be delivered to one or more recipients. It's attached below.
+
+For further assistance, please send mail to <postmaster@sl.co>
+
+If you do so, please include this problem report. You can
+delete your own text from the attached returned message.
+
+                   The mail system
+
+<not-existing@gmail.com>: host
+    gmail-smtp-in.l.google.com[142.251.5.27] said: 550-5.1.1 The email account
+    that you tried to reach does not exist. Please try 550-5.1.1
+    double-checking the recipient's email address for typos or 550-5.1.1
+    unnecessary spaces. Learn more at 550 5.1.1
+    https://support.google.com/mail/?p=NoSuchUser z127si6173191wmc.132 - gsmtp
+    (in reply to RCPT TO command)
+
+--8A32A6333B.1634202884/mx1.sl.co
+Content-Description: Delivery report
+Content-Type: message/delivery-status
+
+Reporting-MTA: dns; mx1.sl.co
+X-Postfix-Queue-ID: 8A32A6333B
+X-Postfix-Sender: rfc822; bounce+5352+@sl.co
+Arrival-Date: Thu, 14 Oct 2021 09:14:44 +0000 (UTC)
+
+Final-Recipient: rfc822; not-existing@gmail.com
+Original-Recipient: rfc822;not-existing@gmail.com
+Action: failed
+Status: 5.1.1
+Remote-MTA: dns; gmail-smtp-in.l.google.com
+Diagnostic-Code: smtp;
+ 550-5.1.1 The email account that you tried to reach does
+    not exist. Please try 550-5.1.1 double-checking the recipient's email
+    address for typos or 550-5.1.1 unnecessary spaces. Learn more at 550 5.1.1
+    https://support.google.com/mail/?p=NoSuchUser z127si6173191wmc.132 - gsmtp
+
+--8A32A6333B.1634202884/mx1.sl.co
+Content-Description: Undelivered Message
+Content-Type: message/rfc822
+Content-Transfer-Encoding: 8bit
+
+Return-Path: <bounce+5352+@sl.co>
+X-SimpleLogin-Client-IP: 90.127.20.84
+Received: from 2a01cb00008c9c001a3eeffffec79eea.ipv6.abo.wanadoo.fr
+ (lfbn-idf1-1-2034-84.w90-127.abo.wanadoo.fr [90.127.20.84])
+	(using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits))
+	(No client certificate requested)
+	by mx1.sl.co (Postfix) with ESMTPS id 8A32A6333B
+	for <not-existing@gmail.com>;
+ Thu, 14 Oct 2021 09:14:44 +0000 (UTC)
+Content-Type: text/plain;
+	charset=us-ascii
+Content-Transfer-Encoding: 7bit
+Mime-Version: 1.0 (Mac OS X Mail 14.0 \(3654.120.0.1.13\))
+Subject: bounce 5
+Message-Id: <F2EFE3CE-3967-49EC-8639-9A5900230F2E@gmail.com>
+X-SimpleLogin-Type: Forward
+X-SimpleLogin-EmailLog-ID: 5352
+X-SimpleLogin-Envelope-From: sender@gmail.com
+X-SimpleLogin-Envelope-To: heyheyalo@sl.co
+date: Thu, 14 Oct 2021 09:14:44 -0000
+From: "First Last - sender at gmail.com"
+ <ra+sender.at.gmail.com+bsppvaap@sl.co>
+To: heyheyalo@sl.co
+List-Unsubscribe: <mailto:unsubsribe@sl.co?subject=26561=>
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=sl.co;
+  i=@sl.co; q=dns/txt; s=dkim; t=1634202884;
+ h=message-id : date :  subject : from : to;
+ bh=ktjzaMYZHA8J5baAHC3QyOmFwAAv/MvNtIz1dvmI3V0=;
+  b=mzf2ZDIVshKSSjw4AQnrOttgRRjzYzZ+49PaPRobt0xFH0E02a2C9Rl/qLEshLHA7amba
+  8iNTzdTkp9UJquzjk3NwM9GCakmSzd9DmFsalkgeErDAKWNo2O2c7aYDHZlK/sp2vgsIcSO
+  1w6sp8sVIRr2JrnFPxFOfsOSkSabeOA=
+
+Alo quoi
+
+
+
+--8A32A6333B.1634202884/mx1.sl.co--
diff --git a/tests/test_email_utils.py b/tests/test_email_utils.py
index 66f3e9b0..bd4aac38 100644
--- a/tests/test_email_utils.py
+++ b/tests/test_email_utils.py
@@ -1,9 +1,10 @@
 import email
+import os
 from email.message import EmailMessage
 
 import arrow
 
-from app.config import MAX_ALERT_24H, EMAIL_DOMAIN, BOUNCE_EMAIL
+from app.config import MAX_ALERT_24H, EMAIL_DOMAIN, BOUNCE_EMAIL, ROOT_DIR
 from app.db import Session
 from app.email_utils import (
     get_email_domain_part,
@@ -31,6 +32,8 @@ from app.email_utils import (
     should_ignore_bounce,
     get_header_unicode,
     parse_full_address,
+    get_orig_message_from_bounce,
+    get_mailbox_bounce_info,
 )
 from app.models import User, CustomDomain, Alias, Contact, EmailLog, IgnoreBounceSender
 
@@ -748,3 +751,21 @@ def test_should_ignore_bounce(flask_client):
 def test_get_header_unicode():
     assert get_header_unicode("ab@cd.com") == "ab@cd.com"
     assert get_header_unicode("=?utf-8?B?w6nDqQ==?=@example.com") == "éé@example.com"
+
+
+def test_get_orig_message_from_bounce():
+    with open(os.path.join(ROOT_DIR, "local_data", "email_tests", "bounce.eml")) as f:
+        bounce_report = email.message_from_file(f)
+
+    orig_msg = get_orig_message_from_bounce(bounce_report)
+    assert orig_msg["X-SimpleLogin-Type"] == "Forward"
+    assert orig_msg["X-SimpleLogin-Envelope-From"] == "sender@gmail.com"
+
+
+def test_get_mailbox_bounce_info():
+    with open(os.path.join(ROOT_DIR, "local_data", "email_tests", "bounce.eml")) as f:
+        bounce_report = email.message_from_file(f)
+
+    orig_msg = get_mailbox_bounce_info(bounce_report)
+    assert orig_msg["Final-Recipient"] == "rfc822; not-existing@gmail.com"
+    assert orig_msg["Original-Recipient"] == "rfc822;not-existing@gmail.com"

From 7add04accc045e99177095e220a500faae682e46 Mon Sep 17 00:00:00 2001
From: Son <son@son>
Date: Thu, 14 Oct 2021 15:42:19 +0200
Subject: [PATCH 13/20] Use alembic instead of flask migrate which depends on
 flask-sqlalchemy

---
 CONTRIBUTING.md          |  2 +-
 alembic.ini              | 83 ++++++++++++++++++++++++++++++++++++++++
 migrations/README        |  1 +
 migrations/alembic.ini   | 45 ----------------------
 migrations/env.py        | 43 ++++++++-------------
 scripts/new-migration.sh | 11 ++++--
 6 files changed, 109 insertions(+), 76 deletions(-)
 create mode 100644 alembic.ini
 create mode 100644 migrations/README
 delete mode 100644 migrations/alembic.ini

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index c2f5b0cf..31c05847 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -76,7 +76,7 @@ docker run -e POSTGRES_PASSWORD=mypassword -e POSTGRES_USER=myuser -e POSTGRES_D
 To run the server:
 
 ```
-flask db upgrade && flask dummy-data && python3 server.py
+alembic upgrade head && flask dummy-data && python3 server.py
 ```
 
 then open http://localhost:7777, you should be able to login with `john@wick.com / password` account.
diff --git a/alembic.ini b/alembic.ini
new file mode 100644
index 00000000..a8502487
--- /dev/null
+++ b/alembic.ini
@@ -0,0 +1,83 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = migrations
+
+# template used to generate migration files
+file_template = %%(year)d_%%(month).2d%%(day).2d%%(hour).2d_%%(rev)s_%%(slug)s
+
+# timezone to use when rendering the date
+# within the migration file as well as the filename.
+# string value is passed to dateutil.tz.gettz()
+# leave blank for localtime
+# timezone =
+
+# max length of characters to apply to the
+# "slug" field
+# truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; this defaults
+# to alembic/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path
+# version_locations = %(here)s/bar %(here)s/bat alembic/versions
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+
+[post_write_hooks]
+# post_write_hooks defines scripts or Python functions that are run
+# on newly generated revision scripts.  See the documentation for further
+# detail and examples
+
+# format using "black" - use the console_scripts runner, against the "black" entrypoint
+# hooks=black
+# black.type=console_scripts
+# black.entrypoint=black
+# black.options=-l 79
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S
diff --git a/migrations/README b/migrations/README
new file mode 100644
index 00000000..98e4f9c4
--- /dev/null
+++ b/migrations/README
@@ -0,0 +1 @@
+Generic single-database configuration.
\ No newline at end of file
diff --git a/migrations/alembic.ini b/migrations/alembic.ini
deleted file mode 100644
index b1b2d575..00000000
--- a/migrations/alembic.ini
+++ /dev/null
@@ -1,45 +0,0 @@
-# A generic, single database configuration.
-
-[alembic]
-# template used to generate migration files
-file_template = %%(year)d_%%(month).2d%%(day).2d%%(hour).2d_%%(rev)s_%%(slug)s
-
-# set to 'true' to run the environment during
-# the 'revision' command, regardless of autogenerate
-# revision_environment = false
-
-
-# Logging configuration
-[loggers]
-keys = root,sqlalchemy,alembic
-
-[handlers]
-keys = console
-
-[formatters]
-keys = generic
-
-[logger_root]
-level = WARN
-handlers = console
-qualname =
-
-[logger_sqlalchemy]
-level = WARN
-handlers =
-qualname = sqlalchemy.engine
-
-[logger_alembic]
-level = INFO
-handlers =
-qualname = alembic
-
-[handler_console]
-class = StreamHandler
-args = (sys.stderr,)
-level = NOTSET
-formatter = generic
-
-[formatter_generic]
-format = %(levelname)-5.5s [%(name)s] %(message)s
-datefmt = %H:%M:%S
diff --git a/migrations/env.py b/migrations/env.py
index 79b8174b..8ba3fcff 100644
--- a/migrations/env.py
+++ b/migrations/env.py
@@ -1,6 +1,3 @@
-from __future__ import with_statement
-
-import logging
 from logging.config import fileConfig
 
 from sqlalchemy import engine_from_config
@@ -15,23 +12,28 @@ config = context.config
 # Interpret the config file for Python logging.
 # This line sets up loggers basically.
 fileConfig(config.config_file_name)
-logger = logging.getLogger('alembic.env')
 
 # add your model's MetaData object here
 # for 'autogenerate' support
 # from myapp import mymodel
 # target_metadata = mymodel.Base.metadata
-from flask import current_app
-config.set_main_option(
-    'sqlalchemy.url', current_app.config.get(
-        'SQLALCHEMY_DATABASE_URI').replace('%', '%%'))
-target_metadata = current_app.extensions['migrate'].db.metadata
+import sys
+
+# hack to be able to import Base
+# cf https://stackoverflow.com/a/58891735/1428034
+sys.path = ['', '..'] + sys.path[1:]
+
+from app.models import Base
+from app.config import DB_URI
+target_metadata = Base.metadata
 
 # other values from the config, defined by the needs of env.py,
 # can be acquired:
 # my_important_option = config.get_main_option("my_important_option")
 # ... etc.
 
+config.set_main_option('sqlalchemy.url', DB_URI)
+
 
 def run_migrations_offline():
     """Run migrations in 'offline' mode.
@@ -47,7 +49,10 @@ def run_migrations_offline():
     """
     url = config.get_main_option("sqlalchemy.url")
     context.configure(
-        url=url, target_metadata=target_metadata, literal_binds=True
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
     )
 
     with context.begin_transaction():
@@ -61,29 +66,15 @@ def run_migrations_online():
     and associate a connection with the context.
 
     """
-
-    # this callback is used to prevent an auto-migration from being generated
-    # when there are no changes to the schema
-    # reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html
-    def process_revision_directives(context, revision, directives):
-        if getattr(config.cmd_opts, 'autogenerate', False):
-            script = directives[0]
-            if script.upgrade_ops.is_empty():
-                directives[:] = []
-                logger.info('No changes in schema detected.')
-
     connectable = engine_from_config(
         config.get_section(config.config_ini_section),
-        prefix='sqlalchemy.',
+        prefix="sqlalchemy.",
         poolclass=pool.NullPool,
     )
 
     with connectable.connect() as connection:
         context.configure(
-            connection=connection,
-            target_metadata=target_metadata,
-            process_revision_directives=process_revision_directives,
-            **current_app.extensions['migrate'].configure_args
+            connection=connection, target_metadata=target_metadata
         )
 
         with context.begin_transaction():
diff --git a/scripts/new-migration.sh b/scripts/new-migration.sh
index d2458117..4d41a08d 100644
--- a/scripts/new-migration.sh
+++ b/scripts/new-migration.sh
@@ -6,11 +6,14 @@
 docker rm -f sl-db
 docker run -p 25432:5432 --name sl-db -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=sl -d postgres:13
 
-# run run `flask db upgrade` to upgrade the DB to the latest stage and
-env DB_URI=postgresql://postgres:postgres@127.0.0.1:25432/sl poetry run flask db upgrade
+# sleep a little bit for the db to be ready
+sleep 3
 
-# finally `flask db migrate` to generate the migration script.
-env DB_URI=postgresql://postgres:postgres@127.0.0.1:25432/sl poetry run flask db migrate
+# upgrade the DB to the latest stage and
+env DB_URI=postgresql://postgres:postgres@127.0.0.1:25432/sl poetry run alembic upgrade head
+
+# generate the migration script.
+env DB_URI=postgresql://postgres:postgres@127.0.0.1:25432/sl poetry run alembic revision --autogenerate
 
 # remove the db
 docker rm -f sl-db
\ No newline at end of file

From fcb94f03318e04573091a94112233a9449857821 Mon Sep 17 00:00:00 2001
From: Son <son@son>
Date: Thu, 14 Oct 2021 15:45:29 +0200
Subject: [PATCH 14/20] add Bounce.info column

---
 app/models.py                                 |  1 +
 .../versions/2021_101415_d750d578b068_.py     | 29 +++++++++++++++++++
 2 files changed, 30 insertions(+)
 create mode 100644 migrations/versions/2021_101415_d750d578b068_.py

diff --git a/app/models.py b/app/models.py
index 5e6bc5df..d7a17ff7 100644
--- a/app/models.py
+++ b/app/models.py
@@ -2514,6 +2514,7 @@ class Bounce(Base, ModelMixin):
 
     __tablename__ = "bounce"
     email = sa.Column(sa.String(256), nullable=False, index=True)
+    info = sa.Column(sa.Text, nullable=True)
 
 
 class TransactionalEmail(Base, ModelMixin):
diff --git a/migrations/versions/2021_101415_d750d578b068_.py b/migrations/versions/2021_101415_d750d578b068_.py
new file mode 100644
index 00000000..49607e79
--- /dev/null
+++ b/migrations/versions/2021_101415_d750d578b068_.py
@@ -0,0 +1,29 @@
+"""empty message
+
+Revision ID: d750d578b068
+Revises: 2fbcad5527d7
+Create Date: 2021-10-14 15:44:57.816738
+
+"""
+import sqlalchemy_utils
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'd750d578b068'
+down_revision = '2fbcad5527d7'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('bounce', sa.Column('info', sa.Text(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('bounce', 'info')
+    # ### end Alembic commands ###

From 72931aa9b73ec5c7724e43ab12ed307facdfb669 Mon Sep 17 00:00:00 2001
From: Son <son@son>
Date: Thu, 14 Oct 2021 15:46:52 +0200
Subject: [PATCH 15/20] fill up Bounce.info

---
 email_handler.py | 37 ++++++++++++++++++++++++++++---------
 1 file changed, 28 insertions(+), 9 deletions(-)

diff --git a/email_handler.py b/email_handler.py
index 39c7a17a..cef58976 100644
--- a/email_handler.py
+++ b/email_handler.py
@@ -123,6 +123,7 @@ from app.email_utils import (
     get_orig_message_from_hotmail_complaint,
     parse_full_address,
     get_orig_message_from_yahoo_complaint,
+    get_mailbox_bounce_info,
 )
 from app.log import LOG, set_message_id
 from app.models import (
@@ -1178,7 +1179,12 @@ def handle_bounce_forward_phase(msg: Message, email_log: EmailLog):
         LOG.e("Use %s default mailbox %s", alias, alias.mailbox)
         mailbox = alias.mailbox
 
-    Bounce.create(email=mailbox.email, commit=True)
+    bounce_info = get_mailbox_bounce_info(msg)
+    if bounce_info:
+        Bounce.create(email=mailbox.email, info=bounce_info.as_string(), commit=True)
+    else:
+        LOG.w("cannot get bounce info")
+        Bounce.create(email=mailbox.email, commit=True)
 
     LOG.d(
         "Handle forward bounce %s -> %s -> %s. %s", contact, alias, mailbox, email_log
@@ -1259,10 +1265,7 @@ def handle_bounce_forward_phase(msg: Message, email_log: EmailLog):
             ignore_smtp_error=True,
         )
     else:
-        LOG.w(
-            "Disable alias %s now",
-            alias,
-        )
+        LOG.w("Disable alias %s %s. Last contact %s", alias, user, contact)
         alias.enabled = False
         Session.commit()
 
@@ -1380,7 +1383,16 @@ def handle_bounce_reply_phase(envelope, msg: Message, email_log: EmailLog):
 
     LOG.d("Handle reply bounce %s -> %s -> %s.%s", mailbox, alias, contact, email_log)
 
-    Bounce.create(email=sanitize_email(contact.website_email), commit=True)
+    bounce_info = get_mailbox_bounce_info(msg)
+    if bounce_info:
+        Bounce.create(
+            email=sanitize_email(contact.website_email),
+            info=bounce_info.as_string(),
+            commit=True,
+        )
+    else:
+        LOG.w("cannot get bounce info")
+        Bounce.create(email=sanitize_email(contact.website_email), commit=True)
 
     # Store the bounced email
     # generate a name for the email
@@ -1625,7 +1637,7 @@ def handle_unsubscribe_user(user_id: int, mail_from: str) -> str:
     return status.E202
 
 
-def handle_transactional_bounce(envelope: Envelope, rcpt_to):
+def handle_transactional_bounce(envelope: Envelope, msg, rcpt_to):
     LOG.d("handle transactional bounce sent to %s", rcpt_to)
 
     # parse the TransactionalEmail
@@ -1635,7 +1647,14 @@ def handle_transactional_bounce(envelope: Envelope, rcpt_to):
     # a transaction might have been deleted in delete_logs()
     if transactional:
         LOG.i("Create bounce for %s", transactional.email)
-        Bounce.create(email=transactional.email, commit=True)
+        bounce_info = get_mailbox_bounce_info(msg)
+        if bounce_info:
+            Bounce.create(
+                email=transactional.email, info=bounce_info.as_string(), commit=True
+            )
+        else:
+            LOG.w("cannot get bounce info")
+            Bounce.create(email=transactional.email, commit=True)
 
 
 def handle_bounce(envelope, email_log: EmailLog, msg: Message) -> str:
@@ -1773,7 +1792,7 @@ def handle(envelope: Envelope) -> str:
         and rcpt_tos[0].endswith(TRANSACTIONAL_BOUNCE_SUFFIX)
     ):
         LOG.d("Handle email sent to sender from %s", mail_from)
-        handle_transactional_bounce(envelope, rcpt_tos[0])
+        handle_transactional_bounce(envelope, msg, rcpt_tos[0])
         return status.E205
 
     if (

From 57bfa7e9334df7d55bcc2be0ae24370d3fedf8b4 Mon Sep 17 00:00:00 2001
From: Son <son@son>
Date: Fri, 15 Oct 2021 10:32:20 +0200
Subject: [PATCH 16/20] make sure that a domain already used in a verified
 mailbox can't be added

---
 app/dashboard/views/custom_domain.py  |  8 +++++++-
 tests/dashboard/test_custom_domain.py | 21 +++++++++++++++++++++
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/app/dashboard/views/custom_domain.py b/app/dashboard/views/custom_domain.py
index cf8a8edb..48feb806 100644
--- a/app/dashboard/views/custom_domain.py
+++ b/app/dashboard/views/custom_domain.py
@@ -43,13 +43,19 @@ def custom_domain():
                 if SLDomain.get_by(domain=new_domain):
                     flash("A custom domain cannot be a built-in domain.", "error")
                 elif CustomDomain.get_by(domain=new_domain):
-                    flash(f"{new_domain} already used", "warning")
+                    flash(f"{new_domain} already used", "error")
                 elif get_email_domain_part(current_user.email) == new_domain:
                     flash(
                         "You cannot add a domain that you are currently using for your personal email. "
                         "Please change your personal email to your real email",
                         "error",
                     )
+                elif Mailbox.filter(
+                    Mailbox.verified == True, Mailbox.email.endswith(f"@{new_domain}")
+                ).first():
+                    flash(
+                        f"{new_domain} already used in a SimpleLogin mailbox", "error"
+                    )
                 else:
                     new_custom_domain = CustomDomain.create(
                         domain=new_domain, user_id=current_user.id
diff --git a/tests/dashboard/test_custom_domain.py b/tests/dashboard/test_custom_domain.py
index d024a8cb..a143bceb 100644
--- a/tests/dashboard/test_custom_domain.py
+++ b/tests/dashboard/test_custom_domain.py
@@ -1,6 +1,7 @@
 from flask import url_for
 
 from app.db import Session
+from app.models import Mailbox
 from tests.utils import login
 
 
@@ -36,3 +37,23 @@ def test_add_domain_same_as_user_email(flask_client):
         b"You cannot add a domain that you are currently using for your personal email"
         in r.data
     )
+
+
+def test_add_domain_used_in_mailbox(flask_client):
+    """cannot add domain if it has been used in a verified mailbox"""
+    user = login(flask_client)
+    user.lifetime = True
+    Session.commit()
+
+    Mailbox.create(
+        user_id=user.id, email="mailbox@new-domain.com", verified=True, commit=True
+    )
+
+    r = flask_client.post(
+        url_for("dashboard.custom_domain"),
+        data={"form-name": "create", "domain": "new-domain.com"},
+        follow_redirects=True,
+    )
+
+    assert r.status_code == 200
+    assert b"new-domain.com already used in a SimpleLogin mailbox" in r.data

From d18bb28ca9232dd013663f15975146889f830b5e Mon Sep 17 00:00:00 2001
From: Son <son@son>
Date: Fri, 15 Oct 2021 10:37:22 +0200
Subject: [PATCH 17/20] add more log to investigate "Cannot parse Postfix queue
 ID" error

---
 email_handler.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/email_handler.py b/email_handler.py
index cef58976..a84569c2 100644
--- a/email_handler.py
+++ b/email_handler.py
@@ -1744,7 +1744,11 @@ def handle(envelope: Envelope) -> str:
     if postfix_queue_id:
         set_message_id(postfix_queue_id)
     else:
-        LOG.d("Cannot parse Postfix queue ID from %s", msg[headers.RECEIVED])
+        LOG.d(
+            "Cannot parse Postfix queue ID from %s %s",
+            msg.get_all(headers.RECEIVED),
+            msg[headers.RECEIVED],
+        )
 
     if should_ignore(mail_from, rcpt_tos):
         LOG.w("Ignore email mail_from=%s rcpt_to=%s", mail_from, rcpt_tos)

From 4d740a4dc00ee32e177d4d3d64663f447de596f3 Mon Sep 17 00:00:00 2001
From: Son <son@son>
Date: Fri, 15 Oct 2021 10:39:29 +0200
Subject: [PATCH 18/20] flake8

---
 app/dashboard/views/custom_domain.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/dashboard/views/custom_domain.py b/app/dashboard/views/custom_domain.py
index 48feb806..0ef4540d 100644
--- a/app/dashboard/views/custom_domain.py
+++ b/app/dashboard/views/custom_domain.py
@@ -51,7 +51,7 @@ def custom_domain():
                         "error",
                     )
                 elif Mailbox.filter(
-                    Mailbox.verified == True, Mailbox.email.endswith(f"@{new_domain}")
+                    Mailbox.verified.is_(True), Mailbox.email.endswith(f"@{new_domain}")
                 ).first():
                     flash(
                         f"{new_domain} already used in a SimpleLogin mailbox", "error"

From 34564f6fa433d44e94475f6956fd97974afb3f50 Mon Sep 17 00:00:00 2001
From: Son <son@son>
Date: Fri, 15 Oct 2021 10:46:22 +0200
Subject: [PATCH 19/20] Add Metric2.nb_total_bounced_last_24h column

---
 app/models.py                                 |  2 ++
 .../versions/2021_101510_2f1b3c759773_.py     | 29 +++++++++++++++++++
 2 files changed, 31 insertions(+)
 create mode 100644 migrations/versions/2021_101510_2f1b3c759773_.py

diff --git a/app/models.py b/app/models.py
index d7a17ff7..50e4b029 100644
--- a/app/models.py
+++ b/app/models.py
@@ -2503,6 +2503,8 @@ class Metric2(Base, ModelMixin):
     nb_block_last_24h = sa.Column(sa.Float, nullable=True)
     nb_reply_last_24h = sa.Column(sa.Float, nullable=True)
     nb_bounced_last_24h = sa.Column(sa.Float, nullable=True)
+    # includes bounces for both forwarding and transactional email
+    nb_total_bounced_last_24h = sa.Column(sa.Float, nullable=True)
 
     nb_verified_custom_domain = sa.Column(sa.Float, nullable=True)
 
diff --git a/migrations/versions/2021_101510_2f1b3c759773_.py b/migrations/versions/2021_101510_2f1b3c759773_.py
new file mode 100644
index 00000000..90128ebc
--- /dev/null
+++ b/migrations/versions/2021_101510_2f1b3c759773_.py
@@ -0,0 +1,29 @@
+"""empty message
+
+Revision ID: 2f1b3c759773
+Revises: d750d578b068
+Create Date: 2021-10-15 10:46:00.389295
+
+"""
+import sqlalchemy_utils
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '2f1b3c759773'
+down_revision = 'd750d578b068'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('metric2', sa.Column('nb_total_bounced_last_24h', sa.Float(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('metric2', 'nb_total_bounced_last_24h')
+    # ### end Alembic commands ###

From 7bfdb821af89190a22232de26a1bcf45501caf18 Mon Sep 17 00:00:00 2001
From: Son <son@son>
Date: Fri, 15 Oct 2021 10:47:07 +0200
Subject: [PATCH 20/20] compute and include nb_total_bounced_last_24h in email
 report

---
 cron.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/cron.py b/cron.py
index 92f721d4..fc1b7d10 100644
--- a/cron.py
+++ b/cron.py
@@ -257,6 +257,7 @@ def compute_metric2() -> Metric2:
         nb_bounced_last_24h=EmailLog.filter(EmailLog.created_at > _24h_ago)
         .filter_by(bounced=True)
         .count(),
+        nb_total_bounced_last_24h=Bounce.filter(Bounce.created_at > _24h_ago).count(),
         nb_reply_last_24h=EmailLog.filter(EmailLog.created_at > _24h_ago)
         .filter_by(is_reply=True)
         .count(),
@@ -408,6 +409,7 @@ nb_forward_last_24h: {stats_today.nb_forward_last_24h} - {increase_percent(stats
 nb_reply_last_24h: {stats_today.nb_reply_last_24h} - {increase_percent(stats_yesterday.nb_reply_last_24h, stats_today.nb_reply_last_24h)}  <br>
 nb_block_last_24h: {stats_today.nb_block_last_24h} - {increase_percent(stats_yesterday.nb_block_last_24h, stats_today.nb_block_last_24h)}  <br>
 nb_bounced_last_24h: {stats_today.nb_bounced_last_24h} - {increase_percent(stats_yesterday.nb_bounced_last_24h, stats_today.nb_bounced_last_24h)}  <br>
+nb_total_bounced_last_24h: {stats_today.nb_total_bounced_last_24h} - {increase_percent(stats_yesterday.nb_total_bounced_last_24h, stats_today.nb_total_bounced_last_24h)}  <br>
 
 nb_custom_domain: {stats_today.nb_verified_custom_domain} - {increase_percent(stats_yesterday.nb_verified_custom_domain, stats_today.nb_verified_custom_domain)}  <br>
 nb_app: {stats_today.nb_app} - {increase_percent(stats_yesterday.nb_app, stats_today.nb_app)}  <br>