Merge branch 'simple-login:master' into master

CONTRIBUTING.md

@@ -76,7 +76,7 @@ docker run -e POSTGRES_PASSWORD=mypassword -e POSTGRES_USER=myuser -e POSTGRES_D
 To run the server:
 
 ```
-flask db upgrade && flask dummy-data && python3 server.py
+alembic upgrade head && flask dummy-data && python3 server.py
 ```
 
 then open http://localhost:7777, you should be able to login with `john@wick.com / password` account.

README.md

@@ -473,6 +473,21 @@ docker run -d \
     simplelogin/app:3.4.0 python email_handler.py
 ```
 
+And finally the `job runner`
+
+```bash
+docker run -d \
+    --name sl-job-runner \
+    -v $(pwd)/sl:/sl \
+    -v $(pwd)/sl/upload:/code/static/upload \
+    -v $(pwd)/simplelogin.env:/code/.env \
+    -v $(pwd)/dkim.key:/dkim.key \
+    -v $(pwd)/dkim.pub.key:/dkim.pub.key \
+    --restart always \
+    --network="sl-network" \
+    simplelogin/app:3.4.0 python job_runner.py
+```
+
 ### Nginx
 
 Install Nginx and make sure to replace `mydomain.com` by your domain

alembic.ini

@@ -0,0 +1,83 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = migrations
+
+# template used to generate migration files
+file_template = %%(year)d_%%(month).2d%%(day).2d%%(hour).2d_%%(rev)s_%%(slug)s
+
+# timezone to use when rendering the date
+# within the migration file as well as the filename.
+# string value is passed to dateutil.tz.gettz()
+# leave blank for localtime
+# timezone =
+
+# max length of characters to apply to the
+# "slug" field
+# truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; this defaults
+# to alembic/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path
+# version_locations = %(here)s/bar %(here)s/bat alembic/versions
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+
+[post_write_hooks]
+# post_write_hooks defines scripts or Python functions that are run
+# on newly generated revision scripts.  See the documentation for further
+# detail and examples
+
+# format using "black" - use the console_scripts runner, against the "black" entrypoint
+# hooks=black
+# black.type=console_scripts
+# black.entrypoint=black
+# black.options=-l 79
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

app/api/views/mailbox.py

@@ -1,10 +1,12 @@
 from smtplib import SMTPRecipientsRefused
 
+import arrow
 from flask import g
 from flask import jsonify
 from flask import request
 
 from app.api.base import api_bp, require_api_auth
+from app.config import JOB_DELETE_MAILBOX
 from app.dashboard.views.mailbox import send_verification_email
 from app.dashboard.views.mailbox_detail import verify_mailbox_change
 from app.db import Session
@@ -13,7 +15,8 @@ from app.email_utils import (
     email_can_be_used_as_mailbox,
     is_valid_email,
 )
-from app.models import Mailbox
+from app.log import LOG
+from app.models import Mailbox, Job
 from app.utils import sanitize_email
 
 
@@ -88,8 +91,14 @@ def delete_mailbox(mailbox_id):
     if mailbox.id == user.default_mailbox_id:
         return jsonify(error="You cannot delete the default mailbox"), 400
 
-    Mailbox.delete(mailbox_id)
+    # Schedule delete account job
-    Session.commit()
+    LOG.w("schedule delete mailbox job for %s", mailbox)
+    Job.create(
+        name=JOB_DELETE_MAILBOX,
+        payload={"mailbox_id": mailbox.id},
+        run_at=arrow.now(),
+        commit=True,
+    )
 
     return jsonify(deleted=True), 200
 

app/config.py

@@ -255,6 +255,8 @@ JOB_ONBOARDING_3 = "onboarding-3"
 JOB_ONBOARDING_4 = "onboarding-4"
 JOB_BATCH_IMPORT = "batch-import"
 JOB_DELETE_ACCOUNT = "delete-account"
+JOB_DELETE_MAILBOX = "delete-mailbox"
+JOB_DELETE_DOMAIN = "delete-domain"
 
 # for pagination
 PAGE_LIMIT = 20

app/dashboard/views/custom_domain.py

@@ -43,13 +43,19 @@ def custom_domain():
                 if SLDomain.get_by(domain=new_domain):
                     flash("A custom domain cannot be a built-in domain.", "error")
                 elif CustomDomain.get_by(domain=new_domain):
-                    flash(f"{new_domain} already used", "warning")
+                    flash(f"{new_domain} already used", "error")
                 elif get_email_domain_part(current_user.email) == new_domain:
                     flash(
                         "You cannot add a domain that you are currently using for your personal email. "
                         "Please change your personal email to your real email",
                         "error",
                     )
+                elif Mailbox.filter(
+                    Mailbox.verified.is_(True), Mailbox.email.endswith(f"@{new_domain}")
+                ).first():
+                    flash(
+                        f"{new_domain} already used in a SimpleLogin mailbox", "error"
+                    )
                 else:
                     new_custom_domain = CustomDomain.create(
                         domain=new_domain, user_id=current_user.id

app/dashboard/views/domain_detail.py

@@ -1,12 +1,11 @@
-from threading import Thread
+import arrow
-
 import re2 as re
 from flask import render_template, request, redirect, url_for, flash
 from flask_login import login_required, current_user
 from flask_wtf import FlaskForm
 from wtforms import StringField, validators, IntegerField
 
-from app.config import EMAIL_SERVERS_WITH_PRIORITY, EMAIL_DOMAIN
+from app.config import EMAIL_SERVERS_WITH_PRIORITY, EMAIL_DOMAIN, JOB_DELETE_DOMAIN
 from app.dashboard.base import dashboard_bp
 from app.db import Session
 from app.dns_utils import (
@@ -15,7 +14,6 @@ from app.dns_utils import (
     get_txt_record,
     get_cname_record,
 )
-from app.email_utils import send_email
 from app.log import LOG
 from app.models import (
     CustomDomain,
@@ -25,6 +23,7 @@ from app.models import (
     DomainMailbox,
     AutoCreateRule,
     AutoCreateRuleMailbox,
+    Job,
 )
 from app.utils import random_string
 
@@ -276,7 +275,16 @@ def domain_detail(custom_domain_id):
         elif request.form.get("form-name") == "delete":
             name = custom_domain.domain
             LOG.d("Schedule deleting %s", custom_domain)
-            Thread(target=delete_domain, args=(custom_domain_id,)).start()
+
+            # Schedule delete domain job
+            LOG.w("schedule delete domain job for %s", custom_domain)
+            Job.create(
+                name=JOB_DELETE_DOMAIN,
+                payload={"custom_domain_id": custom_domain.id},
+                run_at=arrow.now(),
+                commit=True,
+            )
+
             flash(
                 f"{name} scheduled for deletion."
                 f"You will receive a confirmation email when the deletion is finished",
@@ -290,33 +298,6 @@ def domain_detail(custom_domain_id):
     return render_template("dashboard/domain_detail/info.html", **locals())
 
 
-def delete_domain(custom_domain_id: int):
-    from server import create_light_app
-
-    with create_light_app().app_context():
-        custom_domain = CustomDomain.get(custom_domain_id)
-        if not custom_domain:
-            return
-
-        domain_name = custom_domain.domain
-        user = custom_domain.user
-
-        CustomDomain.delete(custom_domain.id)
-        Session.commit()
-
-        LOG.d("Domain %s deleted", domain_name)
-
-        send_email(
-            user.email,
-            f"Your domain {domain_name} has been deleted",
-            f"""Domain {domain_name} along with its aliases are deleted successfully.
-
-Regards,
-SimpleLogin team.
-        """,
-        )
-
-
 @dashboard_bp.route("/domains/<int:custom_domain_id>/trash", methods=["GET", "POST"])
 @login_required
 def domain_detail_trash(custom_domain_id):

app/dashboard/views/mailbox.py

@@ -1,5 +1,4 @@
-from threading import Thread
+import arrow
-
 from flask import render_template, request, redirect, url_for, flash
 from flask_login import login_required, current_user
 from flask_wtf import FlaskForm
@@ -7,7 +6,7 @@ from itsdangerous import Signer
 from wtforms import validators
 from wtforms.fields.html5 import EmailField
 
-from app.config import MAILBOX_SECRET, URL
+from app.config import MAILBOX_SECRET, URL, JOB_DELETE_MAILBOX
 from app.dashboard.base import dashboard_bp
 from app.db import Session
 from app.email_utils import (
@@ -18,7 +17,7 @@ from app.email_utils import (
     is_valid_email,
 )
 from app.log import LOG
-from app.models import Mailbox
+from app.models import Mailbox, Job
 
 
 class NewMailboxForm(FlaskForm):
@@ -51,8 +50,15 @@ def mailbox_route():
                 flash("You cannot delete default mailbox", "error")
                 return redirect(url_for("dashboard.mailbox_route"))
 
-            LOG.d("Schedule deleting %s", mailbox)
+            # Schedule delete account job
-            Thread(target=delete_mailbox, args=(mailbox.id,)).start()
+            LOG.w("schedule delete mailbox job for %s", mailbox)
+            Job.create(
+                name=JOB_DELETE_MAILBOX,
+                payload={"mailbox_id": mailbox.id},
+                run_at=arrow.now(),
+                commit=True,
+            )
+
             flash(
                 f"Mailbox {mailbox.email} scheduled for deletion."
                 f"You will receive a confirmation email when the deletion is finished",

app/email/rate_limit.py

@@ -93,6 +93,9 @@ def rate_limited_reply_phase(reply_email: str) -> bool:
 
 
 def rate_limited(mail_from: str, rcpt_tos: [str]) -> bool:
+    # todo: re-enable rate limiting
+    return False
+
     for rcpt_to in rcpt_tos:
         if is_reply_email(rcpt_to):
             if rate_limited_reply_phase(rcpt_to):

app/email_utils.py

@@ -598,22 +598,53 @@ def mailbox_already_used(email: str, user) -> bool:
     return False
 
 
-def get_orig_message_from_bounce(msg: Message) -> Message:
+def get_orig_message_from_bounce(bounce_report: Message) -> Optional[Message]:
     """parse the original email from Bounce"""
     i = 0
-    for part in msg.walk():
+    for part in bounce_report.walk():
         i += 1
 
-        # the original message is the 4th part
+        # 1st part is the container (bounce report)
-        # 1st part is the root part,  multipart/report
+        # 2nd part is the report from our own Postfix
-        # 2nd is text/plain, Postfix log
+        # 3rd is report from other mailbox
+        # 4th is the container of the original message
         # ...
         # 7th is original message
         if i == 7:
             return part
 
 
-def get_orig_message_from_hotmail_complaint(msg: Message) -> Message:
+def get_mailbox_bounce_info(bounce_report: Message) -> Optional[Message]:
+    """
+    Return the bounce info from the bounce report
+    An example of bounce info:
+
+    Final-Recipient: rfc822; not-existing@gmail.com
+    Original-Recipient: rfc822;not-existing@gmail.com
+    Action: failed
+    Status: 5.1.1
+    Remote-MTA: dns; gmail-smtp-in.l.google.com
+    Diagnostic-Code: smtp;
+     550-5.1.1 The email account that you tried to reach does
+        not exist. Please try 550-5.1.1 double-checking the recipient's email
+        address for typos or 550-5.1.1 unnecessary spaces. Learn more at 550 5.1.1
+        https://support.google.com/mail/?p=NoSuchUser z127si6173191wmc.132 - gsmtp
+
+    """
+    i = 0
+    for part in bounce_report.walk():
+        i += 1
+
+        # 1st part is the container (bounce report)
+        # 2nd part is the report from our own Postfix
+        # 3rd is report from other mailbox
+        # 4th is the container of the original message
+        # 5th is a child of 3rd that contains more info about the bounce
+        if i == 5:
+            return part
+
+
+def get_orig_message_from_hotmail_complaint(msg: Message) -> Optional[Message]:
     i = 0
     for part in msg.walk():
         i += 1
@@ -625,7 +656,7 @@ def get_orig_message_from_hotmail_complaint(msg: Message) -> Message:
             return part
 
 
-def get_orig_message_from_yahoo_complaint(msg: Message) -> Message:
+def get_orig_message_from_yahoo_complaint(msg: Message) -> Optional[Message]:
     i = 0
     for part in msg.walk():
         i += 1

app/models.py

@@ -2503,6 +2503,8 @@ class Metric2(Base, ModelMixin):
     nb_block_last_24h = sa.Column(sa.Float, nullable=True)
     nb_reply_last_24h = sa.Column(sa.Float, nullable=True)
     nb_bounced_last_24h = sa.Column(sa.Float, nullable=True)
+    # includes bounces for both forwarding and transactional email
+    nb_total_bounced_last_24h = sa.Column(sa.Float, nullable=True)
 
     nb_verified_custom_domain = sa.Column(sa.Float, nullable=True)
 
@@ -2514,6 +2516,7 @@ class Bounce(Base, ModelMixin):
 
     __tablename__ = "bounce"
     email = sa.Column(sa.String(256), nullable=False, index=True)
+    info = sa.Column(sa.Text, nullable=True)
 
 
 class TransactionalEmail(Base, ModelMixin):
@@ -2557,7 +2560,7 @@ class IgnoredEmail(Base, ModelMixin):
 class IgnoreBounceSender(Base, ModelMixin):
     """Ignore sender that doesn't correctly handle bounces, for example noreply@github.com"""
 
-    __tablename__ = "ignored_bounce_sender"
+    __tablename__ = "ignore_bounce_sender"
 
     mail_from = sa.Column(sa.String(512), nullable=False, unique=True)
 

cron.py

@@ -257,6 +257,7 @@ def compute_metric2() -> Metric2:
         nb_bounced_last_24h=EmailLog.filter(EmailLog.created_at > _24h_ago)
         .filter_by(bounced=True)
         .count(),
+        nb_total_bounced_last_24h=Bounce.filter(Bounce.created_at > _24h_ago).count(),
         nb_reply_last_24h=EmailLog.filter(EmailLog.created_at > _24h_ago)
         .filter_by(is_reply=True)
         .count(),
@@ -408,6 +409,7 @@ nb_forward_last_24h: {stats_today.nb_forward_last_24h} - {increase_percent(stats
 nb_reply_last_24h: {stats_today.nb_reply_last_24h} - {increase_percent(stats_yesterday.nb_reply_last_24h, stats_today.nb_reply_last_24h)}  <br>
 nb_block_last_24h: {stats_today.nb_block_last_24h} - {increase_percent(stats_yesterday.nb_block_last_24h, stats_today.nb_block_last_24h)}  <br>
 nb_bounced_last_24h: {stats_today.nb_bounced_last_24h} - {increase_percent(stats_yesterday.nb_bounced_last_24h, stats_today.nb_bounced_last_24h)}  <br>
+nb_total_bounced_last_24h: {stats_today.nb_total_bounced_last_24h} - {increase_percent(stats_yesterday.nb_total_bounced_last_24h, stats_today.nb_total_bounced_last_24h)}  <br>
 
 nb_custom_domain: {stats_today.nb_verified_custom_domain} - {increase_percent(stats_yesterday.nb_verified_custom_domain, stats_today.nb_verified_custom_domain)}  <br>
 nb_app: {stats_today.nb_app} - {increase_percent(stats_yesterday.nb_app, stats_today.nb_app)}  <br>

docs/upgrade.md

@@ -126,7 +126,7 @@ for user in User.query.all():
 sudo docker pull simplelogin/app:3.4.0
 
 # Stop SimpleLogin containers
-sudo docker stop sl-email sl-migration sl-app sl-db
+sudo docker stop sl-email sl-migration sl-app sl-db sl-job-runner
 
 # Make sure to remove these containers to avoid conflict
 sudo docker rm -f sl-email sl-migration sl-app sl-db
@@ -193,5 +193,18 @@ sudo docker run -d \
     --restart always \
     --network="sl-network" \
     simplelogin/app:3.4.0 python email_handler.py
+    
+# Run the job runner
+docker run -d \
+    --name sl-job-runner \
+    -v $(pwd)/sl:/sl \
+    -v $(pwd)/sl/upload:/code/static/upload \
+    -v $(pwd)/simplelogin.env:/code/.env \
+    -v $(pwd)/dkim.key:/dkim.key \
+    -v $(pwd)/dkim.pub.key:/dkim.pub.key \
+    --restart always \
+    --network="sl-network" \
+    simplelogin/app:3.4.0 python job_runner.py
+    
 ```
 

email_handler.py

@@ -31,7 +31,6 @@ It should contain the following info:
 
 """
 import argparse
-import asyncio
 import email
 import os
 import time
@@ -124,6 +123,7 @@ from app.email_utils import (
     get_orig_message_from_hotmail_complaint,
     parse_full_address,
     get_orig_message_from_yahoo_complaint,
+    get_mailbox_bounce_info,
 )
 from app.log import LOG, set_message_id
 from app.models import (
@@ -1179,6 +1179,11 @@ def handle_bounce_forward_phase(msg: Message, email_log: EmailLog):
         LOG.e("Use %s default mailbox %s", alias, alias.mailbox)
         mailbox = alias.mailbox
 
+    bounce_info = get_mailbox_bounce_info(msg)
+    if bounce_info:
+        Bounce.create(email=mailbox.email, info=bounce_info.as_string(), commit=True)
+    else:
+        LOG.w("cannot get bounce info")
         Bounce.create(email=mailbox.email, commit=True)
 
     LOG.d(
@@ -1260,10 +1265,7 @@ def handle_bounce_forward_phase(msg: Message, email_log: EmailLog):
             ignore_smtp_error=True,
         )
     else:
-        LOG.w(
+        LOG.w("Disable alias %s %s. Last contact %s", alias, user, contact)
-            "Disable alias %s now",
-            alias,
-        )
         alias.enabled = False
         Session.commit()
 
@@ -1381,6 +1383,15 @@ def handle_bounce_reply_phase(envelope, msg: Message, email_log: EmailLog):
 
     LOG.d("Handle reply bounce %s -> %s -> %s.%s", mailbox, alias, contact, email_log)
 
+    bounce_info = get_mailbox_bounce_info(msg)
+    if bounce_info:
+        Bounce.create(
+            email=sanitize_email(contact.website_email),
+            info=bounce_info.as_string(),
+            commit=True,
+        )
+    else:
+        LOG.w("cannot get bounce info")
         Bounce.create(email=sanitize_email(contact.website_email), commit=True)
 
     # Store the bounced email
@@ -1626,7 +1637,7 @@ def handle_unsubscribe_user(user_id: int, mail_from: str) -> str:
     return status.E202
 
 
-def handle_transactional_bounce(envelope: Envelope, rcpt_to):
+def handle_transactional_bounce(envelope: Envelope, msg, rcpt_to):
     LOG.d("handle transactional bounce sent to %s", rcpt_to)
 
     # parse the TransactionalEmail
@@ -1636,6 +1647,13 @@ def handle_transactional_bounce(envelope: Envelope, rcpt_to):
     # a transaction might have been deleted in delete_logs()
     if transactional:
         LOG.i("Create bounce for %s", transactional.email)
+        bounce_info = get_mailbox_bounce_info(msg)
+        if bounce_info:
+            Bounce.create(
+                email=transactional.email, info=bounce_info.as_string(), commit=True
+            )
+        else:
+            LOG.w("cannot get bounce info")
             Bounce.create(email=transactional.email, commit=True)
 
 
@@ -1712,7 +1730,7 @@ def should_ignore(mail_from: str, rcpt_tos: List[str]) -> bool:
     return False
 
 
-async def handle(envelope: Envelope) -> str:
+def handle(envelope: Envelope) -> str:
     """Return SMTP status"""
 
     # sanitize mail_from, rcpt_tos
@@ -1726,7 +1744,11 @@ async def handle(envelope: Envelope) -> str:
     if postfix_queue_id:
         set_message_id(postfix_queue_id)
     else:
-        LOG.d("Cannot parse Postfix queue ID from %s", msg[headers.RECEIVED])
+        LOG.d(
+            "Cannot parse Postfix queue ID from %s %s",
+            msg.get_all(headers.RECEIVED),
+            msg[headers.RECEIVED],
+        )
 
     if should_ignore(mail_from, rcpt_tos):
         LOG.w("Ignore email mail_from=%s rcpt_to=%s", mail_from, rcpt_tos)
@@ -1774,7 +1796,7 @@ async def handle(envelope: Envelope) -> str:
         and rcpt_tos[0].endswith(TRANSACTIONAL_BOUNCE_SUFFIX)
     ):
         LOG.d("Handle email sent to sender from %s", mail_from)
-        handle_transactional_bounce(envelope, rcpt_tos[0])
+        handle_transactional_bounce(envelope, msg, rcpt_tos[0])
         return status.E205
 
     if (
@@ -1858,21 +1880,8 @@ async def handle(envelope: Envelope) -> str:
                 return status.E523
 
     if rate_limited(mail_from, rcpt_tos):
-        LOG.w(
+        LOG.w("Rate Limiting applied for mail_from:%s rcpt_tos:%s", mail_from, rcpt_tos)
-            "Rate Limiting applied for mail_from:%s rcpt_tos:%s, retry in 60s",
-            mail_from,
-            rcpt_tos,
-        )
-        # slow down the rate a bit
-        await asyncio.sleep(60)
 
-        # rate limit is still applied
-        if rate_limited(mail_from, rcpt_tos):
-            LOG.w(
-                "Rate Limiting (no retry) applied for mail_from:%s rcpt_tos:%s",
-                mail_from,
-                rcpt_tos,
-            )
         # add more logging info. TODO: remove
         if len(rcpt_tos) == 1:
             alias = Alias.get_by(email=rcpt_tos[0])
@@ -1948,7 +1957,7 @@ async def handle(envelope: Envelope) -> str:
 class MailHandler:
     async def handle_DATA(self, server, session, envelope: Envelope):
         try:
-            ret = await self._handle(envelope)
+            ret = self._handle(envelope)
             return ret
         except Exception:
             LOG.e(
@@ -1959,7 +1968,7 @@ class MailHandler:
             return status.E404
 
     @newrelic.agent.background_task(application=newrelic_app)
-    async def _handle(self, envelope: Envelope):
+    def _handle(self, envelope: Envelope):
         start = time.time()
 
         # generate a different message_id to keep track of an email lifecycle
@@ -1972,7 +1981,9 @@ class MailHandler:
             envelope.rcpt_tos,
         )
 
-        ret = await handle(envelope)
+        app = new_app()
+        with app.app_context():
+            ret = handle(envelope)
             elapsed = time.time() - start
             LOG.i(
                 "Finish mail from %s, rctp tos %s, takes %s seconds <<===",

job_runner.py

@@ -12,6 +12,8 @@ from app.config import (
     JOB_ONBOARDING_4,
     JOB_BATCH_IMPORT,
     JOB_DELETE_ACCOUNT,
+    JOB_DELETE_MAILBOX,
+    JOB_DELETE_DOMAIN,
 )
 from app.db import Session
 from app.email_utils import (
@@ -20,7 +22,7 @@ from app.email_utils import (
 )
 from app.import_utils import handle_batch_import
 from app.log import LOG
-from app.models import User, Job, BatchImport
+from app.models import User, Job, BatchImport, Mailbox, CustomDomain
 from server import create_light_app
 
 
@@ -163,6 +165,51 @@ if __name__ == "__main__":
                     render("transactional/account-delete.txt"),
                     render("transactional/account-delete.html"),
                 )
+            elif job.name == JOB_DELETE_MAILBOX:
+                mailbox_id = job.payload.get("mailbox_id")
+                mailbox = Mailbox.get(mailbox_id)
+                if not mailbox:
+                    continue
+
+                mailbox_email = mailbox.email
+                user = mailbox.user
+
+                Mailbox.delete(mailbox_id)
+                Session.commit()
+                LOG.d("Mailbox %s %s deleted", mailbox_id, mailbox_email)
+
+                send_email(
+                    user.email,
+                    f"Your mailbox {mailbox_email} has been deleted",
+                    f"""Mailbox {mailbox_email} along with its aliases are deleted successfully.
+                Regards,
+                SimpleLogin team.
+                        """,
+                )
+
+            elif job.name == JOB_DELETE_DOMAIN:
+                custom_domain_id = job.payload.get("custom_domain_id")
+                custom_domain = CustomDomain.get(custom_domain_id)
+                if not custom_domain:
+                    continue
+
+                domain_name = custom_domain.domain
+                user = custom_domain.user
+
+                CustomDomain.delete(custom_domain.id)
+                Session.commit()
+
+                LOG.d("Domain %s deleted", domain_name)
+
+                send_email(
+                    user.email,
+                    f"Your domain {domain_name} has been deleted",
+                    f"""Domain {domain_name} along with its aliases are deleted successfully.
+                Regards,
+                SimpleLogin team.
+                        """,
+                )
+
             else:
                 LOG.e("Unknown job name %s", job.name)
 

local_data/email_tests/bounce.eml

@@ -0,0 +1,102 @@
+Received: by mx1.sl.co (Postfix)
+	id F09806333D; Thu, 14 Oct 2021 09:14:44 +0000 (UTC)
+Date: Thu, 14 Oct 2021 09:14:44 +0000 (UTC)
+From: mailer-daemon@bounce.sl.co (Mail Delivery System)
+Subject: Undelivered Mail Returned to Sender
+To: bounce+5352+@sl.co
+Auto-Submitted: auto-replied
+MIME-Version: 1.0
+Content-Type: multipart/report; report-type=delivery-status;
+	boundary="8A32A6333B.1634202884/mx1.sl.co"
+Content-Transfer-Encoding: 8bit
+Message-Id: <20211014091444.F09806333D@mx1.sl.co>
+
+This is a MIME-encapsulated message.
+
+--8A32A6333B.1634202884/mx1.sl.co
+Content-Description: Notification
+Content-Type: text/plain; charset=utf-8
+Content-Transfer-Encoding: 8bit
+
+This is the mail system at host mx1.sl.co.
+
+I'm sorry to have to inform you that your message could not
+be delivered to one or more recipients. It's attached below.
+
+For further assistance, please send mail to <postmaster@sl.co>
+
+If you do so, please include this problem report. You can
+delete your own text from the attached returned message.
+
+                   The mail system
+
+<not-existing@gmail.com>: host
+    gmail-smtp-in.l.google.com[142.251.5.27] said: 550-5.1.1 The email account
+    that you tried to reach does not exist. Please try 550-5.1.1
+    double-checking the recipient's email address for typos or 550-5.1.1
+    unnecessary spaces. Learn more at 550 5.1.1
+    https://support.google.com/mail/?p=NoSuchUser z127si6173191wmc.132 - gsmtp
+    (in reply to RCPT TO command)
+
+--8A32A6333B.1634202884/mx1.sl.co
+Content-Description: Delivery report
+Content-Type: message/delivery-status
+
+Reporting-MTA: dns; mx1.sl.co
+X-Postfix-Queue-ID: 8A32A6333B
+X-Postfix-Sender: rfc822; bounce+5352+@sl.co
+Arrival-Date: Thu, 14 Oct 2021 09:14:44 +0000 (UTC)
+
+Final-Recipient: rfc822; not-existing@gmail.com
+Original-Recipient: rfc822;not-existing@gmail.com
+Action: failed
+Status: 5.1.1
+Remote-MTA: dns; gmail-smtp-in.l.google.com
+Diagnostic-Code: smtp;
+ 550-5.1.1 The email account that you tried to reach does
+    not exist. Please try 550-5.1.1 double-checking the recipient's email
+    address for typos or 550-5.1.1 unnecessary spaces. Learn more at 550 5.1.1
+    https://support.google.com/mail/?p=NoSuchUser z127si6173191wmc.132 - gsmtp
+
+--8A32A6333B.1634202884/mx1.sl.co
+Content-Description: Undelivered Message
+Content-Type: message/rfc822
+Content-Transfer-Encoding: 8bit
+
+Return-Path: <bounce+5352+@sl.co>
+X-SimpleLogin-Client-IP: 90.127.20.84
+Received: from 2a01cb00008c9c001a3eeffffec79eea.ipv6.abo.wanadoo.fr
+ (lfbn-idf1-1-2034-84.w90-127.abo.wanadoo.fr [90.127.20.84])
+	(using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits))
+	(No client certificate requested)
+	by mx1.sl.co (Postfix) with ESMTPS id 8A32A6333B
+	for <not-existing@gmail.com>;
+ Thu, 14 Oct 2021 09:14:44 +0000 (UTC)
+Content-Type: text/plain;
+	charset=us-ascii
+Content-Transfer-Encoding: 7bit
+Mime-Version: 1.0 (Mac OS X Mail 14.0 \(3654.120.0.1.13\))
+Subject: bounce 5
+Message-Id: <F2EFE3CE-3967-49EC-8639-9A5900230F2E@gmail.com>
+X-SimpleLogin-Type: Forward
+X-SimpleLogin-EmailLog-ID: 5352
+X-SimpleLogin-Envelope-From: sender@gmail.com
+X-SimpleLogin-Envelope-To: heyheyalo@sl.co
+date: Thu, 14 Oct 2021 09:14:44 -0000
+From: "First Last - sender at gmail.com"
+ <ra+sender.at.gmail.com+bsppvaap@sl.co>
+To: heyheyalo@sl.co
+List-Unsubscribe: <mailto:unsubsribe@sl.co?subject=26561=>
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=sl.co;
+  i=@sl.co; q=dns/txt; s=dkim; t=1634202884;
+ h=message-id : date :  subject : from : to;
+ bh=ktjzaMYZHA8J5baAHC3QyOmFwAAv/MvNtIz1dvmI3V0=;
+  b=mzf2ZDIVshKSSjw4AQnrOttgRRjzYzZ+49PaPRobt0xFH0E02a2C9Rl/qLEshLHA7amba
+  8iNTzdTkp9UJquzjk3NwM9GCakmSzd9DmFsalkgeErDAKWNo2O2c7aYDHZlK/sp2vgsIcSO
+  1w6sp8sVIRr2JrnFPxFOfsOSkSabeOA=
+
+Alo quoi
+
+
+
+--8A32A6333B.1634202884/mx1.sl.co--

migrations/README

@@ -0,0 +1 @@
+Generic single-database configuration.

migrations/alembic.ini

@@ -1,45 +0,0 @@
-# A generic, single database configuration.
-
-[alembic]
-# template used to generate migration files
-file_template = %%(year)d_%%(month).2d%%(day).2d%%(hour).2d_%%(rev)s_%%(slug)s
-
-# set to 'true' to run the environment during
-# the 'revision' command, regardless of autogenerate
-# revision_environment = false
-
-
-# Logging configuration
-[loggers]
-keys = root,sqlalchemy,alembic
-
-[handlers]
-keys = console
-
-[formatters]
-keys = generic
-
-[logger_root]
-level = WARN
-handlers = console
-qualname =
-
-[logger_sqlalchemy]
-level = WARN
-handlers =
-qualname = sqlalchemy.engine
-
-[logger_alembic]
-level = INFO
-handlers =
-qualname = alembic
-
-[handler_console]
-class = StreamHandler
-args = (sys.stderr,)
-level = NOTSET
-formatter = generic
-
-[formatter_generic]
-format = %(levelname)-5.5s [%(name)s] %(message)s
-datefmt = %H:%M:%S

migrations/env.py

@@ -1,6 +1,3 @@
-from __future__ import with_statement
-
-import logging
 from logging.config import fileConfig
 
 from sqlalchemy import engine_from_config
@@ -15,23 +12,28 @@ config = context.config
 # Interpret the config file for Python logging.
 # This line sets up loggers basically.
 fileConfig(config.config_file_name)
-logger = logging.getLogger('alembic.env')
 
 # add your model's MetaData object here
 # for 'autogenerate' support
 # from myapp import mymodel
 # target_metadata = mymodel.Base.metadata
-from flask import current_app
+import sys
-config.set_main_option(
+
-    'sqlalchemy.url', current_app.config.get(
+# hack to be able to import Base
-        'SQLALCHEMY_DATABASE_URI').replace('%', '%%'))
+# cf https://stackoverflow.com/a/58891735/1428034
-target_metadata = current_app.extensions['migrate'].db.metadata
+sys.path = ['', '..'] + sys.path[1:]
+
+from app.models import Base
+from app.config import DB_URI
+target_metadata = Base.metadata
 
 # other values from the config, defined by the needs of env.py,
 # can be acquired:
 # my_important_option = config.get_main_option("my_important_option")
 # ... etc.
 
+config.set_main_option('sqlalchemy.url', DB_URI)
+
 
 def run_migrations_offline():
     """Run migrations in 'offline' mode.
@@ -47,7 +49,10 @@ def run_migrations_offline():
     """
     url = config.get_main_option("sqlalchemy.url")
     context.configure(
-        url=url, target_metadata=target_metadata, literal_binds=True
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
     )
 
     with context.begin_transaction():
@@ -61,29 +66,15 @@ def run_migrations_online():
     and associate a connection with the context.
 
     """
-
-    # this callback is used to prevent an auto-migration from being generated
-    # when there are no changes to the schema
-    # reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html
-    def process_revision_directives(context, revision, directives):
-        if getattr(config.cmd_opts, 'autogenerate', False):
-            script = directives[0]
-            if script.upgrade_ops.is_empty():
-                directives[:] = []
-                logger.info('No changes in schema detected.')
-
     connectable = engine_from_config(
         config.get_section(config.config_ini_section),
-        prefix='sqlalchemy.',
+        prefix="sqlalchemy.",
         poolclass=pool.NullPool,
     )
 
     with connectable.connect() as connection:
         context.configure(
-            connection=connection,
+            connection=connection, target_metadata=target_metadata
-            target_metadata=target_metadata,
-            process_revision_directives=process_revision_directives,
-            **current_app.extensions['migrate'].configure_args
         )
 
         with context.begin_transaction():

migrations/versions/2021_101415_d750d578b068_.py

@@ -0,0 +1,29 @@
+"""empty message
+
+Revision ID: d750d578b068
+Revises: 2fbcad5527d7
+Create Date: 2021-10-14 15:44:57.816738
+
+"""
+import sqlalchemy_utils
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'd750d578b068'
+down_revision = '2fbcad5527d7'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('bounce', sa.Column('info', sa.Text(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('bounce', 'info')
+    # ### end Alembic commands ###

migrations/versions/2021_101510_2f1b3c759773_.py

@@ -0,0 +1,29 @@
+"""empty message
+
+Revision ID: 2f1b3c759773
+Revises: d750d578b068
+Create Date: 2021-10-15 10:46:00.389295
+
+"""
+import sqlalchemy_utils
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '2f1b3c759773'
+down_revision = 'd750d578b068'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('metric2', sa.Column('nb_total_bounced_last_24h', sa.Float(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('metric2', 'nb_total_bounced_last_24h')
+    # ### end Alembic commands ###

scripts/new-migration.sh

@@ -6,11 +6,14 @@
 docker rm -f sl-db
 docker run -p 25432:5432 --name sl-db -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=sl -d postgres:13
 
-# run run `flask db upgrade` to upgrade the DB to the latest stage and
+# sleep a little bit for the db to be ready
-env DB_URI=postgresql://postgres:postgres@127.0.0.1:25432/sl poetry run flask db upgrade
+sleep 3
 
-# finally `flask db migrate` to generate the migration script.
+# upgrade the DB to the latest stage and
-env DB_URI=postgresql://postgres:postgres@127.0.0.1:25432/sl poetry run flask db migrate
+env DB_URI=postgresql://postgres:postgres@127.0.0.1:25432/sl poetry run alembic upgrade head
+
+# generate the migration script.
+env DB_URI=postgresql://postgres:postgres@127.0.0.1:25432/sl poetry run alembic revision --autogenerate
 
 # remove the db
 docker rm -f sl-db

tests/dashboard/test_custom_domain.py

@@ -1,6 +1,7 @@
 from flask import url_for
 
 from app.db import Session
+from app.models import Mailbox
 from tests.utils import login
 
 
@@ -36,3 +37,23 @@ def test_add_domain_same_as_user_email(flask_client):
         b"You cannot add a domain that you are currently using for your personal email"
         in r.data
     )
+
+
+def test_add_domain_used_in_mailbox(flask_client):
+    """cannot add domain if it has been used in a verified mailbox"""
+    user = login(flask_client)
+    user.lifetime = True
+    Session.commit()
+
+    Mailbox.create(
+        user_id=user.id, email="mailbox@new-domain.com", verified=True, commit=True
+    )
+
+    r = flask_client.post(
+        url_for("dashboard.custom_domain"),
+        data={"form-name": "create", "domain": "new-domain.com"},
+        follow_redirects=True,
+    )
+
+    assert r.status_code == 200
+    assert b"new-domain.com already used in a SimpleLogin mailbox" in r.data

tests/test_email_utils.py

@@ -1,9 +1,10 @@
 import email
+import os
 from email.message import EmailMessage
 
 import arrow
 
-from app.config import MAX_ALERT_24H, EMAIL_DOMAIN, BOUNCE_EMAIL
+from app.config import MAX_ALERT_24H, EMAIL_DOMAIN, BOUNCE_EMAIL, ROOT_DIR
 from app.db import Session
 from app.email_utils import (
     get_email_domain_part,
@@ -31,6 +32,8 @@ from app.email_utils import (
     should_ignore_bounce,
     get_header_unicode,
     parse_full_address,
+    get_orig_message_from_bounce,
+    get_mailbox_bounce_info,
 )
 from app.models import User, CustomDomain, Alias, Contact, EmailLog, IgnoreBounceSender
 
@@ -748,3 +751,21 @@ def test_should_ignore_bounce(flask_client):
 def test_get_header_unicode():
     assert get_header_unicode("ab@cd.com") == "ab@cd.com"
     assert get_header_unicode("=?utf-8?B?w6nDqQ==?=@example.com") == "éé@example.com"
+
+
+def test_get_orig_message_from_bounce():
+    with open(os.path.join(ROOT_DIR, "local_data", "email_tests", "bounce.eml")) as f:
+        bounce_report = email.message_from_file(f)
+
+    orig_msg = get_orig_message_from_bounce(bounce_report)
+    assert orig_msg["X-SimpleLogin-Type"] == "Forward"
+    assert orig_msg["X-SimpleLogin-Envelope-From"] == "sender@gmail.com"
+
+
+def test_get_mailbox_bounce_info():
+    with open(os.path.join(ROOT_DIR, "local_data", "email_tests", "bounce.eml")) as f:
+        bounce_report = email.message_from_file(f)
+
+    orig_msg = get_mailbox_bounce_info(bounce_report)
+    assert orig_msg["Final-Recipient"] == "rfc822; not-existing@gmail.com"
+    assert orig_msg["Original-Recipient"] == "rfc822;not-existing@gmail.com"