diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c2f5b0cf..31c05847 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -76,7 +76,7 @@ docker run -e POSTGRES_PASSWORD=mypassword -e POSTGRES_USER=myuser -e POSTGRES_D To run the server: ``` -flask db upgrade && flask dummy-data && python3 server.py +alembic upgrade head && flask dummy-data && python3 server.py ``` then open http://localhost:7777, you should be able to login with `john@wick.com / password` account. diff --git a/README.md b/README.md index 7d0c6d99..faa530ab 100644 --- a/README.md +++ b/README.md @@ -473,6 +473,21 @@ docker run -d \ simplelogin/app:3.4.0 python email_handler.py ``` +And finally the `job runner` + +```bash +docker run -d \ + --name sl-job-runner \ + -v $(pwd)/sl:/sl \ + -v $(pwd)/sl/upload:/code/static/upload \ + -v $(pwd)/simplelogin.env:/code/.env \ + -v $(pwd)/dkim.key:/dkim.key \ + -v $(pwd)/dkim.pub.key:/dkim.pub.key \ + --restart always \ + --network="sl-network" \ + simplelogin/app:3.4.0 python job_runner.py +``` + ### Nginx Install Nginx and make sure to replace `mydomain.com` by your domain diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 00000000..a8502487 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,83 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = migrations + +# template used to generate migration files +file_template = %%(year)d_%%(month).2d%%(day).2d%%(hour).2d_%%(rev)s_%%(slug)s + +# timezone to use when rendering the date +# within the migration file as well as the filename. +# string value is passed to dateutil.tz.gettz() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the +# "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; this defaults +# to alembic/versions. When using multiple version +# directories, initial revisions must be specified with --version-path +# version_locations = %(here)s/bar %(here)s/bat alembic/versions + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks=black +# black.type=console_scripts +# black.entrypoint=black +# black.options=-l 79 + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/app/api/views/mailbox.py b/app/api/views/mailbox.py index 87f9dc79..d235d390 100644 --- a/app/api/views/mailbox.py +++ b/app/api/views/mailbox.py @@ -1,10 +1,12 @@ from smtplib import SMTPRecipientsRefused +import arrow from flask import g from flask import jsonify from flask import request from app.api.base import api_bp, require_api_auth +from app.config import JOB_DELETE_MAILBOX from app.dashboard.views.mailbox import send_verification_email from app.dashboard.views.mailbox_detail import verify_mailbox_change from app.db import Session @@ -13,7 +15,8 @@ from app.email_utils import ( email_can_be_used_as_mailbox, is_valid_email, ) -from app.models import Mailbox +from app.log import LOG +from app.models import Mailbox, Job from app.utils import sanitize_email @@ -88,8 +91,14 @@ def delete_mailbox(mailbox_id): if mailbox.id == user.default_mailbox_id: return jsonify(error="You cannot delete the default mailbox"), 400 - Mailbox.delete(mailbox_id) - Session.commit() + # Schedule delete account job + LOG.w("schedule delete mailbox job for %s", mailbox) + Job.create( + name=JOB_DELETE_MAILBOX, + payload={"mailbox_id": mailbox.id}, + run_at=arrow.now(), + commit=True, + ) return jsonify(deleted=True), 200 diff --git a/app/config.py b/app/config.py index 2553e3f0..7ad3a8ac 100644 --- a/app/config.py +++ b/app/config.py @@ -255,6 +255,8 @@ JOB_ONBOARDING_3 = "onboarding-3" JOB_ONBOARDING_4 = "onboarding-4" JOB_BATCH_IMPORT = "batch-import" JOB_DELETE_ACCOUNT = "delete-account" +JOB_DELETE_MAILBOX = "delete-mailbox" +JOB_DELETE_DOMAIN = "delete-domain" # for pagination PAGE_LIMIT = 20 diff --git a/app/dashboard/views/custom_domain.py b/app/dashboard/views/custom_domain.py index cf8a8edb..0ef4540d 100644 --- a/app/dashboard/views/custom_domain.py +++ b/app/dashboard/views/custom_domain.py @@ -43,13 +43,19 @@ def custom_domain(): if SLDomain.get_by(domain=new_domain): flash("A custom domain cannot be a built-in domain.", "error") elif CustomDomain.get_by(domain=new_domain): - flash(f"{new_domain} already used", "warning") + flash(f"{new_domain} already used", "error") elif get_email_domain_part(current_user.email) == new_domain: flash( "You cannot add a domain that you are currently using for your personal email. " "Please change your personal email to your real email", "error", ) + elif Mailbox.filter( + Mailbox.verified.is_(True), Mailbox.email.endswith(f"@{new_domain}") + ).first(): + flash( + f"{new_domain} already used in a SimpleLogin mailbox", "error" + ) else: new_custom_domain = CustomDomain.create( domain=new_domain, user_id=current_user.id diff --git a/app/dashboard/views/domain_detail.py b/app/dashboard/views/domain_detail.py index 0bdc45da..7ff4a3f5 100644 --- a/app/dashboard/views/domain_detail.py +++ b/app/dashboard/views/domain_detail.py @@ -1,12 +1,11 @@ -from threading import Thread - +import arrow import re2 as re from flask import render_template, request, redirect, url_for, flash from flask_login import login_required, current_user from flask_wtf import FlaskForm from wtforms import StringField, validators, IntegerField -from app.config import EMAIL_SERVERS_WITH_PRIORITY, EMAIL_DOMAIN +from app.config import EMAIL_SERVERS_WITH_PRIORITY, EMAIL_DOMAIN, JOB_DELETE_DOMAIN from app.dashboard.base import dashboard_bp from app.db import Session from app.dns_utils import ( @@ -15,7 +14,6 @@ from app.dns_utils import ( get_txt_record, get_cname_record, ) -from app.email_utils import send_email from app.log import LOG from app.models import ( CustomDomain, @@ -25,6 +23,7 @@ from app.models import ( DomainMailbox, AutoCreateRule, AutoCreateRuleMailbox, + Job, ) from app.utils import random_string @@ -276,7 +275,16 @@ def domain_detail(custom_domain_id): elif request.form.get("form-name") == "delete": name = custom_domain.domain LOG.d("Schedule deleting %s", custom_domain) - Thread(target=delete_domain, args=(custom_domain_id,)).start() + + # Schedule delete domain job + LOG.w("schedule delete domain job for %s", custom_domain) + Job.create( + name=JOB_DELETE_DOMAIN, + payload={"custom_domain_id": custom_domain.id}, + run_at=arrow.now(), + commit=True, + ) + flash( f"{name} scheduled for deletion." f"You will receive a confirmation email when the deletion is finished", @@ -290,33 +298,6 @@ def domain_detail(custom_domain_id): return render_template("dashboard/domain_detail/info.html", **locals()) -def delete_domain(custom_domain_id: int): - from server import create_light_app - - with create_light_app().app_context(): - custom_domain = CustomDomain.get(custom_domain_id) - if not custom_domain: - return - - domain_name = custom_domain.domain - user = custom_domain.user - - CustomDomain.delete(custom_domain.id) - Session.commit() - - LOG.d("Domain %s deleted", domain_name) - - send_email( - user.email, - f"Your domain {domain_name} has been deleted", - f"""Domain {domain_name} along with its aliases are deleted successfully. - -Regards, -SimpleLogin team. - """, - ) - - @dashboard_bp.route("/domains//trash", methods=["GET", "POST"]) @login_required def domain_detail_trash(custom_domain_id): diff --git a/app/dashboard/views/mailbox.py b/app/dashboard/views/mailbox.py index c69d9a55..ef5bf42d 100644 --- a/app/dashboard/views/mailbox.py +++ b/app/dashboard/views/mailbox.py @@ -1,5 +1,4 @@ -from threading import Thread - +import arrow from flask import render_template, request, redirect, url_for, flash from flask_login import login_required, current_user from flask_wtf import FlaskForm @@ -7,7 +6,7 @@ from itsdangerous import Signer from wtforms import validators from wtforms.fields.html5 import EmailField -from app.config import MAILBOX_SECRET, URL +from app.config import MAILBOX_SECRET, URL, JOB_DELETE_MAILBOX from app.dashboard.base import dashboard_bp from app.db import Session from app.email_utils import ( @@ -18,7 +17,7 @@ from app.email_utils import ( is_valid_email, ) from app.log import LOG -from app.models import Mailbox +from app.models import Mailbox, Job class NewMailboxForm(FlaskForm): @@ -51,8 +50,15 @@ def mailbox_route(): flash("You cannot delete default mailbox", "error") return redirect(url_for("dashboard.mailbox_route")) - LOG.d("Schedule deleting %s", mailbox) - Thread(target=delete_mailbox, args=(mailbox.id,)).start() + # Schedule delete account job + LOG.w("schedule delete mailbox job for %s", mailbox) + Job.create( + name=JOB_DELETE_MAILBOX, + payload={"mailbox_id": mailbox.id}, + run_at=arrow.now(), + commit=True, + ) + flash( f"Mailbox {mailbox.email} scheduled for deletion." f"You will receive a confirmation email when the deletion is finished", diff --git a/app/email/rate_limit.py b/app/email/rate_limit.py index 7e78022e..918db145 100644 --- a/app/email/rate_limit.py +++ b/app/email/rate_limit.py @@ -93,6 +93,9 @@ def rate_limited_reply_phase(reply_email: str) -> bool: def rate_limited(mail_from: str, rcpt_tos: [str]) -> bool: + # todo: re-enable rate limiting + return False + for rcpt_to in rcpt_tos: if is_reply_email(rcpt_to): if rate_limited_reply_phase(rcpt_to): diff --git a/app/email_utils.py b/app/email_utils.py index c80aabc0..1904fd3b 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -598,22 +598,53 @@ def mailbox_already_used(email: str, user) -> bool: return False -def get_orig_message_from_bounce(msg: Message) -> Message: +def get_orig_message_from_bounce(bounce_report: Message) -> Optional[Message]: """parse the original email from Bounce""" i = 0 - for part in msg.walk(): + for part in bounce_report.walk(): i += 1 - # the original message is the 4th part - # 1st part is the root part, multipart/report - # 2nd is text/plain, Postfix log + # 1st part is the container (bounce report) + # 2nd part is the report from our own Postfix + # 3rd is report from other mailbox + # 4th is the container of the original message # ... # 7th is original message if i == 7: return part -def get_orig_message_from_hotmail_complaint(msg: Message) -> Message: +def get_mailbox_bounce_info(bounce_report: Message) -> Optional[Message]: + """ + Return the bounce info from the bounce report + An example of bounce info: + + Final-Recipient: rfc822; not-existing@gmail.com + Original-Recipient: rfc822;not-existing@gmail.com + Action: failed + Status: 5.1.1 + Remote-MTA: dns; gmail-smtp-in.l.google.com + Diagnostic-Code: smtp; + 550-5.1.1 The email account that you tried to reach does + not exist. Please try 550-5.1.1 double-checking the recipient's email + address for typos or 550-5.1.1 unnecessary spaces. Learn more at 550 5.1.1 + https://support.google.com/mail/?p=NoSuchUser z127si6173191wmc.132 - gsmtp + + """ + i = 0 + for part in bounce_report.walk(): + i += 1 + + # 1st part is the container (bounce report) + # 2nd part is the report from our own Postfix + # 3rd is report from other mailbox + # 4th is the container of the original message + # 5th is a child of 3rd that contains more info about the bounce + if i == 5: + return part + + +def get_orig_message_from_hotmail_complaint(msg: Message) -> Optional[Message]: i = 0 for part in msg.walk(): i += 1 @@ -625,7 +656,7 @@ def get_orig_message_from_hotmail_complaint(msg: Message) -> Message: return part -def get_orig_message_from_yahoo_complaint(msg: Message) -> Message: +def get_orig_message_from_yahoo_complaint(msg: Message) -> Optional[Message]: i = 0 for part in msg.walk(): i += 1 diff --git a/app/models.py b/app/models.py index f1e09f0b..50e4b029 100644 --- a/app/models.py +++ b/app/models.py @@ -2503,6 +2503,8 @@ class Metric2(Base, ModelMixin): nb_block_last_24h = sa.Column(sa.Float, nullable=True) nb_reply_last_24h = sa.Column(sa.Float, nullable=True) nb_bounced_last_24h = sa.Column(sa.Float, nullable=True) + # includes bounces for both forwarding and transactional email + nb_total_bounced_last_24h = sa.Column(sa.Float, nullable=True) nb_verified_custom_domain = sa.Column(sa.Float, nullable=True) @@ -2514,6 +2516,7 @@ class Bounce(Base, ModelMixin): __tablename__ = "bounce" email = sa.Column(sa.String(256), nullable=False, index=True) + info = sa.Column(sa.Text, nullable=True) class TransactionalEmail(Base, ModelMixin): @@ -2557,7 +2560,7 @@ class IgnoredEmail(Base, ModelMixin): class IgnoreBounceSender(Base, ModelMixin): """Ignore sender that doesn't correctly handle bounces, for example noreply@github.com""" - __tablename__ = "ignored_bounce_sender" + __tablename__ = "ignore_bounce_sender" mail_from = sa.Column(sa.String(512), nullable=False, unique=True) diff --git a/cron.py b/cron.py index 92f721d4..fc1b7d10 100644 --- a/cron.py +++ b/cron.py @@ -257,6 +257,7 @@ def compute_metric2() -> Metric2: nb_bounced_last_24h=EmailLog.filter(EmailLog.created_at > _24h_ago) .filter_by(bounced=True) .count(), + nb_total_bounced_last_24h=Bounce.filter(Bounce.created_at > _24h_ago).count(), nb_reply_last_24h=EmailLog.filter(EmailLog.created_at > _24h_ago) .filter_by(is_reply=True) .count(), @@ -408,6 +409,7 @@ nb_forward_last_24h: {stats_today.nb_forward_last_24h} - {increase_percent(stats nb_reply_last_24h: {stats_today.nb_reply_last_24h} - {increase_percent(stats_yesterday.nb_reply_last_24h, stats_today.nb_reply_last_24h)}
nb_block_last_24h: {stats_today.nb_block_last_24h} - {increase_percent(stats_yesterday.nb_block_last_24h, stats_today.nb_block_last_24h)}
nb_bounced_last_24h: {stats_today.nb_bounced_last_24h} - {increase_percent(stats_yesterday.nb_bounced_last_24h, stats_today.nb_bounced_last_24h)}
+nb_total_bounced_last_24h: {stats_today.nb_total_bounced_last_24h} - {increase_percent(stats_yesterday.nb_total_bounced_last_24h, stats_today.nb_total_bounced_last_24h)}
nb_custom_domain: {stats_today.nb_verified_custom_domain} - {increase_percent(stats_yesterday.nb_verified_custom_domain, stats_today.nb_verified_custom_domain)}
nb_app: {stats_today.nb_app} - {increase_percent(stats_yesterday.nb_app, stats_today.nb_app)}
diff --git a/docs/upgrade.md b/docs/upgrade.md index 904250d7..4e579417 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -126,7 +126,7 @@ for user in User.query.all(): sudo docker pull simplelogin/app:3.4.0 # Stop SimpleLogin containers -sudo docker stop sl-email sl-migration sl-app sl-db +sudo docker stop sl-email sl-migration sl-app sl-db sl-job-runner # Make sure to remove these containers to avoid conflict sudo docker rm -f sl-email sl-migration sl-app sl-db @@ -193,5 +193,18 @@ sudo docker run -d \ --restart always \ --network="sl-network" \ simplelogin/app:3.4.0 python email_handler.py + +# Run the job runner +docker run -d \ + --name sl-job-runner \ + -v $(pwd)/sl:/sl \ + -v $(pwd)/sl/upload:/code/static/upload \ + -v $(pwd)/simplelogin.env:/code/.env \ + -v $(pwd)/dkim.key:/dkim.key \ + -v $(pwd)/dkim.pub.key:/dkim.pub.key \ + --restart always \ + --network="sl-network" \ + simplelogin/app:3.4.0 python job_runner.py + ``` diff --git a/email_handler.py b/email_handler.py index 1f6dda19..a84569c2 100644 --- a/email_handler.py +++ b/email_handler.py @@ -31,7 +31,6 @@ It should contain the following info: """ import argparse -import asyncio import email import os import time @@ -124,6 +123,7 @@ from app.email_utils import ( get_orig_message_from_hotmail_complaint, parse_full_address, get_orig_message_from_yahoo_complaint, + get_mailbox_bounce_info, ) from app.log import LOG, set_message_id from app.models import ( @@ -1179,7 +1179,12 @@ def handle_bounce_forward_phase(msg: Message, email_log: EmailLog): LOG.e("Use %s default mailbox %s", alias, alias.mailbox) mailbox = alias.mailbox - Bounce.create(email=mailbox.email, commit=True) + bounce_info = get_mailbox_bounce_info(msg) + if bounce_info: + Bounce.create(email=mailbox.email, info=bounce_info.as_string(), commit=True) + else: + LOG.w("cannot get bounce info") + Bounce.create(email=mailbox.email, commit=True) LOG.d( "Handle forward bounce %s -> %s -> %s. %s", contact, alias, mailbox, email_log @@ -1260,10 +1265,7 @@ def handle_bounce_forward_phase(msg: Message, email_log: EmailLog): ignore_smtp_error=True, ) else: - LOG.w( - "Disable alias %s now", - alias, - ) + LOG.w("Disable alias %s %s. Last contact %s", alias, user, contact) alias.enabled = False Session.commit() @@ -1381,7 +1383,16 @@ def handle_bounce_reply_phase(envelope, msg: Message, email_log: EmailLog): LOG.d("Handle reply bounce %s -> %s -> %s.%s", mailbox, alias, contact, email_log) - Bounce.create(email=sanitize_email(contact.website_email), commit=True) + bounce_info = get_mailbox_bounce_info(msg) + if bounce_info: + Bounce.create( + email=sanitize_email(contact.website_email), + info=bounce_info.as_string(), + commit=True, + ) + else: + LOG.w("cannot get bounce info") + Bounce.create(email=sanitize_email(contact.website_email), commit=True) # Store the bounced email # generate a name for the email @@ -1626,7 +1637,7 @@ def handle_unsubscribe_user(user_id: int, mail_from: str) -> str: return status.E202 -def handle_transactional_bounce(envelope: Envelope, rcpt_to): +def handle_transactional_bounce(envelope: Envelope, msg, rcpt_to): LOG.d("handle transactional bounce sent to %s", rcpt_to) # parse the TransactionalEmail @@ -1636,7 +1647,14 @@ def handle_transactional_bounce(envelope: Envelope, rcpt_to): # a transaction might have been deleted in delete_logs() if transactional: LOG.i("Create bounce for %s", transactional.email) - Bounce.create(email=transactional.email, commit=True) + bounce_info = get_mailbox_bounce_info(msg) + if bounce_info: + Bounce.create( + email=transactional.email, info=bounce_info.as_string(), commit=True + ) + else: + LOG.w("cannot get bounce info") + Bounce.create(email=transactional.email, commit=True) def handle_bounce(envelope, email_log: EmailLog, msg: Message) -> str: @@ -1712,7 +1730,7 @@ def should_ignore(mail_from: str, rcpt_tos: List[str]) -> bool: return False -async def handle(envelope: Envelope) -> str: +def handle(envelope: Envelope) -> str: """Return SMTP status""" # sanitize mail_from, rcpt_tos @@ -1726,7 +1744,11 @@ async def handle(envelope: Envelope) -> str: if postfix_queue_id: set_message_id(postfix_queue_id) else: - LOG.d("Cannot parse Postfix queue ID from %s", msg[headers.RECEIVED]) + LOG.d( + "Cannot parse Postfix queue ID from %s %s", + msg.get_all(headers.RECEIVED), + msg[headers.RECEIVED], + ) if should_ignore(mail_from, rcpt_tos): LOG.w("Ignore email mail_from=%s rcpt_to=%s", mail_from, rcpt_tos) @@ -1774,7 +1796,7 @@ async def handle(envelope: Envelope) -> str: and rcpt_tos[0].endswith(TRANSACTIONAL_BOUNCE_SUFFIX) ): LOG.d("Handle email sent to sender from %s", mail_from) - handle_transactional_bounce(envelope, rcpt_tos[0]) + handle_transactional_bounce(envelope, msg, rcpt_tos[0]) return status.E205 if ( @@ -1858,37 +1880,24 @@ async def handle(envelope: Envelope) -> str: return status.E523 if rate_limited(mail_from, rcpt_tos): - LOG.w( - "Rate Limiting applied for mail_from:%s rcpt_tos:%s, retry in 60s", - mail_from, - rcpt_tos, - ) - # slow down the rate a bit - await asyncio.sleep(60) + LOG.w("Rate Limiting applied for mail_from:%s rcpt_tos:%s", mail_from, rcpt_tos) - # rate limit is still applied - if rate_limited(mail_from, rcpt_tos): - LOG.w( - "Rate Limiting (no retry) applied for mail_from:%s rcpt_tos:%s", - mail_from, - rcpt_tos, - ) - # add more logging info. TODO: remove - if len(rcpt_tos) == 1: - alias = Alias.get_by(email=rcpt_tos[0]) - if alias: - LOG.w( - "total number email log on %s, %s is %s, %s", - alias, - alias.user, - EmailLog.filter(EmailLog.alias_id == alias.id).count(), - EmailLog.filter(EmailLog.user_id == alias.user_id).count(), - ) + # add more logging info. TODO: remove + if len(rcpt_tos) == 1: + alias = Alias.get_by(email=rcpt_tos[0]) + if alias: + LOG.w( + "total number email log on %s, %s is %s, %s", + alias, + alias.user, + EmailLog.filter(EmailLog.alias_id == alias.id).count(), + EmailLog.filter(EmailLog.user_id == alias.user_id).count(), + ) - if should_ignore_bounce(envelope.mail_from): - return status.E207 - else: - return status.E522 + if should_ignore_bounce(envelope.mail_from): + return status.E207 + else: + return status.E522 # Handle "out of office" auto notice. An automatic response is sent for every forwarded email # todo: remove logging @@ -1948,7 +1957,7 @@ async def handle(envelope: Envelope) -> str: class MailHandler: async def handle_DATA(self, server, session, envelope: Envelope): try: - ret = await self._handle(envelope) + ret = self._handle(envelope) return ret except Exception: LOG.e( @@ -1959,7 +1968,7 @@ class MailHandler: return status.E404 @newrelic.agent.background_task(application=newrelic_app) - async def _handle(self, envelope: Envelope): + def _handle(self, envelope: Envelope): start = time.time() # generate a different message_id to keep track of an email lifecycle @@ -1972,21 +1981,23 @@ class MailHandler: envelope.rcpt_tos, ) - ret = await handle(envelope) - elapsed = time.time() - start - LOG.i( - "Finish mail from %s, rctp tos %s, takes %s seconds <<===", - envelope.mail_from, - envelope.rcpt_tos, - elapsed, - ) - newrelic.agent.record_custom_metric( - "Custom/email_handler_time", elapsed, newrelic_app - ) - newrelic.agent.record_custom_metric( - "Custom/number_incoming_email", 1, newrelic_app - ) - return ret + app = new_app() + with app.app_context(): + ret = handle(envelope) + elapsed = time.time() - start + LOG.i( + "Finish mail from %s, rctp tos %s, takes %s seconds <<===", + envelope.mail_from, + envelope.rcpt_tos, + elapsed, + ) + newrelic.agent.record_custom_metric( + "Custom/email_handler_time", elapsed, newrelic_app + ) + newrelic.agent.record_custom_metric( + "Custom/number_incoming_email", 1, newrelic_app + ) + return ret def main(port: int): diff --git a/job_runner.py b/job_runner.py index f0a9fa7e..a5427d9b 100644 --- a/job_runner.py +++ b/job_runner.py @@ -12,6 +12,8 @@ from app.config import ( JOB_ONBOARDING_4, JOB_BATCH_IMPORT, JOB_DELETE_ACCOUNT, + JOB_DELETE_MAILBOX, + JOB_DELETE_DOMAIN, ) from app.db import Session from app.email_utils import ( @@ -20,7 +22,7 @@ from app.email_utils import ( ) from app.import_utils import handle_batch_import from app.log import LOG -from app.models import User, Job, BatchImport +from app.models import User, Job, BatchImport, Mailbox, CustomDomain from server import create_light_app @@ -163,6 +165,51 @@ if __name__ == "__main__": render("transactional/account-delete.txt"), render("transactional/account-delete.html"), ) + elif job.name == JOB_DELETE_MAILBOX: + mailbox_id = job.payload.get("mailbox_id") + mailbox = Mailbox.get(mailbox_id) + if not mailbox: + continue + + mailbox_email = mailbox.email + user = mailbox.user + + Mailbox.delete(mailbox_id) + Session.commit() + LOG.d("Mailbox %s %s deleted", mailbox_id, mailbox_email) + + send_email( + user.email, + f"Your mailbox {mailbox_email} has been deleted", + f"""Mailbox {mailbox_email} along with its aliases are deleted successfully. + Regards, + SimpleLogin team. + """, + ) + + elif job.name == JOB_DELETE_DOMAIN: + custom_domain_id = job.payload.get("custom_domain_id") + custom_domain = CustomDomain.get(custom_domain_id) + if not custom_domain: + continue + + domain_name = custom_domain.domain + user = custom_domain.user + + CustomDomain.delete(custom_domain.id) + Session.commit() + + LOG.d("Domain %s deleted", domain_name) + + send_email( + user.email, + f"Your domain {domain_name} has been deleted", + f"""Domain {domain_name} along with its aliases are deleted successfully. + Regards, + SimpleLogin team. + """, + ) + else: LOG.e("Unknown job name %s", job.name) diff --git a/local_data/email_tests/bounce.eml b/local_data/email_tests/bounce.eml new file mode 100644 index 00000000..4498aa27 --- /dev/null +++ b/local_data/email_tests/bounce.eml @@ -0,0 +1,102 @@ +Received: by mx1.sl.co (Postfix) + id F09806333D; Thu, 14 Oct 2021 09:14:44 +0000 (UTC) +Date: Thu, 14 Oct 2021 09:14:44 +0000 (UTC) +From: mailer-daemon@bounce.sl.co (Mail Delivery System) +Subject: Undelivered Mail Returned to Sender +To: bounce+5352+@sl.co +Auto-Submitted: auto-replied +MIME-Version: 1.0 +Content-Type: multipart/report; report-type=delivery-status; + boundary="8A32A6333B.1634202884/mx1.sl.co" +Content-Transfer-Encoding: 8bit +Message-Id: <20211014091444.F09806333D@mx1.sl.co> + +This is a MIME-encapsulated message. + +--8A32A6333B.1634202884/mx1.sl.co +Content-Description: Notification +Content-Type: text/plain; charset=utf-8 +Content-Transfer-Encoding: 8bit + +This is the mail system at host mx1.sl.co. + +I'm sorry to have to inform you that your message could not +be delivered to one or more recipients. It's attached below. + +For further assistance, please send mail to + +If you do so, please include this problem report. You can +delete your own text from the attached returned message. + + The mail system + +: host + gmail-smtp-in.l.google.com[142.251.5.27] said: 550-5.1.1 The email account + that you tried to reach does not exist. Please try 550-5.1.1 + double-checking the recipient's email address for typos or 550-5.1.1 + unnecessary spaces. Learn more at 550 5.1.1 + https://support.google.com/mail/?p=NoSuchUser z127si6173191wmc.132 - gsmtp + (in reply to RCPT TO command) + +--8A32A6333B.1634202884/mx1.sl.co +Content-Description: Delivery report +Content-Type: message/delivery-status + +Reporting-MTA: dns; mx1.sl.co +X-Postfix-Queue-ID: 8A32A6333B +X-Postfix-Sender: rfc822; bounce+5352+@sl.co +Arrival-Date: Thu, 14 Oct 2021 09:14:44 +0000 (UTC) + +Final-Recipient: rfc822; not-existing@gmail.com +Original-Recipient: rfc822;not-existing@gmail.com +Action: failed +Status: 5.1.1 +Remote-MTA: dns; gmail-smtp-in.l.google.com +Diagnostic-Code: smtp; + 550-5.1.1 The email account that you tried to reach does + not exist. Please try 550-5.1.1 double-checking the recipient's email + address for typos or 550-5.1.1 unnecessary spaces. Learn more at 550 5.1.1 + https://support.google.com/mail/?p=NoSuchUser z127si6173191wmc.132 - gsmtp + +--8A32A6333B.1634202884/mx1.sl.co +Content-Description: Undelivered Message +Content-Type: message/rfc822 +Content-Transfer-Encoding: 8bit + +Return-Path: +X-SimpleLogin-Client-IP: 90.127.20.84 +Received: from 2a01cb00008c9c001a3eeffffec79eea.ipv6.abo.wanadoo.fr + (lfbn-idf1-1-2034-84.w90-127.abo.wanadoo.fr [90.127.20.84]) + (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)) + (No client certificate requested) + by mx1.sl.co (Postfix) with ESMTPS id 8A32A6333B + for ; + Thu, 14 Oct 2021 09:14:44 +0000 (UTC) +Content-Type: text/plain; + charset=us-ascii +Content-Transfer-Encoding: 7bit +Mime-Version: 1.0 (Mac OS X Mail 14.0 \(3654.120.0.1.13\)) +Subject: bounce 5 +Message-Id: +X-SimpleLogin-Type: Forward +X-SimpleLogin-EmailLog-ID: 5352 +X-SimpleLogin-Envelope-From: sender@gmail.com +X-SimpleLogin-Envelope-To: heyheyalo@sl.co +date: Thu, 14 Oct 2021 09:14:44 -0000 +From: "First Last - sender at gmail.com" + +To: heyheyalo@sl.co +List-Unsubscribe: +DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=sl.co; + i=@sl.co; q=dns/txt; s=dkim; t=1634202884; + h=message-id : date : subject : from : to; + bh=ktjzaMYZHA8J5baAHC3QyOmFwAAv/MvNtIz1dvmI3V0=; + b=mzf2ZDIVshKSSjw4AQnrOttgRRjzYzZ+49PaPRobt0xFH0E02a2C9Rl/qLEshLHA7amba + 8iNTzdTkp9UJquzjk3NwM9GCakmSzd9DmFsalkgeErDAKWNo2O2c7aYDHZlK/sp2vgsIcSO + 1w6sp8sVIRr2JrnFPxFOfsOSkSabeOA= + +Alo quoi + + + +--8A32A6333B.1634202884/mx1.sl.co-- diff --git a/migrations/README b/migrations/README new file mode 100644 index 00000000..98e4f9c4 --- /dev/null +++ b/migrations/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/migrations/alembic.ini b/migrations/alembic.ini deleted file mode 100644 index b1b2d575..00000000 --- a/migrations/alembic.ini +++ /dev/null @@ -1,45 +0,0 @@ -# A generic, single database configuration. - -[alembic] -# template used to generate migration files -file_template = %%(year)d_%%(month).2d%%(day).2d%%(hour).2d_%%(rev)s_%%(slug)s - -# set to 'true' to run the environment during -# the 'revision' command, regardless of autogenerate -# revision_environment = false - - -# Logging configuration -[loggers] -keys = root,sqlalchemy,alembic - -[handlers] -keys = console - -[formatters] -keys = generic - -[logger_root] -level = WARN -handlers = console -qualname = - -[logger_sqlalchemy] -level = WARN -handlers = -qualname = sqlalchemy.engine - -[logger_alembic] -level = INFO -handlers = -qualname = alembic - -[handler_console] -class = StreamHandler -args = (sys.stderr,) -level = NOTSET -formatter = generic - -[formatter_generic] -format = %(levelname)-5.5s [%(name)s] %(message)s -datefmt = %H:%M:%S diff --git a/migrations/env.py b/migrations/env.py index 79b8174b..8ba3fcff 100644 --- a/migrations/env.py +++ b/migrations/env.py @@ -1,6 +1,3 @@ -from __future__ import with_statement - -import logging from logging.config import fileConfig from sqlalchemy import engine_from_config @@ -15,23 +12,28 @@ config = context.config # Interpret the config file for Python logging. # This line sets up loggers basically. fileConfig(config.config_file_name) -logger = logging.getLogger('alembic.env') # add your model's MetaData object here # for 'autogenerate' support # from myapp import mymodel # target_metadata = mymodel.Base.metadata -from flask import current_app -config.set_main_option( - 'sqlalchemy.url', current_app.config.get( - 'SQLALCHEMY_DATABASE_URI').replace('%', '%%')) -target_metadata = current_app.extensions['migrate'].db.metadata +import sys + +# hack to be able to import Base +# cf https://stackoverflow.com/a/58891735/1428034 +sys.path = ['', '..'] + sys.path[1:] + +from app.models import Base +from app.config import DB_URI +target_metadata = Base.metadata # other values from the config, defined by the needs of env.py, # can be acquired: # my_important_option = config.get_main_option("my_important_option") # ... etc. +config.set_main_option('sqlalchemy.url', DB_URI) + def run_migrations_offline(): """Run migrations in 'offline' mode. @@ -47,7 +49,10 @@ def run_migrations_offline(): """ url = config.get_main_option("sqlalchemy.url") context.configure( - url=url, target_metadata=target_metadata, literal_binds=True + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, ) with context.begin_transaction(): @@ -61,29 +66,15 @@ def run_migrations_online(): and associate a connection with the context. """ - - # this callback is used to prevent an auto-migration from being generated - # when there are no changes to the schema - # reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html - def process_revision_directives(context, revision, directives): - if getattr(config.cmd_opts, 'autogenerate', False): - script = directives[0] - if script.upgrade_ops.is_empty(): - directives[:] = [] - logger.info('No changes in schema detected.') - connectable = engine_from_config( config.get_section(config.config_ini_section), - prefix='sqlalchemy.', + prefix="sqlalchemy.", poolclass=pool.NullPool, ) with connectable.connect() as connection: context.configure( - connection=connection, - target_metadata=target_metadata, - process_revision_directives=process_revision_directives, - **current_app.extensions['migrate'].configure_args + connection=connection, target_metadata=target_metadata ) with context.begin_transaction(): diff --git a/migrations/versions/2021_101415_d750d578b068_.py b/migrations/versions/2021_101415_d750d578b068_.py new file mode 100644 index 00000000..49607e79 --- /dev/null +++ b/migrations/versions/2021_101415_d750d578b068_.py @@ -0,0 +1,29 @@ +"""empty message + +Revision ID: d750d578b068 +Revises: 2fbcad5527d7 +Create Date: 2021-10-14 15:44:57.816738 + +""" +import sqlalchemy_utils +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'd750d578b068' +down_revision = '2fbcad5527d7' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('bounce', sa.Column('info', sa.Text(), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('bounce', 'info') + # ### end Alembic commands ### diff --git a/migrations/versions/2021_101510_2f1b3c759773_.py b/migrations/versions/2021_101510_2f1b3c759773_.py new file mode 100644 index 00000000..90128ebc --- /dev/null +++ b/migrations/versions/2021_101510_2f1b3c759773_.py @@ -0,0 +1,29 @@ +"""empty message + +Revision ID: 2f1b3c759773 +Revises: d750d578b068 +Create Date: 2021-10-15 10:46:00.389295 + +""" +import sqlalchemy_utils +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '2f1b3c759773' +down_revision = 'd750d578b068' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('metric2', sa.Column('nb_total_bounced_last_24h', sa.Float(), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('metric2', 'nb_total_bounced_last_24h') + # ### end Alembic commands ### diff --git a/scripts/new-migration.sh b/scripts/new-migration.sh index d2458117..4d41a08d 100644 --- a/scripts/new-migration.sh +++ b/scripts/new-migration.sh @@ -6,11 +6,14 @@ docker rm -f sl-db docker run -p 25432:5432 --name sl-db -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=sl -d postgres:13 -# run run `flask db upgrade` to upgrade the DB to the latest stage and -env DB_URI=postgresql://postgres:postgres@127.0.0.1:25432/sl poetry run flask db upgrade +# sleep a little bit for the db to be ready +sleep 3 -# finally `flask db migrate` to generate the migration script. -env DB_URI=postgresql://postgres:postgres@127.0.0.1:25432/sl poetry run flask db migrate +# upgrade the DB to the latest stage and +env DB_URI=postgresql://postgres:postgres@127.0.0.1:25432/sl poetry run alembic upgrade head + +# generate the migration script. +env DB_URI=postgresql://postgres:postgres@127.0.0.1:25432/sl poetry run alembic revision --autogenerate # remove the db docker rm -f sl-db \ No newline at end of file diff --git a/tests/dashboard/test_custom_domain.py b/tests/dashboard/test_custom_domain.py index d024a8cb..a143bceb 100644 --- a/tests/dashboard/test_custom_domain.py +++ b/tests/dashboard/test_custom_domain.py @@ -1,6 +1,7 @@ from flask import url_for from app.db import Session +from app.models import Mailbox from tests.utils import login @@ -36,3 +37,23 @@ def test_add_domain_same_as_user_email(flask_client): b"You cannot add a domain that you are currently using for your personal email" in r.data ) + + +def test_add_domain_used_in_mailbox(flask_client): + """cannot add domain if it has been used in a verified mailbox""" + user = login(flask_client) + user.lifetime = True + Session.commit() + + Mailbox.create( + user_id=user.id, email="mailbox@new-domain.com", verified=True, commit=True + ) + + r = flask_client.post( + url_for("dashboard.custom_domain"), + data={"form-name": "create", "domain": "new-domain.com"}, + follow_redirects=True, + ) + + assert r.status_code == 200 + assert b"new-domain.com already used in a SimpleLogin mailbox" in r.data diff --git a/tests/test_email_utils.py b/tests/test_email_utils.py index 66f3e9b0..bd4aac38 100644 --- a/tests/test_email_utils.py +++ b/tests/test_email_utils.py @@ -1,9 +1,10 @@ import email +import os from email.message import EmailMessage import arrow -from app.config import MAX_ALERT_24H, EMAIL_DOMAIN, BOUNCE_EMAIL +from app.config import MAX_ALERT_24H, EMAIL_DOMAIN, BOUNCE_EMAIL, ROOT_DIR from app.db import Session from app.email_utils import ( get_email_domain_part, @@ -31,6 +32,8 @@ from app.email_utils import ( should_ignore_bounce, get_header_unicode, parse_full_address, + get_orig_message_from_bounce, + get_mailbox_bounce_info, ) from app.models import User, CustomDomain, Alias, Contact, EmailLog, IgnoreBounceSender @@ -748,3 +751,21 @@ def test_should_ignore_bounce(flask_client): def test_get_header_unicode(): assert get_header_unicode("ab@cd.com") == "ab@cd.com" assert get_header_unicode("=?utf-8?B?w6nDqQ==?=@example.com") == "éé@example.com" + + +def test_get_orig_message_from_bounce(): + with open(os.path.join(ROOT_DIR, "local_data", "email_tests", "bounce.eml")) as f: + bounce_report = email.message_from_file(f) + + orig_msg = get_orig_message_from_bounce(bounce_report) + assert orig_msg["X-SimpleLogin-Type"] == "Forward" + assert orig_msg["X-SimpleLogin-Envelope-From"] == "sender@gmail.com" + + +def test_get_mailbox_bounce_info(): + with open(os.path.join(ROOT_DIR, "local_data", "email_tests", "bounce.eml")) as f: + bounce_report = email.message_from_file(f) + + orig_msg = get_mailbox_bounce_info(bounce_report) + assert orig_msg["Final-Recipient"] == "rfc822; not-existing@gmail.com" + assert orig_msg["Original-Recipient"] == "rfc822;not-existing@gmail.com"