From 0de13ca4d9adce27bb92a4a3cad1d1519e128766 Mon Sep 17 00:00:00 2001 From: Son NK Date: Sat, 14 Mar 2020 16:07:34 +0100 Subject: [PATCH 01/16] add RefusedEmail model --- app/models.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/app/models.py b/app/models.py index aa18c0a1..7e1e28cb 100644 --- a/app/models.py +++ b/app/models.py @@ -344,6 +344,10 @@ def _expiration_5m(): return arrow.now().shift(minutes=5) +def _expiration_7d(): + return arrow.now().shift(days=7) + + class ActivationCode(db.Model, ModelMixin): """For activate user account""" @@ -758,6 +762,14 @@ class ForwardEmailLog(db.Model, ModelMixin): # usually because the forwarded email is too spammy bounced = db.Column(db.Boolean, nullable=False, default=False, server_default="0") + # Point to the email that has been refused + refused_email_id = db.Column( + db.ForeignKey("refused_email.id", ondelete="SET NULL"), nullable=True + ) + + refused_email = db.relationship("RefusedEmail") + forward = db.relationship(ForwardEmail) + class Subscription(db.Model, ModelMixin): # Come from Paddle @@ -958,3 +970,24 @@ class AccountActivation(db.Model, ModelMixin): CheckConstraint(tries >= 0, name="account_activation_tries_positive"), {}, ) + + +class RefusedEmail(db.Model, ModelMixin): + """Store emails that have been refused, i.e. bounced or classified as spams""" + + # Store the full report, including logs from Sending & Receiving MTA + full_report_path = db.Column(db.String(128), unique=True, nullable=False) + + # The original email, to display to user + path = db.Column(db.String(128), unique=True, nullable=False) + + user_id = db.Column(db.ForeignKey(User.id, ondelete="cascade"), nullable=False) + + # the email content will be deleted at this date + delete_at = db.Column(ArrowType, nullable=False, default=_expiration_7d) + + def get_url(self, expires_in=3600): + return s3.get_url(self.path, expires_in) + + def __repr__(self): + return f"" From c3b85115ca8c6f29a7fff1de07e0223205f245d9 Mon Sep 17 00:00:00 2001 From: Son NK Date: Sat, 14 Mar 2020 16:10:09 +0100 Subject: [PATCH 02/16] Add refused-email view --- app/dashboard/__init__.py | 1 + .../templates/dashboard/refused_email.html | 39 +++++++++++++++++++ .../templates/dashboard/setting.html | 19 +++++++++ app/dashboard/views/refused_email.py | 30 ++++++++++++++ 4 files changed, 89 insertions(+) create mode 100644 app/dashboard/templates/dashboard/refused_email.html create mode 100644 app/dashboard/views/refused_email.py diff --git a/app/dashboard/__init__.py b/app/dashboard/__init__.py index 3d981918..aedd11a1 100644 --- a/app/dashboard/__init__.py +++ b/app/dashboard/__init__.py @@ -17,4 +17,5 @@ from .views import ( mailbox, deleted_alias, mailbox_detail, + refused_email, ) diff --git a/app/dashboard/templates/dashboard/refused_email.html b/app/dashboard/templates/dashboard/refused_email.html new file mode 100644 index 00000000..a42b7ec2 --- /dev/null +++ b/app/dashboard/templates/dashboard/refused_email.html @@ -0,0 +1,39 @@ +{% extends 'default.html' %} + +{% block title %} + Refused Emails +{% endblock %} + +{% set active_page = "setting" %} + +{% block default_content %} +
+

Refused Emails

+ + {% if fels|length == 0 %} +
+ You don't have any refused email. +
+ {% endif %} + + {% for fel in fels %} + {% set refused_email = fel.refused_email %} + {% set forward = fel.forward %} + +
+ From: {{ forward.website_from or forward.website_email }}
+ To: {{ forward.gen_email.email }}
+
+ Sent {{ refused_email.created_at | dt }} +
+ + + View → +
This will download a ".eml" file that you can open in your favorite email client
+ +
+ {% endfor %} + + +
+{% endblock %} \ No newline at end of file diff --git a/app/dashboard/templates/dashboard/setting.html b/app/dashboard/templates/dashboard/setting.html index 01df04ae..b8793376 100644 --- a/app/dashboard/templates/dashboard/setting.html +++ b/app/dashboard/templates/dashboard/setting.html @@ -183,6 +183,25 @@ +
+
+
Refused Emails +
+ When an email sent to your alias is classified as spam or refused by your email provider, + it usually means your alias has been leaked to a spammer.
+ In this case SimpleLogin will keep a copy of this email (so it isn't lost) + and notify you so you can take a look at its content and take appropriate actions.
+ + The emails are deleted in 7 days. + This is an exceptional case where SimpleLogin stores the email. +
+
+ + See refused emails + +
+
+
Export Data diff --git a/app/dashboard/views/refused_email.py b/app/dashboard/views/refused_email.py new file mode 100644 index 00000000..4b5e84db --- /dev/null +++ b/app/dashboard/views/refused_email.py @@ -0,0 +1,30 @@ +from flask import render_template, request +from flask_login import login_required + +from app.dashboard.base import dashboard_bp +from app.models import ForwardEmailLog + + +@dashboard_bp.route("/refused_email", methods=["GET", "POST"]) +@login_required +def refused_email_route(): + # Highlight a refused email + highlight_fel_id = request.args.get("highlight_fel_id") + if highlight_fel_id: + highlight_fel_id = int(highlight_fel_id) + + fels: [ForwardEmailLog] = ForwardEmailLog.query.filter( + ForwardEmailLog.refused_email_id != None + ).all() + + # make sure the highlighted fel is the first fel + highlight_index = None + for ix, fel in enumerate(fels): + if fel.id == highlight_fel_id: + highlight_index = ix + break + + if highlight_index: + fels.insert(0, fels.pop(highlight_index)) + + return render_template("dashboard/refused_email.html", **locals()) From 0bb983068092ecc1a72bfa74e51839d2a9c7d4cd Mon Sep 17 00:00:00 2001 From: Son NK Date: Sat, 14 Mar 2020 16:34:23 +0100 Subject: [PATCH 03/16] Store the bounced email in email handling. --- app/email_utils.py | 15 ++++++ email_handler.py | 53 +++++++++++++++---- .../emails/transactional/bounced-email.html | 4 +- .../emails/transactional/bounced-email.txt | 3 +- 4 files changed, 64 insertions(+), 11 deletions(-) diff --git a/app/email_utils.py b/app/email_utils.py index ca9a8e44..2db45f8a 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -363,3 +363,18 @@ def mailbox_already_used(email: str, user) -> bool: return True return False + + +def get_orig_message_from_bounce(msg: Message) -> Message: + """parse the original email from Bounce""" + i = 0 + for part in msg.walk(): + i += 1 + + # the original message is the 4th part + # 1st part is the root part, multipart/report + # 2nd is text/plain, Postfix log + # ... + # 7th is original message + if i == 7: + return part diff --git a/email_handler.py b/email_handler.py index e1f05306..238510bf 100644 --- a/email_handler.py +++ b/email_handler.py @@ -37,21 +37,19 @@ from email.mime.application import MIMEApplication from email.mime.multipart import MIMEMultipart from email.parser import Parser from email.policy import SMTPUTF8 +from io import BytesIO from smtplib import SMTP from typing import Optional from aiosmtpd.controller import Controller -import gnupg +from app import pgp_utils, s3 from app.config import ( EMAIL_DOMAIN, POSTFIX_SERVER, URL, ALIAS_DOMAINS, - ADMIN_EMAIL, - SUPPORT_EMAIL, POSTFIX_SUBMISSION_TLS, - GNUPGHOME, ) from app.email_utils import ( get_email_name, @@ -65,6 +63,7 @@ from app.email_utils import ( send_cannot_create_domain_alias, email_belongs_to_alias_domains, render, + get_orig_message_from_bounce, ) from app.extensions import db from app.log import LOG @@ -76,10 +75,10 @@ from app.models import ( Directory, User, DeletedAlias, + RefusedEmail, ) from app.utils import random_string from server import create_app -from app import pgp_utils # fix the database connection leak issue @@ -406,7 +405,12 @@ def handle_reply(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> str: # in this case Postfix will try to send a bounce report to original sender, which is # the "reply email" if envelope.mail_from == "<>": - LOG.error("Bounce when sending to alias %s, user %s", alias, gen_email.user) + LOG.error( + "Bounce when sending to alias %s from %s, user %s", + alias, + forward_email.website_from, + gen_email.user, + ) handle_bounce( alias, envelope, forward_email, gen_email, msg, smtp, user, mailbox_email @@ -513,7 +517,9 @@ def handle_reply(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> str: def handle_bounce( alias, envelope, forward_email, gen_email, msg, smtp, user, mailbox_email ): - ForwardEmailLog.create(forward_id=forward_email.id, bounced=True) + fel: ForwardEmailLog = ForwardEmailLog.create( + forward_id=forward_email.id, bounced=True + ) db.session.commit() nb_bounced = ForwardEmailLog.filter_by( @@ -521,6 +527,30 @@ def handle_bounce( ).count() disable_alias_link = f"{URL}/dashboard/unsubscribe/{gen_email.id}" + # Store the bounced email + random_name = random_string(50) + + full_report_path = f"refused-emails/full-{random_name}.eml" + s3.upload_from_bytesio(full_report_path, BytesIO(msg.as_bytes())) + + file_path = f"refused-emails/{random_name}.eml" + orig_msg = get_orig_message_from_bounce(msg) + s3.upload_from_bytesio(file_path, BytesIO(orig_msg.as_bytes())) + + refused_email = RefusedEmail.create( + path=file_path, full_report_path=full_report_path, user_id=user.id + ) + db.session.flush() + + fel.refused_email_id = refused_email.id + db.session.commit() + + LOG.d("Create refused email %s", refused_email) + + refused_email_url = ( + URL + f"/dashboard/refused_email?highlight_fel_id=" + str(fel.id) + ) + # inform user if this is the first bounced email if nb_bounced == 1: LOG.d( @@ -530,7 +560,9 @@ def handle_bounce( alias, ) send_email( - mailbox_email, + # TOOD: use mailbox_email instead + user.email, + # mailbox_email, f"Email from {forward_email.website_from} to {alias} cannot be delivered to your inbox", render( "transactional/bounced-email.txt", @@ -539,6 +571,7 @@ def handle_bounce( website_from=forward_email.website_from, website_email=forward_email.website_email, disable_alias_link=disable_alias_link, + refused_email_url=refused_email_url, ), render( "transactional/bounced-email.html", @@ -547,8 +580,10 @@ def handle_bounce( website_from=forward_email.website_from, website_email=forward_email.website_email, disable_alias_link=disable_alias_link, + refused_email_url=refused_email_url, ), - bounced_email=msg, + # cannot include bounce email as it can contain spammy text + # bounced_email=msg, ) # disable the alias the second time email is bounced elif nb_bounced >= 2: diff --git a/templates/emails/transactional/bounced-email.html b/templates/emails/transactional/bounced-email.html index e1e3361e..2358289d 100644 --- a/templates/emails/transactional/bounced-email.html +++ b/templates/emails/transactional/bounced-email.html @@ -4,7 +4,9 @@ {{ render_text("Hi " + name) }} {{ render_text("An email sent to your alias " + alias + " from " + website_email + " was refused (or bounced) by your email provider.") }} - {{ render_text("This is usually due to the email being considered as spam by your email provider. The email is included at the end of this message so you can take a look at its content.") }} + {{ render_text('This is usually due to the email being considered as spam by your email provider.') }} + + {{ render_button("View the refused email", refused_email_url) }} {{ render_text('To avoid spams forwarded by SimpleLogin server, please consider the following options:') }} diff --git a/templates/emails/transactional/bounced-email.txt b/templates/emails/transactional/bounced-email.txt index 0fd0c688..8b8bcb8f 100644 --- a/templates/emails/transactional/bounced-email.txt +++ b/templates/emails/transactional/bounced-email.txt @@ -3,7 +3,8 @@ Hi {{name}} An email sent to your alias {{alias}} from {{website_from}} was refused (or bounced) by your email provider. This is usually due to the email being considered as spam by your email provider. -The email is included at the end of this message so you can take a look at its content. +You can view this email here: +{{ refused_email_url }} To avoid spams forwarded by SimpleLogin server, please consider the following options: From 7bf1baaafa4e0e49ef362e207ad5e57d3f16d245 Mon Sep 17 00:00:00 2001 From: Son NK Date: Sat, 14 Mar 2020 16:36:18 +0100 Subject: [PATCH 04/16] Add sql migration --- .../versions/2020_031416_11a35b448f83_.py | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 migrations/versions/2020_031416_11a35b448f83_.py diff --git a/migrations/versions/2020_031416_11a35b448f83_.py b/migrations/versions/2020_031416_11a35b448f83_.py new file mode 100644 index 00000000..49b0d34a --- /dev/null +++ b/migrations/versions/2020_031416_11a35b448f83_.py @@ -0,0 +1,45 @@ +"""empty message + +Revision ID: 11a35b448f83 +Revises: 628a5438295c +Create Date: 2020-03-14 16:35:13.564982 + +""" +import sqlalchemy_utils +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '11a35b448f83' +down_revision = '628a5438295c' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('refused_email', + sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('created_at', sqlalchemy_utils.types.arrow.ArrowType(), nullable=False), + sa.Column('updated_at', sqlalchemy_utils.types.arrow.ArrowType(), nullable=True), + sa.Column('full_report_path', sa.String(length=128), nullable=False), + sa.Column('path', sa.String(length=128), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('delete_at', sqlalchemy_utils.types.arrow.ArrowType(), nullable=False), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='cascade'), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('full_report_path'), + sa.UniqueConstraint('path') + ) + op.add_column('forward_email_log', sa.Column('refused_email_id', sa.Integer(), nullable=True)) + op.create_foreign_key(None, 'forward_email_log', 'refused_email', ['refused_email_id'], ['id'], ondelete='SET NULL') + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint(None, 'forward_email_log', type_='foreignkey') + op.drop_column('forward_email_log', 'refused_email_id') + op.drop_table('refused_email') + # ### end Alembic commands ### From 69198ff08ad29e7b24e7d1e8dbcb181bc3ca13cc Mon Sep 17 00:00:00 2001 From: Son NK Date: Sat, 14 Mar 2020 22:24:02 +0100 Subject: [PATCH 05/16] delete all unnecessary headers in PGP --- app/email_utils.py | 9 +++++++++ email_handler.py | 12 ++++++++++++ init_app.py | 2 ++ server.py | 2 +- 4 files changed, 24 insertions(+), 1 deletion(-) diff --git a/app/email_utils.py b/app/email_utils.py index 2db45f8a..2099d169 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -307,6 +307,15 @@ def delete_header(msg: Message, header: str): del msg._headers[i] +def delete_all_headers_except(msg: Message, headers: [str]): + headers = [h.lower() for h in headers] + + for i in reversed(range(len(msg._headers))): + header_name = msg._headers[i][0].lower() + if header_name not in headers: + del msg._headers[i] + + def email_belongs_to_alias_domains(email: str) -> bool: """return True if an email ends with one of the alias domains provided by SimpleLogin""" for domain in ALIAS_DOMAINS: diff --git a/email_handler.py b/email_handler.py index 238510bf..c6110204 100644 --- a/email_handler.py +++ b/email_handler.py @@ -64,6 +64,7 @@ from app.email_utils import ( email_belongs_to_alias_domains, render, get_orig_message_from_bounce, + delete_all_headers_except, ) from app.extensions import db from app.log import LOG @@ -269,6 +270,17 @@ def prepare_pgp_message(orig_msg: Message, pgp_fingerprint: str): if header_name != "Content-Type".lower(): msg[header_name] = orig_msg._headers[i][1] + # Delete unnecessary headers in orig_msg except to save space + delete_all_headers_except( + orig_msg, + [ + "MIME-Version", + "Content-Type", + "Content-Disposition", + "Content-Transfer-Encoding", + ], + ) + first = MIMEApplication( _subtype="pgp-encrypted", _encoder=encoders.encode_7or8bit, _data="" ) diff --git a/init_app.py b/init_app.py index bf5fc728..54bb1239 100644 --- a/init_app.py +++ b/init_app.py @@ -22,6 +22,8 @@ def load_pgp_public_keys(app): db.session.commit() + LOG.d("Finish load_pgp_public_keys") + if __name__ == "__main__": app = create_app() diff --git a/server.py b/server.py index 46abbff9..24c27d0f 100644 --- a/server.py +++ b/server.py @@ -151,7 +151,7 @@ def fake_data(): db.session.commit() api_key = ApiKey.create(user_id=user.id, name="Chrome") - api_key.code = "codeCH" + api_key.code = "code" api_key = ApiKey.create(user_id=user.id, name="Firefox") api_key.code = "codeFF" From 5db92b049d41c85db100dac1e574911e9b4c0b74 Mon Sep 17 00:00:00 2001 From: Son NK Date: Sat, 14 Mar 2020 23:00:33 +0100 Subject: [PATCH 06/16] Inform refused email to mailbox --- email_handler.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/email_handler.py b/email_handler.py index c6110204..3f462e1c 100644 --- a/email_handler.py +++ b/email_handler.py @@ -572,9 +572,7 @@ def handle_bounce( alias, ) send_email( - # TOOD: use mailbox_email instead - user.email, - # mailbox_email, + mailbox_email, f"Email from {forward_email.website_from} to {alias} cannot be delivered to your inbox", render( "transactional/bounced-email.txt", From 0525e5822a39a967ded13042359609e87aee474f Mon Sep 17 00:00:00 2001 From: Son NK Date: Sun, 15 Mar 2020 10:50:46 +0100 Subject: [PATCH 07/16] Not include original email in automatic disable alias email --- email_handler.py | 5 ++++- templates/emails/transactional/automatic-disable-alias.html | 5 +++-- templates/emails/transactional/automatic-disable-alias.txt | 4 +++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/email_handler.py b/email_handler.py index 3f462e1c..6f5c88fd 100644 --- a/email_handler.py +++ b/email_handler.py @@ -614,6 +614,7 @@ def handle_bounce( alias=alias, website_from=forward_email.website_from, website_email=forward_email.website_email, + refused_email_url=refused_email_url, ), render( "transactional/automatic-disable-alias.html", @@ -621,8 +622,10 @@ def handle_bounce( alias=alias, website_from=forward_email.website_from, website_email=forward_email.website_email, + refused_email_url=refused_email_url, ), - bounced_email=msg, + # cannot include bounce email as it can contain spammy text + # bounced_email=msg, ) diff --git a/templates/emails/transactional/automatic-disable-alias.html b/templates/emails/transactional/automatic-disable-alias.html index d3ff7a50..87140c40 100644 --- a/templates/emails/transactional/automatic-disable-alias.html +++ b/templates/emails/transactional/automatic-disable-alias.html @@ -5,8 +5,9 @@ {{ render_text("There are at least 2 emails sent to your alias " + alias + " from " + website_email + " that have been refused (or bounced) by your email provider.") }} - {{ render_text("This is usually due to the email being considered as spam by your email provider. - The email is included at the end of this message so you can take a look at its content.") }} + {{ render_text("This is usually due to the email being considered as spam by your email provider.") }} + + {{ render_button("View the refused email", refused_email_url) }} {{ render_text('As security measure, we have disabled the alias ' + alias) }} diff --git a/templates/emails/transactional/automatic-disable-alias.txt b/templates/emails/transactional/automatic-disable-alias.txt index 5de9afc7..639da5c1 100644 --- a/templates/emails/transactional/automatic-disable-alias.txt +++ b/templates/emails/transactional/automatic-disable-alias.txt @@ -3,7 +3,9 @@ Hi {{name}} There are at least 2 emails sent to your alias {{alias}} from {{website_from}} that have been refused (or bounced) by your email provider. This is usually due to the email being considered as spam by your email provider. -The email is included at the end of this message so you can take a look at its content. +You can view this email here: +{{ refused_email_url }} + As security measure, we have disabled the alias {{alias}}. From a923d9ad6adfd4bf39d56d15d92b7841f8262bc7 Mon Sep 17 00:00:00 2001 From: Son NK Date: Sun, 15 Mar 2020 11:10:37 +0100 Subject: [PATCH 08/16] Add refused_email.deleted column --- app/models.py | 3 ++ .../versions/2020_031510_9081f1a90939_.py | 29 +++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 migrations/versions/2020_031510_9081f1a90939_.py diff --git a/app/models.py b/app/models.py index 7e1e28cb..8b02d430 100644 --- a/app/models.py +++ b/app/models.py @@ -986,6 +986,9 @@ class RefusedEmail(db.Model, ModelMixin): # the email content will be deleted at this date delete_at = db.Column(ArrowType, nullable=False, default=_expiration_7d) + # toggle this when email content (stored at full_report_path & path are deleted) + deleted = db.Column(db.Boolean, nullable=False, default=False, server_default="0") + def get_url(self, expires_in=3600): return s3.get_url(self.path, expires_in) diff --git a/migrations/versions/2020_031510_9081f1a90939_.py b/migrations/versions/2020_031510_9081f1a90939_.py new file mode 100644 index 00000000..a723fcd3 --- /dev/null +++ b/migrations/versions/2020_031510_9081f1a90939_.py @@ -0,0 +1,29 @@ +"""empty message + +Revision ID: 9081f1a90939 +Revises: 11a35b448f83 +Create Date: 2020-03-15 10:51:17.341046 + +""" +import sqlalchemy_utils +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '9081f1a90939' +down_revision = '11a35b448f83' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('refused_email', sa.Column('deleted', sa.Boolean(), server_default='0', nullable=False)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('refused_email', 'deleted') + # ### end Alembic commands ### From 71a9fc38a9f3583209edcd40763e04d013cdef5d Mon Sep 17 00:00:00 2001 From: Son NK Date: Sun, 15 Mar 2020 11:11:16 +0100 Subject: [PATCH 09/16] Add cronjob to delete refused emails --- app/s3.py | 8 ++++++++ cron.py | 21 +++++++++++++++++++++ crontab.yml | 6 ++++++ 3 files changed, 35 insertions(+) diff --git a/app/s3.py b/app/s3.py index 1008d0ca..133f9433 100644 --- a/app/s3.py +++ b/app/s3.py @@ -53,3 +53,11 @@ def get_url(key: str, expires_in=3600) -> str: ClientMethod="get_object", Params={"Bucket": BUCKET, "Key": key}, ) + + +def delete(path: str): + if LOCAL_FILE_UPLOAD: + os.remove(os.path.join(UPLOAD_DIR, path)) + else: + o = _session.resource("s3").Bucket(BUCKET).Object(path) + o.delete() diff --git a/cron.py b/cron.py index 0c1a7d66..fa0a742c 100644 --- a/cron.py +++ b/cron.py @@ -2,6 +2,7 @@ import argparse import arrow +from app import s3 from app.config import IGNORED_EMAILS, ADMIN_EMAIL from app.email_utils import send_email, send_trial_end_soon_email, render from app.extensions import db @@ -15,6 +16,7 @@ from app.models import ( CustomDomain, Client, ManualSubscription, + RefusedEmail, ) from server import create_app @@ -30,6 +32,21 @@ def notify_trial_end(): send_trial_end_soon_email(user) +def delete_refused_emails(): + for refused_email in RefusedEmail.query.filter(RefusedEmail.deleted == False).all(): + if arrow.now().shift(days=1) > refused_email.deleted_at >= arrow.now(): + LOG.d("Delete refused email %s", refused_email) + s3.delete(refused_email.path) + s3.delete(refused_email.full_report_path) + + # do not set path and full_report_path to null + # so we can check later that the files are indeed deleted + refused_email.deleted = True + db.session.commit() + + LOG.d("Finish delete_refused_emails") + + def notify_premium_end(): """sent to user who has canceled their subscription and who has their subscription ending soon""" for sub in Subscription.query.filter(Subscription.cancelled == True).all(): @@ -172,6 +189,7 @@ if __name__ == "__main__": "notify_trial_end", "notify_manual_subscription_end", "notify_premium_end", + "delete_refused_emails" ], ) args = parser.parse_args() @@ -191,3 +209,6 @@ if __name__ == "__main__": elif args.job == "notify_premium_end": LOG.d("Notify users with premium ending soon") notify_premium_end() + elif args.job == "delete_refused_emails": + LOG.d("Deleted refused emails") + delete_refused_emails() diff --git a/crontab.yml b/crontab.yml index 6b742e59..0c453757 100644 --- a/crontab.yml +++ b/crontab.yml @@ -22,3 +22,9 @@ jobs: shell: /bin/bash schedule: "0 10 * * *" captureStderr: true + + - name: SimpleLogin Delete Refused Emails + command: python /code/cron.py -j delete_refused_emails + shell: /bin/bash + schedule: "0 11 * * *" + captureStderr: true From e21e27eefad0244d797f3b165473f55352a7b1cb Mon Sep 17 00:00:00 2001 From: Son NK Date: Sun, 15 Mar 2020 11:14:58 +0100 Subject: [PATCH 10/16] Hide download for deleted refused emails --- app/dashboard/templates/dashboard/refused_email.html | 9 ++++++--- cron.py | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/app/dashboard/templates/dashboard/refused_email.html b/app/dashboard/templates/dashboard/refused_email.html index a42b7ec2..64c68394 100644 --- a/app/dashboard/templates/dashboard/refused_email.html +++ b/app/dashboard/templates/dashboard/refused_email.html @@ -27,9 +27,12 @@ Sent {{ refused_email.created_at | dt }}
- - View → -
This will download a ".eml" file that you can open in your favorite email client
+ {% if refused_email.deleted %} + Email deleted {{ refused_email.deleted_at | dt }} + {% else %} + Download → +
This will download a ".eml" file that you can open in your favorite email client
+ {% endif %}
{% endfor %} diff --git a/cron.py b/cron.py index fa0a742c..3e0873c7 100644 --- a/cron.py +++ b/cron.py @@ -189,7 +189,7 @@ if __name__ == "__main__": "notify_trial_end", "notify_manual_subscription_end", "notify_premium_end", - "delete_refused_emails" + "delete_refused_emails", ], ) args = parser.parse_args() From 45d560fd7002a1e43543c2834abbf50aee9d05ce Mon Sep 17 00:00:00 2001 From: Son NK Date: Sun, 15 Mar 2020 12:14:43 +0100 Subject: [PATCH 11/16] fix --- app/dashboard/templates/dashboard/refused_email.html | 2 +- cron.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/dashboard/templates/dashboard/refused_email.html b/app/dashboard/templates/dashboard/refused_email.html index 64c68394..27b7e8b1 100644 --- a/app/dashboard/templates/dashboard/refused_email.html +++ b/app/dashboard/templates/dashboard/refused_email.html @@ -28,7 +28,7 @@
{% if refused_email.deleted %} - Email deleted {{ refused_email.deleted_at | dt }} + Email deleted {{ refused_email.delete_at | dt }} {% else %} Download →
This will download a ".eml" file that you can open in your favorite email client
diff --git a/cron.py b/cron.py index 3e0873c7..3bd08891 100644 --- a/cron.py +++ b/cron.py @@ -34,7 +34,7 @@ def notify_trial_end(): def delete_refused_emails(): for refused_email in RefusedEmail.query.filter(RefusedEmail.deleted == False).all(): - if arrow.now().shift(days=1) > refused_email.deleted_at >= arrow.now(): + if arrow.now().shift(days=1) > refused_email.delete_at >= arrow.now(): LOG.d("Delete refused email %s", refused_email) s3.delete(refused_email.path) s3.delete(refused_email.full_report_path) From 9cdf766825aa6686cfa70fa57f2a4c8d440d925b Mon Sep 17 00:00:00 2001 From: Son NK Date: Sun, 15 Mar 2020 12:15:11 +0100 Subject: [PATCH 12/16] Send refused email notif to user email instead of mailbox --- email_handler.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/email_handler.py b/email_handler.py index 6f5c88fd..ed7ee9b4 100644 --- a/email_handler.py +++ b/email_handler.py @@ -572,7 +572,8 @@ def handle_bounce( alias, ) send_email( - mailbox_email, + # use user mail here as only user is authenticated to see the refused email + user.email, f"Email from {forward_email.website_from} to {alias} cannot be delivered to your inbox", render( "transactional/bounced-email.txt", @@ -582,6 +583,7 @@ def handle_bounce( website_email=forward_email.website_email, disable_alias_link=disable_alias_link, refused_email_url=refused_email_url, + mailbox_email=mailbox_email ), render( "transactional/bounced-email.html", @@ -591,6 +593,7 @@ def handle_bounce( website_email=forward_email.website_email, disable_alias_link=disable_alias_link, refused_email_url=refused_email_url, + mailbox_email=mailbox_email ), # cannot include bounce email as it can contain spammy text # bounced_email=msg, @@ -606,7 +609,8 @@ def handle_bounce( db.session.commit() send_email( - mailbox_email, + # use user mail here as only user is authenticated to see the refused email + user.email, f"Alias {alias} has been disabled due to second undelivered email from {forward_email.website_from}", render( "transactional/automatic-disable-alias.txt", @@ -615,6 +619,7 @@ def handle_bounce( website_from=forward_email.website_from, website_email=forward_email.website_email, refused_email_url=refused_email_url, + mailbox_email=mailbox_email ), render( "transactional/automatic-disable-alias.html", @@ -623,6 +628,7 @@ def handle_bounce( website_from=forward_email.website_from, website_email=forward_email.website_email, refused_email_url=refused_email_url, + mailbox_email=mailbox_email ), # cannot include bounce email as it can contain spammy text # bounced_email=msg, From 4b21f49f4905aec35110902b78822092c60dedbc Mon Sep 17 00:00:00 2001 From: Son NK Date: Sun, 15 Mar 2020 12:25:01 +0100 Subject: [PATCH 13/16] add mailbox email into notif email --- templates/emails/transactional/automatic-disable-alias.html | 6 +++--- templates/emails/transactional/automatic-disable-alias.txt | 2 +- templates/emails/transactional/bounced-email.html | 2 +- templates/emails/transactional/bounced-email.txt | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/templates/emails/transactional/automatic-disable-alias.html b/templates/emails/transactional/automatic-disable-alias.html index 87140c40..fa17c8b9 100644 --- a/templates/emails/transactional/automatic-disable-alias.html +++ b/templates/emails/transactional/automatic-disable-alias.html @@ -2,14 +2,14 @@ {% block content %} {{ render_text("Hi " + name) }} - {{ render_text("There are at least 2 emails sent to your alias " + alias + " from " + website_email + - " that have been refused (or bounced) by your email provider.") }} + {{ render_text("There are at least 2 emails sent to your alias " + alias + "" + " (mailbox "+ mailbox_email + ") from " + website_email + " that have been refused (or bounced) by your email provider.") }} + {{ render_text("This is usually due to the email being considered as spam by your email provider.") }} {{ render_button("View the refused email", refused_email_url) }} - {{ render_text('As security measure, we have disabled the alias ' + alias) }} + {{ render_text('As security measure, we have disabled the alias ' + alias + ".") }} {{ render_text('Please let us know if you have any question.') }} diff --git a/templates/emails/transactional/automatic-disable-alias.txt b/templates/emails/transactional/automatic-disable-alias.txt index 639da5c1..5d03a643 100644 --- a/templates/emails/transactional/automatic-disable-alias.txt +++ b/templates/emails/transactional/automatic-disable-alias.txt @@ -1,6 +1,6 @@ Hi {{name}} -There are at least 2 emails sent to your alias {{alias}} from {{website_from}} that have been refused (or bounced) by your email provider. +There are at least 2 emails sent to your alias {{alias}} (mailbox {{mailbox_email}}) from {{website_from}} that have been refused (or bounced) by your email provider. This is usually due to the email being considered as spam by your email provider. You can view this email here: diff --git a/templates/emails/transactional/bounced-email.html b/templates/emails/transactional/bounced-email.html index 2358289d..29829b06 100644 --- a/templates/emails/transactional/bounced-email.html +++ b/templates/emails/transactional/bounced-email.html @@ -2,7 +2,7 @@ {% block content %} {{ render_text("Hi " + name) }} - {{ render_text("An email sent to your alias " + alias + " from " + website_email + " was refused (or bounced) by your email provider.") }} + {{ render_text("An email sent to your alias " + alias + "" + " (mailbox "+ mailbox_email + ") from " + website_email + " was refused (or bounced) by your email provider.") }} {{ render_text('This is usually due to the email being considered as spam by your email provider.') }} diff --git a/templates/emails/transactional/bounced-email.txt b/templates/emails/transactional/bounced-email.txt index 8b8bcb8f..e8625b3e 100644 --- a/templates/emails/transactional/bounced-email.txt +++ b/templates/emails/transactional/bounced-email.txt @@ -1,6 +1,6 @@ Hi {{name}} -An email sent to your alias {{alias}} from {{website_from}} was refused (or bounced) by your email provider. +An email sent to your alias {{alias}} (mailbox {{mailbox_email}}) from {{website_from}} was refused (or bounced) by your email provider. This is usually due to the email being considered as spam by your email provider. You can view this email here: From b3977e5efdacd97123f445ea5aee96889622aeeb Mon Sep 17 00:00:00 2001 From: Son NK Date: Sun, 15 Mar 2020 12:26:35 +0100 Subject: [PATCH 14/16] reformat --- email_handler.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/email_handler.py b/email_handler.py index ed7ee9b4..1a69b6ed 100644 --- a/email_handler.py +++ b/email_handler.py @@ -583,7 +583,7 @@ def handle_bounce( website_email=forward_email.website_email, disable_alias_link=disable_alias_link, refused_email_url=refused_email_url, - mailbox_email=mailbox_email + mailbox_email=mailbox_email, ), render( "transactional/bounced-email.html", @@ -593,7 +593,7 @@ def handle_bounce( website_email=forward_email.website_email, disable_alias_link=disable_alias_link, refused_email_url=refused_email_url, - mailbox_email=mailbox_email + mailbox_email=mailbox_email, ), # cannot include bounce email as it can contain spammy text # bounced_email=msg, @@ -619,7 +619,7 @@ def handle_bounce( website_from=forward_email.website_from, website_email=forward_email.website_email, refused_email_url=refused_email_url, - mailbox_email=mailbox_email + mailbox_email=mailbox_email, ), render( "transactional/automatic-disable-alias.html", @@ -628,7 +628,7 @@ def handle_bounce( website_from=forward_email.website_from, website_email=forward_email.website_email, refused_email_url=refused_email_url, - mailbox_email=mailbox_email + mailbox_email=mailbox_email, ), # cannot include bounce email as it can contain spammy text # bounced_email=msg, From eb3063a57f13f25f56a3e7f828a0f550bb334b3b Mon Sep 17 00:00:00 2001 From: Son NK Date: Sun, 15 Mar 2020 18:06:57 +0100 Subject: [PATCH 15/16] Improve wording --- .../templates/dashboard/refused_email.html | 13 ++++++++++++- .../transactional/automatic-disable-alias.html | 2 +- .../transactional/automatic-disable-alias.txt | 2 +- templates/emails/transactional/bounced-email.html | 2 +- templates/emails/transactional/bounced-email.txt | 2 +- 5 files changed, 16 insertions(+), 5 deletions(-) diff --git a/app/dashboard/templates/dashboard/refused_email.html b/app/dashboard/templates/dashboard/refused_email.html index 27b7e8b1..12cdab78 100644 --- a/app/dashboard/templates/dashboard/refused_email.html +++ b/app/dashboard/templates/dashboard/refused_email.html @@ -10,6 +10,16 @@

Refused Emails

+
+ This page shows all emails that have been refused (or bounced) by your mailbox.
+ Usually this is because your mailbox thinks these emails are spams.
+ - If a refused email is indeed spam, this means the alias is now in the hands of a spammer, + in this case you should disable this alias.
+ - Otherwise, you should create a filter to avoid your email provider from blocking these emails.
+ Contact us↗ if you need any help. + +
+ {% if fels|length == 0 %}
You don't have any refused email. @@ -30,7 +40,8 @@ {% if refused_email.deleted %} Email deleted {{ refused_email.delete_at | dt }} {% else %} - Download → + Download →
This will download a ".eml" file that you can open in your favorite email client
{% endif %} diff --git a/templates/emails/transactional/automatic-disable-alias.html b/templates/emails/transactional/automatic-disable-alias.html index fa17c8b9..2a040814 100644 --- a/templates/emails/transactional/automatic-disable-alias.html +++ b/templates/emails/transactional/automatic-disable-alias.html @@ -2,7 +2,7 @@ {% block content %} {{ render_text("Hi " + name) }} - {{ render_text("There are at least 2 emails sent to your alias " + alias + "" + " (mailbox "+ mailbox_email + ") from " + website_email + " that have been refused (or bounced) by your email provider.") }} + {{ render_text("There are at least 2 emails sent to your alias " + alias + " from " + website_email + " that have been refused (or bounced) by your mailbox " + mailbox_email + ".") }} {{ render_text("This is usually due to the email being considered as spam by your email provider.") }} diff --git a/templates/emails/transactional/automatic-disable-alias.txt b/templates/emails/transactional/automatic-disable-alias.txt index 5d03a643..db6a5c21 100644 --- a/templates/emails/transactional/automatic-disable-alias.txt +++ b/templates/emails/transactional/automatic-disable-alias.txt @@ -1,6 +1,6 @@ Hi {{name}} -There are at least 2 emails sent to your alias {{alias}} (mailbox {{mailbox_email}}) from {{website_from}} that have been refused (or bounced) by your email provider. +There are at least 2 emails sent to your alias {{alias}} from {{website_from}} that have been refused (or bounced) by your mailbox {{mailbox_email}}. This is usually due to the email being considered as spam by your email provider. You can view this email here: diff --git a/templates/emails/transactional/bounced-email.html b/templates/emails/transactional/bounced-email.html index 29829b06..41179c62 100644 --- a/templates/emails/transactional/bounced-email.html +++ b/templates/emails/transactional/bounced-email.html @@ -2,7 +2,7 @@ {% block content %} {{ render_text("Hi " + name) }} - {{ render_text("An email sent to your alias " + alias + "" + " (mailbox "+ mailbox_email + ") from " + website_email + " was refused (or bounced) by your email provider.") }} + {{ render_text("An email sent to your alias " + alias + " from " + website_email + " was refused (or bounced) by your mailbox " + mailbox_email + ".") }} {{ render_text('This is usually due to the email being considered as spam by your email provider.') }} diff --git a/templates/emails/transactional/bounced-email.txt b/templates/emails/transactional/bounced-email.txt index e8625b3e..c47ca855 100644 --- a/templates/emails/transactional/bounced-email.txt +++ b/templates/emails/transactional/bounced-email.txt @@ -1,6 +1,6 @@ Hi {{name}} -An email sent to your alias {{alias}} (mailbox {{mailbox_email}}) from {{website_from}} was refused (or bounced) by your email provider. +An email sent to your alias {{alias}} from {{website_from}} was refused (or bounced) by your mailbox {{mailbox_email}}. This is usually due to the email being considered as spam by your email provider. You can view this email here: From b19be41a5ed3062a18d3db5648c63ec51d71c673 Mon Sep 17 00:00:00 2001 From: Son NK Date: Sun, 15 Mar 2020 18:39:59 +0100 Subject: [PATCH 16/16] Support download email file in browser --- app/s3.py | 23 ++++++++++++++++++- email_handler.py | 10 ++++---- .../automatic-disable-alias.html | 1 + 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/app/s3.py b/app/s3.py index 133f9433..dd09bbf1 100644 --- a/app/s3.py +++ b/app/s3.py @@ -34,7 +34,28 @@ def upload_from_bytesio(key: str, bs: BytesIO, content_type="string"): else: _session.resource("s3").Bucket(BUCKET).put_object( - Key=key, Body=bs, ContentType=content_type + Key=key, Body=bs, ContentType=content_type, + ) + + +def upload_email_from_bytesio(path: str, bs: BytesIO, filename): + bs.seek(0) + + if LOCAL_FILE_UPLOAD: + file_path = os.path.join(UPLOAD_DIR, path) + file_dir = os.path.dirname(file_path) + os.makedirs(file_dir, exist_ok=True) + with open(file_path, "wb") as f: + f.write(bs.read()) + + else: + _session.resource("s3").Bucket(BUCKET).put_object( + Key=path, + Body=bs, + # Support saving a remote file using Http header + # Also supports Safari. More info at + # https://github.com/eligrey/FileSaver.js/wiki/Saving-a-remote-file#using-http-header + ContentDisposition=f'attachment; filename="{filename}.eml";', ) diff --git a/email_handler.py b/email_handler.py index 1a69b6ed..b98143b4 100644 --- a/email_handler.py +++ b/email_handler.py @@ -30,6 +30,7 @@ It should contain the following info: """ +import uuid import time from email import encoders from email.message import Message @@ -540,14 +541,15 @@ def handle_bounce( disable_alias_link = f"{URL}/dashboard/unsubscribe/{gen_email.id}" # Store the bounced email - random_name = random_string(50) + orig_msg = get_orig_message_from_bounce(msg) + # generate a name for the email + random_name = str(uuid.uuid4()) full_report_path = f"refused-emails/full-{random_name}.eml" - s3.upload_from_bytesio(full_report_path, BytesIO(msg.as_bytes())) + s3.upload_email_from_bytesio(full_report_path, BytesIO(msg.as_bytes()), random_name) file_path = f"refused-emails/{random_name}.eml" - orig_msg = get_orig_message_from_bounce(msg) - s3.upload_from_bytesio(file_path, BytesIO(orig_msg.as_bytes())) + s3.upload_email_from_bytesio(file_path, BytesIO(orig_msg.as_bytes()), random_name) refused_email = RefusedEmail.create( path=file_path, full_report_path=full_report_path, user_id=user.id diff --git a/templates/emails/transactional/automatic-disable-alias.html b/templates/emails/transactional/automatic-disable-alias.html index 2a040814..91845973 100644 --- a/templates/emails/transactional/automatic-disable-alias.html +++ b/templates/emails/transactional/automatic-disable-alias.html @@ -14,4 +14,5 @@ {{ render_text('Please let us know if you have any question.') }} {{ render_text('Thanks,
SimpleLogin Team.') }} + {{ raw_url(refused_email_url) }} {% endblock %}