Export Data
diff --git a/app/dashboard/views/refused_email.py b/app/dashboard/views/refused_email.py
new file mode 100644
index 00000000..4b5e84db
--- /dev/null
+++ b/app/dashboard/views/refused_email.py
@@ -0,0 +1,30 @@
+from flask import render_template, request
+from flask_login import login_required
+
+from app.dashboard.base import dashboard_bp
+from app.models import ForwardEmailLog
+
+
+@dashboard_bp.route("/refused_email", methods=["GET", "POST"])
+@login_required
+def refused_email_route():
+ # Highlight a refused email
+ highlight_fel_id = request.args.get("highlight_fel_id")
+ if highlight_fel_id:
+ highlight_fel_id = int(highlight_fel_id)
+
+ fels: [ForwardEmailLog] = ForwardEmailLog.query.filter(
+ ForwardEmailLog.refused_email_id != None
+ ).all()
+
+ # make sure the highlighted fel is the first fel
+ highlight_index = None
+ for ix, fel in enumerate(fels):
+ if fel.id == highlight_fel_id:
+ highlight_index = ix
+ break
+
+ if highlight_index:
+ fels.insert(0, fels.pop(highlight_index))
+
+ return render_template("dashboard/refused_email.html", **locals())
diff --git a/app/email_utils.py b/app/email_utils.py
index ca9a8e44..2099d169 100644
--- a/app/email_utils.py
+++ b/app/email_utils.py
@@ -307,6 +307,15 @@ def delete_header(msg: Message, header: str):
del msg._headers[i]
+def delete_all_headers_except(msg: Message, headers: [str]):
+ headers = [h.lower() for h in headers]
+
+ for i in reversed(range(len(msg._headers))):
+ header_name = msg._headers[i][0].lower()
+ if header_name not in headers:
+ del msg._headers[i]
+
+
def email_belongs_to_alias_domains(email: str) -> bool:
"""return True if an email ends with one of the alias domains provided by SimpleLogin"""
for domain in ALIAS_DOMAINS:
@@ -363,3 +372,18 @@ def mailbox_already_used(email: str, user) -> bool:
return True
return False
+
+
+def get_orig_message_from_bounce(msg: Message) -> Message:
+ """parse the original email from Bounce"""
+ i = 0
+ for part in msg.walk():
+ i += 1
+
+ # the original message is the 4th part
+ # 1st part is the root part, multipart/report
+ # 2nd is text/plain, Postfix log
+ # ...
+ # 7th is original message
+ if i == 7:
+ return part
diff --git a/app/models.py b/app/models.py
index aa18c0a1..8b02d430 100644
--- a/app/models.py
+++ b/app/models.py
@@ -344,6 +344,10 @@ def _expiration_5m():
return arrow.now().shift(minutes=5)
+def _expiration_7d():
+ return arrow.now().shift(days=7)
+
+
class ActivationCode(db.Model, ModelMixin):
"""For activate user account"""
@@ -758,6 +762,14 @@ class ForwardEmailLog(db.Model, ModelMixin):
# usually because the forwarded email is too spammy
bounced = db.Column(db.Boolean, nullable=False, default=False, server_default="0")
+ # Point to the email that has been refused
+ refused_email_id = db.Column(
+ db.ForeignKey("refused_email.id", ondelete="SET NULL"), nullable=True
+ )
+
+ refused_email = db.relationship("RefusedEmail")
+ forward = db.relationship(ForwardEmail)
+
class Subscription(db.Model, ModelMixin):
# Come from Paddle
@@ -958,3 +970,27 @@ class AccountActivation(db.Model, ModelMixin):
CheckConstraint(tries >= 0, name="account_activation_tries_positive"),
{},
)
+
+
+class RefusedEmail(db.Model, ModelMixin):
+ """Store emails that have been refused, i.e. bounced or classified as spams"""
+
+ # Store the full report, including logs from Sending & Receiving MTA
+ full_report_path = db.Column(db.String(128), unique=True, nullable=False)
+
+ # The original email, to display to user
+ path = db.Column(db.String(128), unique=True, nullable=False)
+
+ user_id = db.Column(db.ForeignKey(User.id, ondelete="cascade"), nullable=False)
+
+ # the email content will be deleted at this date
+ delete_at = db.Column(ArrowType, nullable=False, default=_expiration_7d)
+
+ # toggle this when email content (stored at full_report_path & path are deleted)
+ deleted = db.Column(db.Boolean, nullable=False, default=False, server_default="0")
+
+ def get_url(self, expires_in=3600):
+ return s3.get_url(self.path, expires_in)
+
+ def __repr__(self):
+ return f""
diff --git a/app/s3.py b/app/s3.py
index 1008d0ca..dd09bbf1 100644
--- a/app/s3.py
+++ b/app/s3.py
@@ -34,7 +34,28 @@ def upload_from_bytesio(key: str, bs: BytesIO, content_type="string"):
else:
_session.resource("s3").Bucket(BUCKET).put_object(
- Key=key, Body=bs, ContentType=content_type
+ Key=key, Body=bs, ContentType=content_type,
+ )
+
+
+def upload_email_from_bytesio(path: str, bs: BytesIO, filename):
+ bs.seek(0)
+
+ if LOCAL_FILE_UPLOAD:
+ file_path = os.path.join(UPLOAD_DIR, path)
+ file_dir = os.path.dirname(file_path)
+ os.makedirs(file_dir, exist_ok=True)
+ with open(file_path, "wb") as f:
+ f.write(bs.read())
+
+ else:
+ _session.resource("s3").Bucket(BUCKET).put_object(
+ Key=path,
+ Body=bs,
+ # Support saving a remote file using Http header
+ # Also supports Safari. More info at
+ # https://github.com/eligrey/FileSaver.js/wiki/Saving-a-remote-file#using-http-header
+ ContentDisposition=f'attachment; filename="{filename}.eml";',
)
@@ -53,3 +74,11 @@ def get_url(key: str, expires_in=3600) -> str:
ClientMethod="get_object",
Params={"Bucket": BUCKET, "Key": key},
)
+
+
+def delete(path: str):
+ if LOCAL_FILE_UPLOAD:
+ os.remove(os.path.join(UPLOAD_DIR, path))
+ else:
+ o = _session.resource("s3").Bucket(BUCKET).Object(path)
+ o.delete()
diff --git a/cron.py b/cron.py
index 0c1a7d66..3bd08891 100644
--- a/cron.py
+++ b/cron.py
@@ -2,6 +2,7 @@ import argparse
import arrow
+from app import s3
from app.config import IGNORED_EMAILS, ADMIN_EMAIL
from app.email_utils import send_email, send_trial_end_soon_email, render
from app.extensions import db
@@ -15,6 +16,7 @@ from app.models import (
CustomDomain,
Client,
ManualSubscription,
+ RefusedEmail,
)
from server import create_app
@@ -30,6 +32,21 @@ def notify_trial_end():
send_trial_end_soon_email(user)
+def delete_refused_emails():
+ for refused_email in RefusedEmail.query.filter(RefusedEmail.deleted == False).all():
+ if arrow.now().shift(days=1) > refused_email.delete_at >= arrow.now():
+ LOG.d("Delete refused email %s", refused_email)
+ s3.delete(refused_email.path)
+ s3.delete(refused_email.full_report_path)
+
+ # do not set path and full_report_path to null
+ # so we can check later that the files are indeed deleted
+ refused_email.deleted = True
+ db.session.commit()
+
+ LOG.d("Finish delete_refused_emails")
+
+
def notify_premium_end():
"""sent to user who has canceled their subscription and who has their subscription ending soon"""
for sub in Subscription.query.filter(Subscription.cancelled == True).all():
@@ -172,6 +189,7 @@ if __name__ == "__main__":
"notify_trial_end",
"notify_manual_subscription_end",
"notify_premium_end",
+ "delete_refused_emails",
],
)
args = parser.parse_args()
@@ -191,3 +209,6 @@ if __name__ == "__main__":
elif args.job == "notify_premium_end":
LOG.d("Notify users with premium ending soon")
notify_premium_end()
+ elif args.job == "delete_refused_emails":
+ LOG.d("Deleted refused emails")
+ delete_refused_emails()
diff --git a/crontab.yml b/crontab.yml
index 6b742e59..0c453757 100644
--- a/crontab.yml
+++ b/crontab.yml
@@ -22,3 +22,9 @@ jobs:
shell: /bin/bash
schedule: "0 10 * * *"
captureStderr: true
+
+ - name: SimpleLogin Delete Refused Emails
+ command: python /code/cron.py -j delete_refused_emails
+ shell: /bin/bash
+ schedule: "0 11 * * *"
+ captureStderr: true
diff --git a/email_handler.py b/email_handler.py
index e1f05306..b98143b4 100644
--- a/email_handler.py
+++ b/email_handler.py
@@ -30,6 +30,7 @@ It should contain the following info:
"""
+import uuid
import time
from email import encoders
from email.message import Message
@@ -37,21 +38,19 @@ from email.mime.application import MIMEApplication
from email.mime.multipart import MIMEMultipart
from email.parser import Parser
from email.policy import SMTPUTF8
+from io import BytesIO
from smtplib import SMTP
from typing import Optional
from aiosmtpd.controller import Controller
-import gnupg
+from app import pgp_utils, s3
from app.config import (
EMAIL_DOMAIN,
POSTFIX_SERVER,
URL,
ALIAS_DOMAINS,
- ADMIN_EMAIL,
- SUPPORT_EMAIL,
POSTFIX_SUBMISSION_TLS,
- GNUPGHOME,
)
from app.email_utils import (
get_email_name,
@@ -65,6 +64,8 @@ from app.email_utils import (
send_cannot_create_domain_alias,
email_belongs_to_alias_domains,
render,
+ get_orig_message_from_bounce,
+ delete_all_headers_except,
)
from app.extensions import db
from app.log import LOG
@@ -76,10 +77,10 @@ from app.models import (
Directory,
User,
DeletedAlias,
+ RefusedEmail,
)
from app.utils import random_string
from server import create_app
-from app import pgp_utils
# fix the database connection leak issue
@@ -270,6 +271,17 @@ def prepare_pgp_message(orig_msg: Message, pgp_fingerprint: str):
if header_name != "Content-Type".lower():
msg[header_name] = orig_msg._headers[i][1]
+ # Delete unnecessary headers in orig_msg except to save space
+ delete_all_headers_except(
+ orig_msg,
+ [
+ "MIME-Version",
+ "Content-Type",
+ "Content-Disposition",
+ "Content-Transfer-Encoding",
+ ],
+ )
+
first = MIMEApplication(
_subtype="pgp-encrypted", _encoder=encoders.encode_7or8bit, _data=""
)
@@ -406,7 +418,12 @@ def handle_reply(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> str:
# in this case Postfix will try to send a bounce report to original sender, which is
# the "reply email"
if envelope.mail_from == "<>":
- LOG.error("Bounce when sending to alias %s, user %s", alias, gen_email.user)
+ LOG.error(
+ "Bounce when sending to alias %s from %s, user %s",
+ alias,
+ forward_email.website_from,
+ gen_email.user,
+ )
handle_bounce(
alias, envelope, forward_email, gen_email, msg, smtp, user, mailbox_email
@@ -513,7 +530,9 @@ def handle_reply(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> str:
def handle_bounce(
alias, envelope, forward_email, gen_email, msg, smtp, user, mailbox_email
):
- ForwardEmailLog.create(forward_id=forward_email.id, bounced=True)
+ fel: ForwardEmailLog = ForwardEmailLog.create(
+ forward_id=forward_email.id, bounced=True
+ )
db.session.commit()
nb_bounced = ForwardEmailLog.filter_by(
@@ -521,6 +540,31 @@ def handle_bounce(
).count()
disable_alias_link = f"{URL}/dashboard/unsubscribe/{gen_email.id}"
+ # Store the bounced email
+ orig_msg = get_orig_message_from_bounce(msg)
+ # generate a name for the email
+ random_name = str(uuid.uuid4())
+
+ full_report_path = f"refused-emails/full-{random_name}.eml"
+ s3.upload_email_from_bytesio(full_report_path, BytesIO(msg.as_bytes()), random_name)
+
+ file_path = f"refused-emails/{random_name}.eml"
+ s3.upload_email_from_bytesio(file_path, BytesIO(orig_msg.as_bytes()), random_name)
+
+ refused_email = RefusedEmail.create(
+ path=file_path, full_report_path=full_report_path, user_id=user.id
+ )
+ db.session.flush()
+
+ fel.refused_email_id = refused_email.id
+ db.session.commit()
+
+ LOG.d("Create refused email %s", refused_email)
+
+ refused_email_url = (
+ URL + f"/dashboard/refused_email?highlight_fel_id=" + str(fel.id)
+ )
+
# inform user if this is the first bounced email
if nb_bounced == 1:
LOG.d(
@@ -530,7 +574,8 @@ def handle_bounce(
alias,
)
send_email(
- mailbox_email,
+ # use user mail here as only user is authenticated to see the refused email
+ user.email,
f"Email from {forward_email.website_from} to {alias} cannot be delivered to your inbox",
render(
"transactional/bounced-email.txt",
@@ -539,6 +584,8 @@ def handle_bounce(
website_from=forward_email.website_from,
website_email=forward_email.website_email,
disable_alias_link=disable_alias_link,
+ refused_email_url=refused_email_url,
+ mailbox_email=mailbox_email,
),
render(
"transactional/bounced-email.html",
@@ -547,8 +594,11 @@ def handle_bounce(
website_from=forward_email.website_from,
website_email=forward_email.website_email,
disable_alias_link=disable_alias_link,
+ refused_email_url=refused_email_url,
+ mailbox_email=mailbox_email,
),
- bounced_email=msg,
+ # cannot include bounce email as it can contain spammy text
+ # bounced_email=msg,
)
# disable the alias the second time email is bounced
elif nb_bounced >= 2:
@@ -561,7 +611,8 @@ def handle_bounce(
db.session.commit()
send_email(
- mailbox_email,
+ # use user mail here as only user is authenticated to see the refused email
+ user.email,
f"Alias {alias} has been disabled due to second undelivered email from {forward_email.website_from}",
render(
"transactional/automatic-disable-alias.txt",
@@ -569,6 +620,8 @@ def handle_bounce(
alias=alias,
website_from=forward_email.website_from,
website_email=forward_email.website_email,
+ refused_email_url=refused_email_url,
+ mailbox_email=mailbox_email,
),
render(
"transactional/automatic-disable-alias.html",
@@ -576,8 +629,11 @@ def handle_bounce(
alias=alias,
website_from=forward_email.website_from,
website_email=forward_email.website_email,
+ refused_email_url=refused_email_url,
+ mailbox_email=mailbox_email,
),
- bounced_email=msg,
+ # cannot include bounce email as it can contain spammy text
+ # bounced_email=msg,
)
diff --git a/init_app.py b/init_app.py
index bf5fc728..54bb1239 100644
--- a/init_app.py
+++ b/init_app.py
@@ -22,6 +22,8 @@ def load_pgp_public_keys(app):
db.session.commit()
+ LOG.d("Finish load_pgp_public_keys")
+
if __name__ == "__main__":
app = create_app()
diff --git a/migrations/versions/2020_031416_11a35b448f83_.py b/migrations/versions/2020_031416_11a35b448f83_.py
new file mode 100644
index 00000000..49b0d34a
--- /dev/null
+++ b/migrations/versions/2020_031416_11a35b448f83_.py
@@ -0,0 +1,45 @@
+"""empty message
+
+Revision ID: 11a35b448f83
+Revises: 628a5438295c
+Create Date: 2020-03-14 16:35:13.564982
+
+"""
+import sqlalchemy_utils
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '11a35b448f83'
+down_revision = '628a5438295c'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.create_table('refused_email',
+ sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+ sa.Column('created_at', sqlalchemy_utils.types.arrow.ArrowType(), nullable=False),
+ sa.Column('updated_at', sqlalchemy_utils.types.arrow.ArrowType(), nullable=True),
+ sa.Column('full_report_path', sa.String(length=128), nullable=False),
+ sa.Column('path', sa.String(length=128), nullable=False),
+ sa.Column('user_id', sa.Integer(), nullable=False),
+ sa.Column('delete_at', sqlalchemy_utils.types.arrow.ArrowType(), nullable=False),
+ sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='cascade'),
+ sa.PrimaryKeyConstraint('id'),
+ sa.UniqueConstraint('full_report_path'),
+ sa.UniqueConstraint('path')
+ )
+ op.add_column('forward_email_log', sa.Column('refused_email_id', sa.Integer(), nullable=True))
+ op.create_foreign_key(None, 'forward_email_log', 'refused_email', ['refused_email_id'], ['id'], ondelete='SET NULL')
+ # ### end Alembic commands ###
+
+
+def downgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.drop_constraint(None, 'forward_email_log', type_='foreignkey')
+ op.drop_column('forward_email_log', 'refused_email_id')
+ op.drop_table('refused_email')
+ # ### end Alembic commands ###
diff --git a/migrations/versions/2020_031510_9081f1a90939_.py b/migrations/versions/2020_031510_9081f1a90939_.py
new file mode 100644
index 00000000..a723fcd3
--- /dev/null
+++ b/migrations/versions/2020_031510_9081f1a90939_.py
@@ -0,0 +1,29 @@
+"""empty message
+
+Revision ID: 9081f1a90939
+Revises: 11a35b448f83
+Create Date: 2020-03-15 10:51:17.341046
+
+"""
+import sqlalchemy_utils
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '9081f1a90939'
+down_revision = '11a35b448f83'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.add_column('refused_email', sa.Column('deleted', sa.Boolean(), server_default='0', nullable=False))
+ # ### end Alembic commands ###
+
+
+def downgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.drop_column('refused_email', 'deleted')
+ # ### end Alembic commands ###
diff --git a/server.py b/server.py
index 46abbff9..24c27d0f 100644
--- a/server.py
+++ b/server.py
@@ -151,7 +151,7 @@ def fake_data():
db.session.commit()
api_key = ApiKey.create(user_id=user.id, name="Chrome")
- api_key.code = "codeCH"
+ api_key.code = "code"
api_key = ApiKey.create(user_id=user.id, name="Firefox")
api_key.code = "codeFF"
diff --git a/templates/emails/transactional/automatic-disable-alias.html b/templates/emails/transactional/automatic-disable-alias.html
index d3ff7a50..91845973 100644
--- a/templates/emails/transactional/automatic-disable-alias.html
+++ b/templates/emails/transactional/automatic-disable-alias.html
@@ -2,15 +2,17 @@
{% block content %}
{{ render_text("Hi " + name) }}
- {{ render_text("There are at least 2 emails sent to your alias " + alias + " from " + website_email +
- " that have been refused (or bounced) by your email provider.") }}
+ {{ render_text("There are at least 2 emails sent to your alias " + alias + " from " + website_email + " that have been refused (or bounced) by your mailbox " + mailbox_email + ".") }}
- {{ render_text("This is usually due to the email being considered as spam by your email provider.
- The email is included at the end of this message so you can take a look at its content.") }}
- {{ render_text('As security measure, we have disabled the alias ' + alias) }}
+ {{ render_text("This is usually due to the email being considered as spam by your email provider.") }}
+
+ {{ render_button("View the refused email", refused_email_url) }}
+
+ {{ render_text('As security measure, we have disabled the alias ' + alias + ".") }}
{{ render_text('Please let us know if you have any question.') }}
{{ render_text('Thanks,
SimpleLogin Team.') }}
+ {{ raw_url(refused_email_url) }}
{% endblock %}
diff --git a/templates/emails/transactional/automatic-disable-alias.txt b/templates/emails/transactional/automatic-disable-alias.txt
index 5de9afc7..db6a5c21 100644
--- a/templates/emails/transactional/automatic-disable-alias.txt
+++ b/templates/emails/transactional/automatic-disable-alias.txt
@@ -1,9 +1,11 @@
Hi {{name}}
-There are at least 2 emails sent to your alias {{alias}} from {{website_from}} that have been refused (or bounced) by your email provider.
+There are at least 2 emails sent to your alias {{alias}} from {{website_from}} that have been refused (or bounced) by your mailbox {{mailbox_email}}.
This is usually due to the email being considered as spam by your email provider.
-The email is included at the end of this message so you can take a look at its content.
+You can view this email here:
+{{ refused_email_url }}
+
As security measure, we have disabled the alias {{alias}}.
diff --git a/templates/emails/transactional/bounced-email.html b/templates/emails/transactional/bounced-email.html
index e1e3361e..41179c62 100644
--- a/templates/emails/transactional/bounced-email.html
+++ b/templates/emails/transactional/bounced-email.html
@@ -2,9 +2,11 @@
{% block content %}
{{ render_text("Hi " + name) }}
- {{ render_text("An email sent to your alias " + alias + " from " + website_email + " was refused (or bounced) by your email provider.") }}
+ {{ render_text("An email sent to your alias " + alias + " from " + website_email + " was refused (or bounced) by your mailbox " + mailbox_email + ".") }}
- {{ render_text("This is usually due to the email being considered as spam by your email provider. The email is included at the end of this message so you can take a look at its content.") }}
+ {{ render_text('This is usually due to the email being considered as spam by your email provider.') }}
+
+ {{ render_button("View the refused email", refused_email_url) }}
{{ render_text('To avoid spams forwarded by SimpleLogin server, please consider the following options:') }}
diff --git a/templates/emails/transactional/bounced-email.txt b/templates/emails/transactional/bounced-email.txt
index 0fd0c688..c47ca855 100644
--- a/templates/emails/transactional/bounced-email.txt
+++ b/templates/emails/transactional/bounced-email.txt
@@ -1,9 +1,10 @@
Hi {{name}}
-An email sent to your alias {{alias}} from {{website_from}} was refused (or bounced) by your email provider.
+An email sent to your alias {{alias}} from {{website_from}} was refused (or bounced) by your mailbox {{mailbox_email}}.
This is usually due to the email being considered as spam by your email provider.
-The email is included at the end of this message so you can take a look at its content.
+You can view this email here:
+{{ refused_email_url }}
To avoid spams forwarded by SimpleLogin server, please consider the following options: