Mirror
/
app-MAIL-temp
mirror of https://github.com/simple-login/app.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #110 from simple-login/refused-email 

Handle refused email
This commit is contained in:
Son Nguyen Kim 2020-03-15 19:08:35 +01:00 committed by GitHub
parent 28c6c68a80 b19be41a5e
commit 93765335ed
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 382 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
app/dashboard/__init__.py
View File

@ -17,4 +17,5 @@ from .views import (
mailbox,
deleted_alias,
mailbox_detail,
refused_email,
)

53
app/dashboard/templates/dashboard/refused_email.html Normal file
View File

@ -0,0 +1,53 @@
{% extends 'default.html' %}
{% block title %}
Refused Emails
{% endblock %}
{% set active_page = "setting" %}
{% block default_content %}
<div style="max-width: 60em; margin: auto">
<h1 class="h3 mb-5"> Refused Emails </h1>
<div class="alert alert-info">
This page shows all emails that have been <b>refused</b> (or bounced) by your mailbox. <br>
Usually this is because your mailbox thinks these emails are <b>spams</b>. <br>
- If a refused email is indeed spam, this means the alias is now in the hands of a spammer,
in this case you should <b>disable</b> this alias. <br>
- Otherwise, you should create a <b>filter</b> to avoid your email provider from blocking these emails. <br>
<a href="mailto:hi@simplelogin.io">Contact us↗</a> if you need any help.
</div>
{% if fels|length == 0 %}
<div class="my-4 p-4 card">
You don't have any refused email.
</div>
{% endif %}
{% for fel in fels %}
{% set refused_email = fel.refused_email %}
{% set forward = fel.forward %}
<div class="card p-4 shadow-sm {% if fel.id == highlight_fel_id %} highlight-row {% endif %}">
From: {{ forward.website_from or forward.website_email }} <br>
To: {{ forward.gen_email.email }} <br>
<div class="small-text">
Sent {{ refused_email.created_at | dt }}
</div>
{% if refused_email.deleted %}
Email deleted {{ refused_email.delete_at | dt }}
{% else %}
<a href="{{ refused_email.get_url() }}" download
class="mt-4">Download →</a>
<div class="small-text">This will download a ".eml" file that you can open in your favorite email client</div>
{% endif %}
</div>
{% endfor %}
</div>
{% endblock %}

19
app/dashboard/templates/dashboard/setting.html
View File

@ -183,6 +183,25 @@
</div>
</div>
<div class="card">
<div class="card-body">
<div class="card-title">Refused Emails
<div class="small-text mt-1 mb-3" style="max-width: 40rem">
When an email sent to your alias is classified as spam or refused by your email provider,
it usually means your alias has been leaked to a spammer. <br>
In this case SimpleLogin will keep a copy of this email (so it isn't lost)
and notify you so you can take a look at its content and take appropriate actions. <br>
The emails are deleted in 7 days.
This is an exceptional case where SimpleLogin stores the email.
</div>
</div>
<a href="{{ url_for('dashboard.refused_email_route') }}" class="btn btn-outline-primary">
See refused emails
</a>
</div>
</div>
<div class="card">
<div class="card-body">
<div class="card-title">Export Data

30
app/dashboard/views/refused_email.py Normal file
View File

@ -0,0 +1,30 @@
from flask import render_template, request
from flask_login import login_required
from app.dashboard.base import dashboard_bp
from app.models import ForwardEmailLog
@dashboard_bp.route("/refused_email", methods=["GET", "POST"])
@login_required
def refused_email_route():
# Highlight a refused email
highlight_fel_id = request.args.get("highlight_fel_id")
if highlight_fel_id:
highlight_fel_id = int(highlight_fel_id)
fels: [ForwardEmailLog] = ForwardEmailLog.query.filter(
ForwardEmailLog.refused_email_id != None
).all()
# make sure the highlighted fel is the first fel
highlight_index = None
for ix, fel in enumerate(fels):
if fel.id == highlight_fel_id:
highlight_index = ix
break
if highlight_index:
fels.insert(0, fels.pop(highlight_index))
return render_template("dashboard/refused_email.html", **locals())

24
app/email_utils.py
View File

@ -307,6 +307,15 @@ def delete_header(msg: Message, header: str):
del msg._headers[i]
def delete_all_headers_except(msg: Message, headers: [str]):
headers = [h.lower() for h in headers]
for i in reversed(range(len(msg._headers))):
header_name = msg._headers[i][0].lower()
if header_name not in headers:
del msg._headers[i]
def email_belongs_to_alias_domains(email: str) -> bool:
"""return True if an email ends with one of the alias domains provided by SimpleLogin"""
for domain in ALIAS_DOMAINS:
@ -363,3 +372,18 @@ def mailbox_already_used(email: str, user) -> bool:
return True
return False
def get_orig_message_from_bounce(msg: Message) -> Message:
"""parse the original email from Bounce"""
i = 0
for part in msg.walk():
i += 1
# the original message is the 4th part
# 1st part is the root part, multipart/report
# 2nd is text/plain, Postfix log
# ...
# 7th is original message
if i == 7:
return part

36
app/models.py
View File

@ -344,6 +344,10 @@ def _expiration_5m():
return arrow.now().shift(minutes=5)
def _expiration_7d():
return arrow.now().shift(days=7)
class ActivationCode(db.Model, ModelMixin):
"""For activate user account"""
@ -758,6 +762,14 @@ class ForwardEmailLog(db.Model, ModelMixin):
# usually because the forwarded email is too spammy
bounced = db.Column(db.Boolean, nullable=False, default=False, server_default="0")
# Point to the email that has been refused
refused_email_id = db.Column(
db.ForeignKey("refused_email.id", ondelete="SET NULL"), nullable=True
)
refused_email = db.relationship("RefusedEmail")
forward = db.relationship(ForwardEmail)
class Subscription(db.Model, ModelMixin):
# Come from Paddle
@ -958,3 +970,27 @@ class AccountActivation(db.Model, ModelMixin):
CheckConstraint(tries >= 0, name="account_activation_tries_positive"),
{},
)
class RefusedEmail(db.Model, ModelMixin):
"""Store emails that have been refused, i.e. bounced or classified as spams"""
# Store the full report, including logs from Sending & Receiving MTA
full_report_path = db.Column(db.String(128), unique=True, nullable=False)
# The original email, to display to user
path = db.Column(db.String(128), unique=True, nullable=False)
user_id = db.Column(db.ForeignKey(User.id, ondelete="cascade"), nullable=False)
# the email content will be deleted at this date
delete_at = db.Column(ArrowType, nullable=False, default=_expiration_7d)
# toggle this when email content (stored at full_report_path & path are deleted)
deleted = db.Column(db.Boolean, nullable=False, default=False, server_default="0")
def get_url(self, expires_in=3600):
return s3.get_url(self.path, expires_in)
def __repr__(self):
return f"<Refused Email {self.id} {self.path} {self.delete_at}>"

31
app/s3.py
View File

@ -34,7 +34,28 @@ def upload_from_bytesio(key: str, bs: BytesIO, content_type="string"):
else:
_session.resource("s3").Bucket(BUCKET).put_object(
Key=key, Body=bs, ContentType=content_type
Key=key, Body=bs, ContentType=content_type,
)
def upload_email_from_bytesio(path: str, bs: BytesIO, filename):
bs.seek(0)
if LOCAL_FILE_UPLOAD:
file_path = os.path.join(UPLOAD_DIR, path)
file_dir = os.path.dirname(file_path)
os.makedirs(file_dir, exist_ok=True)
with open(file_path, "wb") as f:
f.write(bs.read())
else:
_session.resource("s3").Bucket(BUCKET).put_object(
Key=path,
Body=bs,
# Support saving a remote file using Http header
# Also supports Safari. More info at
# https://github.com/eligrey/FileSaver.js/wiki/Saving-a-remote-file#using-http-header
ContentDisposition=f'attachment; filename="{filename}.eml";',
)
@ -53,3 +74,11 @@ def get_url(key: str, expires_in=3600) -> str:
ClientMethod="get_object",
Params={"Bucket": BUCKET, "Key": key},
)
def delete(path: str):
if LOCAL_FILE_UPLOAD:
os.remove(os.path.join(UPLOAD_DIR, path))
else:
o = _session.resource("s3").Bucket(BUCKET).Object(path)
o.delete()

21
cron.py
View File

@ -2,6 +2,7 @@ import argparse
import arrow
from app import s3
from app.config import IGNORED_EMAILS, ADMIN_EMAIL
from app.email_utils import send_email, send_trial_end_soon_email, render
from app.extensions import db
@ -15,6 +16,7 @@ from app.models import (
CustomDomain,
Client,
ManualSubscription,
RefusedEmail,
)
from server import create_app
@ -30,6 +32,21 @@ def notify_trial_end():
send_trial_end_soon_email(user)
def delete_refused_emails():
for refused_email in RefusedEmail.query.filter(RefusedEmail.deleted == False).all():
if arrow.now().shift(days=1) > refused_email.delete_at >= arrow.now():
LOG.d("Delete refused email %s", refused_email)
s3.delete(refused_email.path)
s3.delete(refused_email.full_report_path)
# do not set path and full_report_path to null
# so we can check later that the files are indeed deleted
refused_email.deleted = True
db.session.commit()
LOG.d("Finish delete_refused_emails")
def notify_premium_end():
"""sent to user who has canceled their subscription and who has their subscription ending soon"""
for sub in Subscription.query.filter(Subscription.cancelled == True).all():
@ -172,6 +189,7 @@ if __name__ == "__main__":
"notify_trial_end",
"notify_manual_subscription_end",
"notify_premium_end",
"delete_refused_emails",
],
)
args = parser.parse_args()
@ -191,3 +209,6 @@ if __name__ == "__main__":
elif args.job == "notify_premium_end":
LOG.d("Notify users with premium ending soon")
notify_premium_end()
elif args.job == "delete_refused_emails":
LOG.d("Deleted refused emails")
delete_refused_emails()

6
crontab.yml
View File

@ -22,3 +22,9 @@ jobs:
shell: /bin/bash
schedule: "0 10 * * *"
captureStderr: true
- name: SimpleLogin Delete Refused Emails
command: python /code/cron.py -j delete_refused_emails
shell: /bin/bash
schedule: "0 11 * * *"
captureStderr: true

78
email_handler.py
View File

@ -30,6 +30,7 @@ It should contain the following info:
"""
import uuid
import time
from email import encoders
from email.message import Message
@ -37,21 +38,19 @@ from email.mime.application import MIMEApplication
from email.mime.multipart import MIMEMultipart
from email.parser import Parser
from email.policy import SMTPUTF8
from io import BytesIO
from smtplib import SMTP
from typing import Optional
from aiosmtpd.controller import Controller
import gnupg
from app import pgp_utils, s3
from app.config import (
EMAIL_DOMAIN,
POSTFIX_SERVER,
URL,
ALIAS_DOMAINS,
ADMIN_EMAIL,
SUPPORT_EMAIL,
POSTFIX_SUBMISSION_TLS,
GNUPGHOME,
)
from app.email_utils import (
get_email_name,
@ -65,6 +64,8 @@ from app.email_utils import (
send_cannot_create_domain_alias,
email_belongs_to_alias_domains,
render,
get_orig_message_from_bounce,
delete_all_headers_except,
)
from app.extensions import db
from app.log import LOG
@ -76,10 +77,10 @@ from app.models import (
Directory,
User,
DeletedAlias,
RefusedEmail,
)
from app.utils import random_string
from server import create_app
from app import pgp_utils
# fix the database connection leak issue
@ -270,6 +271,17 @@ def prepare_pgp_message(orig_msg: Message, pgp_fingerprint: str):
if header_name != "Content-Type".lower():
msg[header_name] = orig_msg._headers[i][1]
# Delete unnecessary headers in orig_msg except to save space
delete_all_headers_except(
orig_msg,
[
"MIME-Version",
"Content-Type",
"Content-Disposition",
"Content-Transfer-Encoding",
],
)
first = MIMEApplication(
_subtype="pgp-encrypted", _encoder=encoders.encode_7or8bit, _data=""
)
@ -406,7 +418,12 @@ def handle_reply(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> str:
# in this case Postfix will try to send a bounce report to original sender, which is
# the "reply email"
if envelope.mail_from == "<>":
LOG.error("Bounce when sending to alias %s, user %s", alias, gen_email.user)
LOG.error(
"Bounce when sending to alias %s from %s, user %s",
alias,
forward_email.website_from,
gen_email.user,
)
handle_bounce(
alias, envelope, forward_email, gen_email, msg, smtp, user, mailbox_email
@ -513,7 +530,9 @@ def handle_reply(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> str:
def handle_bounce(
alias, envelope, forward_email, gen_email, msg, smtp, user, mailbox_email
):
ForwardEmailLog.create(forward_id=forward_email.id, bounced=True)
fel: ForwardEmailLog = ForwardEmailLog.create(
forward_id=forward_email.id, bounced=True
)
db.session.commit()
nb_bounced = ForwardEmailLog.filter_by(
@ -521,6 +540,31 @@ def handle_bounce(
).count()
disable_alias_link = f"{URL}/dashboard/unsubscribe/{gen_email.id}"
# Store the bounced email
orig_msg = get_orig_message_from_bounce(msg)
# generate a name for the email
random_name = str(uuid.uuid4())
full_report_path = f"refused-emails/full-{random_name}.eml"
s3.upload_email_from_bytesio(full_report_path, BytesIO(msg.as_bytes()), random_name)
file_path = f"refused-emails/{random_name}.eml"
s3.upload_email_from_bytesio(file_path, BytesIO(orig_msg.as_bytes()), random_name)
refused_email = RefusedEmail.create(
path=file_path, full_report_path=full_report_path, user_id=user.id
)
db.session.flush()
fel.refused_email_id = refused_email.id
db.session.commit()
LOG.d("Create refused email %s", refused_email)
refused_email_url = (
URL + f"/dashboard/refused_email?highlight_fel_id=" + str(fel.id)
)
# inform user if this is the first bounced email
if nb_bounced == 1:
LOG.d(
@ -530,7 +574,8 @@ def handle_bounce(
alias,
)
send_email(
mailbox_email,
# use user mail here as only user is authenticated to see the refused email
user.email,
f"Email from {forward_email.website_from} to {alias} cannot be delivered to your inbox",
render(
"transactional/bounced-email.txt",
@ -539,6 +584,8 @@ def handle_bounce(
website_from=forward_email.website_from,
website_email=forward_email.website_email,
disable_alias_link=disable_alias_link,
refused_email_url=refused_email_url,
mailbox_email=mailbox_email,
),
render(
"transactional/bounced-email.html",
@ -547,8 +594,11 @@ def handle_bounce(
website_from=forward_email.website_from,
website_email=forward_email.website_email,
disable_alias_link=disable_alias_link,
refused_email_url=refused_email_url,
mailbox_email=mailbox_email,
),
bounced_email=msg,
# cannot include bounce email as it can contain spammy text
# bounced_email=msg,
)
# disable the alias the second time email is bounced
elif nb_bounced >= 2:
@ -561,7 +611,8 @@ def handle_bounce(
db.session.commit()
send_email(
mailbox_email,
# use user mail here as only user is authenticated to see the refused email
user.email,
f"Alias {alias} has been disabled due to second undelivered email from {forward_email.website_from}",
render(
"transactional/automatic-disable-alias.txt",
@ -569,6 +620,8 @@ def handle_bounce(
alias=alias,
website_from=forward_email.website_from,
website_email=forward_email.website_email,
refused_email_url=refused_email_url,
mailbox_email=mailbox_email,
),
render(
"transactional/automatic-disable-alias.html",
@ -576,8 +629,11 @@ def handle_bounce(
alias=alias,
website_from=forward_email.website_from,
website_email=forward_email.website_email,
refused_email_url=refused_email_url,
mailbox_email=mailbox_email,
),
bounced_email=msg,
# cannot include bounce email as it can contain spammy text
# bounced_email=msg,
)

2
init_app.py
View File

@ -22,6 +22,8 @@ def load_pgp_public_keys(app):
db.session.commit()
LOG.d("Finish load_pgp_public_keys")
if __name__ == "__main__":
app = create_app()

45
migrations/versions/2020_031416_11a35b448f83_.py Normal file
View File

@ -0,0 +1,45 @@
"""empty message
Revision ID: 11a35b448f83
Revises: 628a5438295c
Create Date: 2020-03-14 16:35:13.564982
"""
import sqlalchemy_utils
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '11a35b448f83'
down_revision = '628a5438295c'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('refused_email',
sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
sa.Column('created_at', sqlalchemy_utils.types.arrow.ArrowType(), nullable=False),
sa.Column('updated_at', sqlalchemy_utils.types.arrow.ArrowType(), nullable=True),
sa.Column('full_report_path', sa.String(length=128), nullable=False),
sa.Column('path', sa.String(length=128), nullable=False),
sa.Column('user_id', sa.Integer(), nullable=False),
sa.Column('delete_at', sqlalchemy_utils.types.arrow.ArrowType(), nullable=False),
sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='cascade'),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('full_report_path'),
sa.UniqueConstraint('path')
)
op.add_column('forward_email_log', sa.Column('refused_email_id', sa.Integer(), nullable=True))
op.create_foreign_key(None, 'forward_email_log', 'refused_email', ['refused_email_id'], ['id'], ondelete='SET NULL')
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_constraint(None, 'forward_email_log', type_='foreignkey')
op.drop_column('forward_email_log', 'refused_email_id')
op.drop_table('refused_email')
# ### end Alembic commands ###

29
migrations/versions/2020_031510_9081f1a90939_.py Normal file
View File

@ -0,0 +1,29 @@
"""empty message
Revision ID: 9081f1a90939
Revises: 11a35b448f83
Create Date: 2020-03-15 10:51:17.341046
"""
import sqlalchemy_utils
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '9081f1a90939'
down_revision = '11a35b448f83'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('refused_email', sa.Column('deleted', sa.Boolean(), server_default='0', nullable=False))
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column('refused_email', 'deleted')
# ### end Alembic commands ###

2
server.py
View File

@ -151,7 +151,7 @@ def fake_data():
db.session.commit()
api_key = ApiKey.create(user_id=user.id, name="Chrome")
api_key.code = "codeCH"
api_key.code = "code"
api_key = ApiKey.create(user_id=user.id, name="Firefox")
api_key.code = "codeFF"

12
templates/emails/transactional/automatic-disable-alias.html
View File

@ -2,15 +2,17 @@
{% block content %}
{{ render_text("Hi " + name) }}
{{ render_text("There are at least 2 emails sent to your alias <b>" + alias + "</b> from <b>" + website_email +
"</b> that have been <b>refused</b> (or bounced) by your email provider.") }}
{{ render_text("There are at least 2 emails sent to your alias <b>" + alias + "</b> from <b>" + website_email + "</b> that have been <b>refused</b> (or bounced) by your mailbox " + mailbox_email + ".") }}
{{ render_text("This is usually due to the email being considered as <b>spam</b> by your email provider.
The email is included at the end of this message so you can take a look at its content.") }}
{{ render_text('As security measure, we have <b>disabled</b> the alias ' + alias) }}
{{ render_text("This is usually due to the email being considered as <b>spam</b> by your email provider.") }}
{{ render_button("View the refused email", refused_email_url) }}
{{ render_text('As security measure, we have <b>disabled</b> the alias ' + alias + ".") }}
{{ render_text('Please let us know if you have any question.') }}
{{ render_text('Thanks, <br />SimpleLogin Team.') }}
{{ raw_url(refused_email_url) }}
{% endblock %}

6
templates/emails/transactional/automatic-disable-alias.txt
View File

@ -1,9 +1,11 @@
Hi {{name}}
There are at least 2 emails sent to your alias {{alias}} from {{website_from}} that have been refused (or bounced) by your email provider.
There are at least 2 emails sent to your alias {{alias}} from {{website_from}} that have been refused (or bounced) by your mailbox {{mailbox_email}}.
This is usually due to the email being considered as spam by your email provider.
The email is included at the end of this message so you can take a look at its content.
You can view this email here:
{{ refused_email_url }}
As security measure, we have disabled the alias {{alias}}.

6
templates/emails/transactional/bounced-email.html
View File

@ -2,9 +2,11 @@
{% block content %}
{{ render_text("Hi " + name) }}
{{ render_text("An email sent to your alias <b>" + alias + "</b> from <b>" + website_email + "</b> was <b>refused</b> (or <em>bounced</em>) by your email provider.") }}
{{ render_text("An email sent to your alias <b>" + alias + "</b> from <b>" + website_email + "</b> was <b>refused</b> (or bounced) by your mailbox " + mailbox_email + ".") }}
{{ render_text("This is usually due to the email being considered as <b>spam</b> by your email provider. The email is included at the end of this message so you can take a look at its content.") }}
{{ render_text('This is usually due to the email being considered as <b>spam</b> by your email provider.') }}
{{ render_button("View the refused email", refused_email_url) }}
{{ render_text('To avoid spams forwarded by SimpleLogin server, please consider the following options:') }}

5
templates/emails/transactional/bounced-email.txt
View File

@ -1,9 +1,10 @@
Hi {{name}}
An email sent to your alias {{alias}} from {{website_from}} was refused (or bounced) by your email provider.
An email sent to your alias {{alias}} from {{website_from}} was refused (or bounced) by your mailbox {{mailbox_email}}.
This is usually due to the email being considered as spam by your email provider.
The email is included at the end of this message so you can take a look at its content.
You can view this email here:
{{ refused_email_url }}
To avoid spams forwarded by SimpleLogin server, please consider the following options: