From 08db23658a8dbc204c984469ca89df59351c2510 Mon Sep 17 00:00:00 2001 From: Son NK <> Date: Sat, 15 Aug 2020 16:33:48 +0200 Subject: [PATCH 1/6] add SPAMASSASSIN_HOST param --- app/config.py | 4 ++++ example.env | 5 ++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/app/config.py b/app/config.py index 0e5dbf53..8f8d2024 100644 --- a/app/config.py +++ b/app/config.py @@ -302,3 +302,7 @@ PLAUSIBLE_DOMAIN = os.environ.get("PLAUSIBLE_DOMAIN") # server host HOST = socket.gethostname() + +# by default use a tolerant score +MAX_SPAM_SCORE = 10 +SPAMASSASSIN_HOST = os.environ.get("SPAMASSASSIN_HOST") diff --git a/example.env b/example.env index ca552674..baed77d7 100644 --- a/example.env +++ b/example.env @@ -153,4 +153,7 @@ DISABLE_ONBOARDING=true # Set the 2 below variables to enable Plausible Analytics # PLAUSIBLE_HOST=https://plausible.io -# PLAUSIBLE_DOMAIN=yourdomain.com \ No newline at end of file +# PLAUSIBLE_DOMAIN=yourdomain.com + +# Spamassassin server +# SPAMASSASSIN_HOST = 127.0.0.1 \ No newline at end of file From c4dd980cf6b35caa066b4e84140d3ce0b5fa72c5 Mon Sep 17 00:00:00 2001 From: Son NK <> Date: Sat, 15 Aug 2020 16:35:18 +0200 Subject: [PATCH 2/6] add aiospamc to requirement --- requirements.in | 3 ++- requirements.txt | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/requirements.in b/requirements.in index 11fb6e49..6dc20d42 100644 --- a/requirements.in +++ b/requirements.in @@ -43,4 +43,5 @@ pyspf Flask-Limiter memory_profiler gevent -aiocontextvars \ No newline at end of file +aiocontextvars +aiospamc \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 26806347..b1e2a90d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,7 @@ aiocontextvars==0.2.2 # via -r requirements.in aiohttp==3.5.4 # via raven-aiohttp, yacron aiosmtpd==1.2 # via -r requirements.in aiosmtplib==1.0.6 # via yacron +aiospamc==0.6.1 # via -r requirements.in alembic==1.0.10 # via flask-migrate appnope==0.1.0 # via ipython arrow==0.14.2 # via -r requirements.in @@ -23,7 +24,7 @@ boto3==1.9.167 # via -r requirements.in, watchtower botocore==1.12.167 # via boto3, s3transfer cachetools==4.0.0 # via google-auth cbor2==5.1.0 # via webauthn -certifi==2019.3.9 # via requests, sentry-sdk +certifi==2019.11.28 # via aiospamc, requests, sentry-sdk cffi==1.12.3 # via bcrypt, cryptography chardet==3.0.4 # via aiohttp, requests click==7.0 # via flask, pip-tools From 673b08712caad5626612b0dfb5f4043bf1a8baf5 Mon Sep 17 00:00:00 2001 From: Son NK <> Date: Sat, 15 Aug 2020 16:38:16 +0200 Subject: [PATCH 3/6] use SPAMASSASSIN server if available --- email_handler.py | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/email_handler.py b/email_handler.py index 9bd4a90f..7322aced 100644 --- a/email_handler.py +++ b/email_handler.py @@ -43,6 +43,7 @@ from io import BytesIO from smtplib import SMTP from typing import List, Tuple +import aiospamc import arrow import spf from aiosmtpd.controller import Controller @@ -66,6 +67,8 @@ from app.config import ( POSTFIX_PORT, SENDER, SENDER_DIR, + SPAMASSASSIN_HOST, + MAX_SPAM_SCORE, ) from app.email_utils import ( send_email, @@ -359,7 +362,7 @@ def prepare_pgp_message(orig_msg: Message, pgp_fingerprint: str): return msg -def handle_forward( +async def handle_forward( envelope, smtp: SMTP, msg: Message, rcpt_to: str ) -> List[Tuple[bool, str]]: """return whether an email has been delivered and @@ -391,7 +394,7 @@ def handle_forward( ret = [] for mailbox in alias.mailboxes: ret.append( - forward_email_to_mailbox( + await forward_email_to_mailbox( alias, copy(msg), email_log, contact, envelope, smtp, mailbox, user ) ) @@ -399,7 +402,7 @@ def handle_forward( return ret -def forward_email_to_mailbox( +async def forward_email_to_mailbox( alias, msg: Message, email_log: EmailLog, @@ -421,7 +424,19 @@ def forward_email_to_mailbox( ) return False, "550 SL E14" - is_spam, spam_status = get_spam_info(msg, max_score=user.max_spam_score) + spam_status = "" + is_spam = False + + if SPAMASSASSIN_HOST: + spam_score = await get_spam_score(msg) + if (user.max_spam_score and spam_score > user.max_spam_score) or ( + not user.max_spam_score and spam_score > MAX_SPAM_SCORE + ): + is_spam = True + spam_status = "Spam detected by SpamAssassin server" + else: + is_spam, spam_status = get_spam_info(msg, max_score=user.max_spam_score) + if is_spam: LOG.warning("Email detected as spam. Alias: %s, from: %s", alias, contact) email_log.is_spam = True @@ -1065,7 +1080,7 @@ def handle_sender_email(envelope: Envelope): return "250 email to sender accepted" -def handle(envelope: Envelope, smtp: SMTP) -> str: +async def handle(envelope: Envelope, smtp: SMTP) -> str: """Return SMTP status""" # unsubscribe request if UNSUBSCRIBER and envelope.rcpt_tos == [UNSUBSCRIBER]: @@ -1106,7 +1121,7 @@ def handle(envelope: Envelope, smtp: SMTP) -> str: msg["From"], rcpt_to, ) - for is_delivered, smtp_status in handle_forward( + for is_delivered, smtp_status in await handle_forward( envelope, smtp, msg, rcpt_to ): res.append((is_delivered, smtp_status)) @@ -1120,6 +1135,11 @@ def handle(envelope: Envelope, smtp: SMTP) -> str: return res[0][1] +async def get_spam_score(message) -> float: + response = await aiospamc.check(message, host=SPAMASSASSIN_HOST) + return response.headers["Spam"].score + + class MailHandler: async def handle_DATA(self, server, session, envelope: Envelope): start = time.time() @@ -1137,7 +1157,7 @@ class MailHandler: app = new_app() with app.app_context(): - ret = handle(envelope, smtp) + ret = await handle(envelope, smtp) LOG.debug("takes %s seconds <<===", time.time() - start) return ret From bf555ed6059a535ad16cb850b4038ad92e34ed10 Mon Sep 17 00:00:00 2001 From: Son NK <> Date: Sat, 15 Aug 2020 16:53:57 +0200 Subject: [PATCH 4/6] detect spam in reply phase --- app/config.py | 2 + email_handler.py | 122 +++++++++++++----- .../transactional/spam-email-reply-phase.html | 17 +++ .../transactional/spam-email-reply-phase.txt | 15 +++ 4 files changed, 125 insertions(+), 31 deletions(-) create mode 100644 templates/emails/transactional/spam-email-reply-phase.html create mode 100644 templates/emails/transactional/spam-email-reply-phase.txt diff --git a/app/config.py b/app/config.py index 8f8d2024..f4e2afe3 100644 --- a/app/config.py +++ b/app/config.py @@ -306,3 +306,5 @@ HOST = socket.gethostname() # by default use a tolerant score MAX_SPAM_SCORE = 10 SPAMASSASSIN_HOST = os.environ.get("SPAMASSASSIN_HOST") +# use a more restrictive score when replying +MAX_REPLY_PHASE_SPAM_SCORE = 5 diff --git a/email_handler.py b/email_handler.py index 7322aced..9b784d99 100644 --- a/email_handler.py +++ b/email_handler.py @@ -69,6 +69,7 @@ from app.config import ( SENDER_DIR, SPAMASSASSIN_HOST, MAX_SPAM_SCORE, + MAX_REPLY_PHASE_SPAM_SCORE, ) from app.email_utils import ( send_email, @@ -424,6 +425,7 @@ async def forward_email_to_mailbox( ) return False, "550 SL E14" + # Spam check spam_status = "" is_spam = False @@ -524,7 +526,7 @@ async def forward_email_to_mailbox( return True, "250 Message accepted for delivery" -def handle_reply(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> (bool, str): +async def handle_reply(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> (bool, str): """ return whether an email has been delivered and the smtp status ("250 Message accepted", "550 Non-existent email address", etc) @@ -577,6 +579,33 @@ def handle_reply(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> (bool, str # cannot use 4** here as sender will retry. 5** because that generates bounce report return True, "250 SL E11" + # Spam check + spam_status = "" + is_spam = False + + # do not use user.max_spam_score here + if SPAMASSASSIN_HOST: + spam_score = await get_spam_score(msg) + if spam_score > MAX_REPLY_PHASE_SPAM_SCORE: + is_spam = True + spam_status = "Spam detected by SpamAssassin server" + else: + is_spam, spam_status = get_spam_info(msg, max_score=MAX_REPLY_PHASE_SPAM_SCORE) + + if is_spam: + LOG.exception( + "Reply phase - email sent from %s to %s detected as spam", alias, contact + ) + email_log = EmailLog.create( + contact_id=contact.id, is_reply=True, user_id=contact.user_id + ) + email_log.is_spam = True + email_log.spam_status = spam_status + db.session.commit() + + handle_spam(contact, alias, msg, user, mailbox.email, email_log, is_reply=True) + return False, "550 SL E15 Email detected as spam" + delete_header(msg, _IP_HEADER) delete_header(msg, "DKIM-Signature") @@ -941,6 +970,7 @@ def handle_spam( user: User, mailbox_email: str, email_log: EmailLog, + is_reply=False, # whether the email is in forward or reply phase ): # Store the report & original email orig_msg = get_orig_message_from_spamassassin_report(msg) @@ -972,35 +1002,65 @@ def handle_spam( ) disable_alias_link = f"{URL}/dashboard/unsubscribe/{alias.id}" - # inform user - LOG.d( - "Inform user %s about spam email sent by %s to alias %s", - user, - contact.website_email, - alias.email, - ) - send_email_with_rate_control( - user, - ALERT_SPAM_EMAIL, - mailbox_email, - f"Email from {contact.website_email} to {alias.email} is detected as spam", - render( - "transactional/spam-email.txt", - name=user.name, - alias=alias, - website_email=contact.website_email, - disable_alias_link=disable_alias_link, - refused_email_url=refused_email_url, - ), - render( - "transactional/spam-email.html", - name=user.name, - alias=alias, - website_email=contact.website_email, - disable_alias_link=disable_alias_link, - refused_email_url=refused_email_url, - ), - ) + if is_reply: + LOG.d( + "Inform user %s about spam email sent from alias %s to %s", + user, + alias, + contact, + ) + send_email_with_rate_control( + user, + ALERT_SPAM_EMAIL, + mailbox_email, + f"Email from {contact.website_email} to {alias.email} is detected as spam", + render( + "transactional/spam-email-reply-phase.txt", + name=user.name, + alias=alias, + website_email=contact.website_email, + disable_alias_link=disable_alias_link, + refused_email_url=refused_email_url, + ), + render( + "transactional/spam-email-reply-phase.html", + name=user.name, + alias=alias, + website_email=contact.website_email, + disable_alias_link=disable_alias_link, + refused_email_url=refused_email_url, + ), + ) + else: + # inform user + LOG.d( + "Inform user %s about spam email sent by %s to alias %s", + user, + contact, + alias, + ) + send_email_with_rate_control( + user, + ALERT_SPAM_EMAIL, + mailbox_email, + f"Email from {contact.website_email} to {alias.email} is detected as spam", + render( + "transactional/spam-email.txt", + name=user.name, + alias=alias, + website_email=contact.website_email, + disable_alias_link=disable_alias_link, + refused_email_url=refused_email_url, + ), + render( + "transactional/spam-email.html", + name=user.name, + alias=alias, + website_email=contact.website_email, + disable_alias_link=disable_alias_link, + refused_email_url=refused_email_url, + ), + ) def handle_unsubscribe(envelope: Envelope): @@ -1112,7 +1172,7 @@ async def handle(envelope: Envelope, smtp: SMTP) -> str: LOG.debug( ">>> Reply phase %s(%s) -> %s", envelope.mail_from, msg["From"], rcpt_to ) - is_delivered, smtp_status = handle_reply(envelope, smtp, msg, rcpt_to) + is_delivered, smtp_status = await handle_reply(envelope, smtp, msg, rcpt_to) res.append((is_delivered, smtp_status)) else: # Forward case LOG.debug( diff --git a/templates/emails/transactional/spam-email-reply-phase.html b/templates/emails/transactional/spam-email-reply-phase.html new file mode 100644 index 00000000..3ec3c76c --- /dev/null +++ b/templates/emails/transactional/spam-email-reply-phase.html @@ -0,0 +1,17 @@ +{% extends "base.html" %} + +{% block content %} + {{ render_text("Hi " + name) }} + {{ render_text("An email sent from your alias " + alias.email + " to " + website_email + " is detected as spam by our Spam Detection Engine (SpamAssassin).") }} + + {{ render_text('In most of the cases, the email will be refused by your contact.') }} + + {{ render_button("View the email", refused_email_url) }} + + {{ render_text('The email is automatically deleted in 7 days.') }} + + {{ render_text('Please let us know if you have any question by replying to this email.') }} + + {{ render_text('Thanks,
SimpleLogin Team.') }} + {{ raw_url(disable_alias_link) }} +{% endblock %} diff --git a/templates/emails/transactional/spam-email-reply-phase.txt b/templates/emails/transactional/spam-email-reply-phase.txt new file mode 100644 index 00000000..792db3f9 --- /dev/null +++ b/templates/emails/transactional/spam-email-reply-phase.txt @@ -0,0 +1,15 @@ +Hi {{name}} + +An email sent from your alias {{alias.email}} to {{website_email}} is detected as spam by our Spam Detection Engine (SpamAssassin). + +In most of the cases, the email will be refused by your contact. + +You can view this email here: +{{ refused_email_url }} + +The email is automatically deleted in 7 days. + +Please let us know if you have any question by replying to this email. + +Best, +SimpleLogin team. \ No newline at end of file From f9300009e5acd537bf708a4dc379c43470e33eed Mon Sep 17 00:00:00 2001 From: Son NK <> Date: Sat, 15 Aug 2020 16:56:16 +0200 Subject: [PATCH 5/6] refactor: rename forward -> contact --- app/dashboard/templates/dashboard/refused_email.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/dashboard/templates/dashboard/refused_email.html b/app/dashboard/templates/dashboard/refused_email.html index 4d273b99..a32ecda4 100644 --- a/app/dashboard/templates/dashboard/refused_email.html +++ b/app/dashboard/templates/dashboard/refused_email.html @@ -30,15 +30,15 @@ {% for email_log in email_logs %} {% set refused_email = email_log.refused_email %} - {% set forward = email_log.forward %} - {% set alias = forward.alias %} + {% set contact = email_log.contact %} + {% set alias = contact.alias %}
Sent {{ refused_email.created_at | dt }}
- From: {{ forward.website_email }}
+ From: {{ contact.website_email }}
To: {{ alias.email }} From 359eec23c072301df5bfc6f43b2246f93d44080a Mon Sep 17 00:00:00 2001 From: Son NK <> Date: Sat, 15 Aug 2020 16:58:11 +0200 Subject: [PATCH 6/6] take into account spam email during reply phase on refused email page --- app/dashboard/templates/dashboard/refused_email.html | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/app/dashboard/templates/dashboard/refused_email.html b/app/dashboard/templates/dashboard/refused_email.html index a32ecda4..f7a28e3a 100644 --- a/app/dashboard/templates/dashboard/refused_email.html +++ b/app/dashboard/templates/dashboard/refused_email.html @@ -38,13 +38,18 @@ Sent {{ refused_email.created_at | dt }}
- From: {{ contact.website_email }}
+ {% if email_log.is_reply %} + From: {{ alias.email }}
+ To: {{ contact.website_email }} + {% else %} + From: {{ contact.website_email }}
- + To: {{ alias.email }} Disable Alias + {% endif %} {% if refused_email.deleted %}
@@ -53,7 +58,7 @@ {% else %} Download → -
This will download a ".eml" file that you can open in your favorite email client
+
This will download a ".eml" file that you can open in your email client
{% endif %}