diff --git a/app/api/views/auth.py b/app/api/views/auth.py index 105e0bb6..a149f9b4 100644 --- a/app/api/views/auth.py +++ b/app/api/views/auth.py @@ -19,6 +19,7 @@ from app.email_utils import ( send_email, render, ) +from app.events.auth_event import LoginEvent, RegisterEvent from app.extensions import limiter from app.log import LOG from app.models import User, ApiKey, SocialAuth, AccountActivation @@ -55,16 +56,20 @@ def auth_login(): user = User.filter_by(email=email).first() if not user or not user.check_password(password): + LoginEvent(LoginEvent.ActionType.failed, LoginEvent.Source.api).send() return jsonify(error="Email or password incorrect"), 400 elif user.disabled: + LoginEvent(LoginEvent.ActionType.disabled_login, LoginEvent.Source.api).send() return jsonify(error="Account disabled"), 400 elif not user.activated: + LoginEvent(LoginEvent.ActionType.not_activated, LoginEvent.Source.api).send() return jsonify(error="Account not activated"), 422 elif user.fido_enabled(): # allow user who has TOTP enabled to continue using the mobile app if not user.enable_otp: return jsonify(error="Currently we don't support FIDO on mobile yet"), 403 + LoginEvent(LoginEvent.ActionType.success, LoginEvent.Source.api).send() return jsonify(**auth_payload(user, device)), 200 @@ -88,14 +93,20 @@ def auth_register(): password = data.get("password") if DISABLE_REGISTRATION: + RegisterEvent(RegisterEvent.ActionType.failed, RegisterEvent.Source.api).send() return jsonify(error="registration is closed"), 400 if not email_can_be_used_as_mailbox(email) or personal_email_already_used(email): + RegisterEvent( + RegisterEvent.ActionType.invalid_email, RegisterEvent.Source.api + ).send() return jsonify(error=f"cannot use {email} as personal inbox"), 400 if not password or len(password) < 8: + RegisterEvent(RegisterEvent.ActionType.failed, RegisterEvent.Source.api).send() return jsonify(error="password too short"), 400 if len(password) > 100: + RegisterEvent(RegisterEvent.ActionType.failed, RegisterEvent.Source.api).send() return jsonify(error="password too long"), 400 LOG.d("create user %s", email) @@ -114,6 +125,7 @@ def auth_register(): render("transactional/code-activation.html", code=code), ) + RegisterEvent(RegisterEvent.ActionType.success, RegisterEvent.Source.api).send() return jsonify(msg="User needs to confirm their account"), 200 diff --git a/app/auth/views/login.py b/app/auth/views/login.py index c68ff887..a08ca774 100644 --- a/app/auth/views/login.py +++ b/app/auth/views/login.py @@ -5,6 +5,7 @@ from wtforms import StringField, validators from app.auth.base import auth_bp from app.auth.views.login_utils import after_login +from app.events.auth_event import LoginEvent from app.extensions import limiter from app.log import LOG from app.models import User @@ -43,18 +44,22 @@ def login(): g.deduct_limit = True form.password.data = None flash("Email or password incorrect", "error") + LoginEvent(LoginEvent.ActionType.failed).send() elif user.disabled: flash( "Your account is disabled. Please contact SimpleLogin team to re-enable your account.", "error", ) + LoginEvent(LoginEvent.ActionType.disabled_login).send() elif not user.activated: show_resend_activation = True flash( "Please check your inbox for the activation email. You can also have this email re-sent", "error", ) + LoginEvent(LoginEvent.ActionType.not_activated).send() else: + LoginEvent(LoginEvent.ActionType.success).send() return after_login(user, next_url) return render_template( diff --git a/app/auth/views/register.py b/app/auth/views/register.py index abbc56f5..60e6b01d 100644 --- a/app/auth/views/register.py +++ b/app/auth/views/register.py @@ -13,6 +13,7 @@ from app.email_utils import ( email_can_be_used_as_mailbox, personal_email_already_used, ) +from app.events.auth_event import RegisterEvent from app.log import LOG from app.models import User, ActivationCode from app.utils import random_string, encode_url, sanitize_email @@ -60,6 +61,7 @@ def register(): hcaptcha_res, ) flash("Wrong Captcha", "error") + RegisterEvent(RegisterEvent.ActionType.catpcha_failed).send() return render_template( "auth/register.html", form=form, @@ -70,10 +72,11 @@ def register(): email = sanitize_email(form.email.data) if not email_can_be_used_as_mailbox(email): flash("You cannot use this email address as your personal inbox.", "error") - + RegisterEvent(RegisterEvent.ActionType.email_in_use).send() else: if personal_email_already_used(email): flash(f"Email {email} already used", "error") + RegisterEvent(RegisterEvent.ActionType.email_in_use).send() else: LOG.d("create user %s", email) user = User.create( @@ -86,8 +89,10 @@ def register(): try: send_activation_email(user, next_url) + RegisterEvent(RegisterEvent.ActionType.success).send() except Exception: flash("Invalid email, are you sure the email is correct?", "error") + RegisterEvent(RegisterEvent.ActionType.invalid_email).send() return redirect(url_for("auth.register")) return render_template("auth/register_waiting_activation.html") diff --git a/app/config.py b/app/config.py index 133ef37a..fb77039f 100644 --- a/app/config.py +++ b/app/config.py @@ -302,6 +302,9 @@ MAX_ALERT_24H = 4 # When a reverse-alias receives emails from un unknown mailbox ALERT_REVERSE_ALIAS_UNKNOWN_MAILBOX = "reverse_alias_unknown_mailbox" +# When somebody is trying to spoof a reply +ALERT_DMARC_FAILED_REPLY_PHASE = "dmarc_failed_reply_phase" + # When a forwarding email is bounced ALERT_BOUNCE_EMAIL = "bounce" diff --git a/app/email_utils.py b/app/email_utils.py index b631ce59..e321e96b 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -8,6 +8,9 @@ import random import time import uuid from copy import deepcopy + +from aiosmtpd.smtp import Envelope + from email import policy, message_from_bytes, message_from_string from email.header import decode_header, Header from email.message import Message, EmailMessage @@ -74,10 +77,7 @@ from app.models import ( TransactionalEmail, IgnoreBounceSender, InvalidMailboxDomain, - DmarcCheckResult, VerpType, - SpamdResult, - SPFCheckResult, ) from app.utils import ( random_string, @@ -972,7 +972,10 @@ def add_header(msg: Message, text_header, html_header) -> Message: elif content_type in ("multipart/alternative", "multipart/related"): new_parts = [] for part in msg.get_payload(): - new_parts.append(add_header(part, text_header, html_header)) + if isinstance(part, Message): + new_parts.append(add_header(part, text_header, html_header)) + else: + new_parts.append(part) clone_msg = copy(msg) clone_msg.set_payload(new_parts) return clone_msg @@ -1437,7 +1440,7 @@ def save_email_for_debugging(msg: Message, file_name_prefix=None) -> str: if TEMP_DIR: file_name = str(uuid.uuid4()) + ".eml" if file_name_prefix: - file_name = file_name_prefix + file_name + file_name = "{}-{}".format(file_name_prefix, file_name) with open(os.path.join(TEMP_DIR, file_name), "wb") as f: f.write(msg.as_bytes()) @@ -1448,30 +1451,22 @@ def save_email_for_debugging(msg: Message, file_name_prefix=None) -> str: return "" -def get_spamd_result(msg: Message) -> Optional[SpamdResult]: - spam_result_header = msg.get_all(headers.SPAMD_RESULT) - if not spam_result_header: - newrelic.agent.record_custom_event("SpamdCheck", {"header": "missing"}) - return None +def save_envelope_for_debugging(envelope: Envelope, file_name_prefix=None) -> str: + """Save envelope for debugging to temporary location + Return the file path + """ + if TEMP_DIR: + file_name = str(uuid.uuid4()) + ".eml" + if file_name_prefix: + file_name = "{}-{}".format(file_name_prefix, file_name) - spam_entries = [entry.strip() for entry in str(spam_result_header[-1]).split("\n")] - for entry_pos in range(len(spam_entries)): - sep = spam_entries[entry_pos].find("(") - if sep > -1: - spam_entries[entry_pos] = spam_entries[entry_pos][:sep] + with open(os.path.join(TEMP_DIR, file_name), "wb") as f: + f.write(envelope.original_content) - spamd_result = SpamdResult() - - for header_value, dmarc_result in DmarcCheckResult.get_string_dict().items(): - if header_value in spam_entries: - spamd_result.set_dmarc_result(dmarc_result) - for header_value, spf_result in SPFCheckResult.get_string_dict().items(): - if header_value in spam_entries: - spamd_result.set_spf_result(spf_result) - - newrelic.agent.record_custom_event("SpamdCheck", spamd_result.event_data()) - return spamd_result + LOG.d("envelope saved to %s", file_name) + return file_name + return "" def generate_verp_email( verp_type: VerpType, object_id: int, sender_domain: Optional[str] = None diff --git a/app/events/auth_event.py b/app/events/auth_event.py new file mode 100644 index 00000000..492c4e89 --- /dev/null +++ b/app/events/auth_event.py @@ -0,0 +1,46 @@ +import newrelic + +from app.models import EnumE + + +class LoginEvent: + class ActionType(EnumE): + success = 0 + failed = 1 + disabled_login = 2 + not_activated = 3 + + class Source(EnumE): + web = 0 + api = 1 + + def __init__(self, action: ActionType, source: Source = Source.web): + self.action = action + self.source = source + + def send(self): + newrelic.agent.record_custom_event( + "LoginEvent", {"action": self.action.name, "source": self.source.name} + ) + + +class RegisterEvent: + class ActionType(EnumE): + success = 0 + failed = 1 + catpcha_failed = 2 + email_in_use = 3 + invalid_email = 4 + + class Source(EnumE): + web = 0 + api = 1 + + def __init__(self, action: ActionType, source: Source = Source.web): + self.action = action + self.source = source + + def send(self): + newrelic.agent.record_custom_event( + "RegisterEvent", {"action": self.action.name, "source": self.source.name} + ) diff --git a/tests/email/__init__.py b/app/handler/__init__.py similarity index 100% rename from tests/email/__init__.py rename to app/handler/__init__.py diff --git a/app/handler/dmarc.py b/app/handler/dmarc.py new file mode 100644 index 00000000..62eed243 --- /dev/null +++ b/app/handler/dmarc.py @@ -0,0 +1,158 @@ +import uuid +from io import BytesIO +from typing import Optional, Tuple + +from aiosmtpd.handlers import Message +from aiosmtpd.smtp import Envelope + +from app import s3 +from app.config import ( + DMARC_CHECK_ENABLED, + ALERT_QUARANTINE_DMARC, + ALERT_DMARC_FAILED_REPLY_PHASE, +) +from app.email import headers, status +from app.email_utils import ( + get_header_unicode, + send_email_with_rate_control, + render, + add_or_replace_header, + to_bytes, + add_header, +) +from app.handler.spamd_result import SpamdResult, Phase, DmarcCheckResult +from app.log import LOG +from app.models import Alias, Contact, Notification, EmailLog, RefusedEmail + + +def apply_dmarc_policy_for_forward_phase( + alias: Alias, contact: Contact, envelope: Envelope, msg: Message +) -> Tuple[Message, Optional[str]]: + spam_result = SpamdResult.extract_from_headers(msg, Phase.forward) + if not DMARC_CHECK_ENABLED or not spam_result: + return msg, None + + from_header = get_header_unicode(msg[headers.FROM]) + + if spam_result.dmarc == DmarcCheckResult.soft_fail: + LOG.w( + f"dmarc forward: soft_fail from contact {contact.email} to alias {alias.email}." + f"mail_from:{envelope.mail_from}, from_header: {from_header}" + ) + changed_msg = add_header( + msg, + f"""This email failed anti-phishing checks when it was received by SimpleLogin, be careful with its content. +More info on https://simplelogin.io/docs/getting-started/anti-phishing/ + """, + f""" +
+ This email failed anti-phishing checks when it was received by SimpleLogin, be careful with its content. + More info on anti-phishing measure +
+ """, + ) + return changed_msg, None + + if spam_result.dmarc in ( + DmarcCheckResult.quarantine, + DmarcCheckResult.reject, + ): + LOG.w( + f"dmarc forward: put email from {contact} to {alias} to quarantine. {spam_result.event_data()}, " + f"mail_from:{envelope.mail_from}, from_header: {msg[headers.FROM]}" + ) + email_log = quarantine_dmarc_failed_forward_email(alias, contact, envelope, msg) + Notification.create( + user_id=alias.user_id, + title=f"{alias.email} has a new mail in quarantine", + message=Notification.render( + "notification/message-quarantine.html", alias=alias + ), + commit=True, + ) + user = alias.user + send_email_with_rate_control( + user, + ALERT_QUARANTINE_DMARC, + user.email, + f"An email sent to {alias.email} has been quarantined", + render( + "transactional/message-quarantine-dmarc.txt.jinja2", + from_header=from_header, + alias=alias, + refused_email_url=email_log.get_dashboard_url(), + ), + render( + "transactional/message-quarantine-dmarc.html", + from_header=from_header, + alias=alias, + refused_email_url=email_log.get_dashboard_url(), + ), + max_nb_alert=10, + ignore_smtp_error=True, + ) + return msg, status.E215 + + return msg, None + + +def quarantine_dmarc_failed_forward_email(alias, contact, envelope, msg) -> EmailLog: + add_or_replace_header(msg, headers.SL_DIRECTION, "Forward") + msg[headers.SL_ENVELOPE_FROM] = envelope.mail_from + random_name = str(uuid.uuid4()) + s3_report_path = f"refused-emails/full-{random_name}.eml" + s3.upload_email_from_bytesio( + s3_report_path, BytesIO(to_bytes(msg)), f"full-{random_name}" + ) + refused_email = RefusedEmail.create( + full_report_path=s3_report_path, user_id=alias.user_id, flush=True + ) + return EmailLog.create( + user_id=alias.user_id, + mailbox_id=alias.mailbox_id, + contact_id=contact.id, + alias_id=alias.id, + message_id=str(msg[headers.MESSAGE_ID]), + refused_email_id=refused_email.id, + is_spam=True, + blocked=True, + commit=True, + ) + + +def apply_dmarc_policy_for_reply_phase( + alias_from: Alias, contact_recipient: Contact, envelope: Envelope, msg: Message +) -> Optional[str]: + spam_result = SpamdResult.extract_from_headers(msg, Phase.reply) + if not DMARC_CHECK_ENABLED or not spam_result: + return None + + if spam_result.dmarc not in ( + DmarcCheckResult.quarantine, + DmarcCheckResult.reject, + DmarcCheckResult.soft_fail, + ): + return None + LOG.w( + f"dmarc reply: Put email from {alias_from.email} to {contact_recipient} into quarantine. {spam_result.event_data()}, " + f"mail_from:{envelope.mail_from}, from_header: {msg[headers.FROM]}" + ) + send_email_with_rate_control( + alias_from.user, + ALERT_DMARC_FAILED_REPLY_PHASE, + alias_from.user.email, + f"Attempt to send an email to your contact {contact_recipient.email} from {envelope.mail_from}", + render( + "transactional/spoof-reply.txt.jinja2", + contact=contact_recipient, + alias=alias_from, + sender=envelope.mail_from, + ), + render( + "transactional/spoof-reply.html", + contact=contact_recipient, + alias=alias_from, + sender=envelope.mail_from, + ), + ) + return status.E215 diff --git a/app/handler/spamd_result.py b/app/handler/spamd_result.py new file mode 100644 index 00000000..57cc250b --- /dev/null +++ b/app/handler/spamd_result.py @@ -0,0 +1,127 @@ +from __future__ import annotations +from typing import Dict, Optional + +import newrelic + +from app.email import headers +from app.models import EnumE +from email.message import Message + + +class Phase(EnumE): + unknown = 0 + forward = 1 + reply = 2 + + +class DmarcCheckResult(EnumE): + allow = 0 + soft_fail = 1 + quarantine = 2 + reject = 3 + not_available = 4 + bad_policy = 5 + + @staticmethod + def get_string_dict(): + return { + "DMARC_POLICY_ALLOW": DmarcCheckResult.allow, + "DMARC_POLICY_SOFTFAIL": DmarcCheckResult.soft_fail, + "DMARC_POLICY_QUARANTINE": DmarcCheckResult.quarantine, + "DMARC_POLICY_REJECT": DmarcCheckResult.reject, + "DMARC_NA": DmarcCheckResult.not_available, + "DMARC_BAD_POLICY": DmarcCheckResult.bad_policy, + } + + +class SPFCheckResult(EnumE): + allow = 0 + fail = 1 + soft_fail = 1 + neutral = 2 + temp_error = 3 + not_available = 4 + perm_error = 5 + + @staticmethod + def get_string_dict(): + return { + "R_SPF_ALLOW": SPFCheckResult.allow, + "R_SPF_FAIL": SPFCheckResult.fail, + "R_SPF_SOFTFAIL": SPFCheckResult.soft_fail, + "R_SPF_NEUTRAL": SPFCheckResult.neutral, + "R_SPF_DNSFAIL": SPFCheckResult.temp_error, + "R_SPF_NA": SPFCheckResult.not_available, + "R_SPF_PERMFAIL": SPFCheckResult.perm_error, + } + + +class SpamdResult: + def __init__(self, phase: Phase = Phase.unknown): + self.phase: Phase = phase + self.dmarc: DmarcCheckResult = DmarcCheckResult.not_available + self.spf: SPFCheckResult = SPFCheckResult.not_available + + def set_dmarc_result(self, dmarc_result: DmarcCheckResult): + self.dmarc = dmarc_result + + def set_spf_result(self, spf_result: SPFCheckResult): + self.spf = spf_result + + def event_data(self) -> Dict: + return { + "header": "present", + "dmarc": self.dmarc.name, + "spf": self.spf.name, + "phase": self.phase.name, + } + + @classmethod + def extract_from_headers( + cls, msg: Message, phase: Phase = Phase.unknown + ) -> Optional[SpamdResult]: + cached = cls._get_from_message(msg) + if cached: + return cached + + spam_result_header = msg.get_all(headers.SPAMD_RESULT) + if not spam_result_header: + return None + + spam_entries = [ + entry.strip() for entry in str(spam_result_header[-1]).split("\n") + ] + for entry_pos in range(len(spam_entries)): + sep = spam_entries[entry_pos].find("(") + if sep > -1: + spam_entries[entry_pos] = spam_entries[entry_pos][:sep] + + spamd_result = SpamdResult(phase) + + for header_value, dmarc_result in DmarcCheckResult.get_string_dict().items(): + if header_value in spam_entries: + spamd_result.set_dmarc_result(dmarc_result) + break + for header_value, spf_result in SPFCheckResult.get_string_dict().items(): + if header_value in spam_entries: + spamd_result.set_spf_result(spf_result) + break + + cls._store_in_message(spamd_result, msg) + return spamd_result + + @classmethod + def _store_in_message(cls, check: SpamdResult, msg: Message): + msg.spamd_check = check + + @classmethod + def _get_from_message(cls, msg: Message) -> Optional[SpamdResult]: + return getattr(msg, "spamd_check", None) + + @classmethod + def send_to_new_relic(cls, msg: Message): + check = cls._get_from_message(msg) + if check: + newrelic.agent.record_custom_event("SpamdCheck", check.event_data()) + else: + newrelic.agent.record_custom_event("SpamdCheck", {"header": "missing"}) diff --git a/app/models.py b/app/models.py index 12bf2863..ed30ca52 100644 --- a/app/models.py +++ b/app/models.py @@ -3,7 +3,7 @@ import os import random import uuid from email.utils import formataddr -from typing import List, Tuple, Optional, Dict +from typing import List, Tuple, Optional import arrow import sqlalchemy as sa @@ -237,63 +237,6 @@ class AuditLogActionEnum(EnumE): extend_subscription = 7 -class DmarcCheckResult(EnumE): - allow = 0 - soft_fail = 1 - quarantine = 2 - reject = 3 - not_available = 4 - bad_policy = 5 - - @staticmethod - def get_string_dict(): - return { - "DMARC_POLICY_ALLOW": DmarcCheckResult.allow, - "DMARC_POLICY_SOFTFAIL": DmarcCheckResult.soft_fail, - "DMARC_POLICY_QUARANTINE": DmarcCheckResult.quarantine, - "DMARC_POLICY_REJECT": DmarcCheckResult.reject, - "DMARC_NA": DmarcCheckResult.not_available, - "DMARC_BAD_POLICY": DmarcCheckResult.bad_policy, - } - - -class SPFCheckResult(EnumE): - allow = 0 - fail = 1 - soft_fail = 1 - neutral = 2 - temp_error = 3 - not_available = 4 - perm_error = 5 - - @staticmethod - def get_string_dict(): - return { - "R_SPF_ALLOW": SPFCheckResult.allow, - "R_SPF_FAIL": SPFCheckResult.fail, - "R_SPF_SOFTFAIL": SPFCheckResult.soft_fail, - "R_SPF_NEUTRAL": SPFCheckResult.neutral, - "R_SPF_DNSFAIL": SPFCheckResult.temp_error, - "R_SPF_NA": SPFCheckResult.not_available, - "R_SPF_PERMFAIL": SPFCheckResult.perm_error, - } - - -class SpamdResult: - def __init__(self): - self.dmarc: DmarcCheckResult = DmarcCheckResult.not_available - self.spf: SPFCheckResult = SPFCheckResult.not_available - - def set_dmarc_result(self, dmarc_result: DmarcCheckResult): - self.dmarc = dmarc_result - - def set_spf_result(self, spf_result: SPFCheckResult): - self.spf = spf_result - - def event_data(self) -> Dict: - return {"header": "present", "dmarc": self.dmarc, "spf": self.spf} - - class VerpType(EnumE): bounce_forward = 0 bounce_reply = 1 diff --git a/email_handler.py b/email_handler.py index 3a4fa3a5..a7f8d9de 100644 --- a/email_handler.py +++ b/email_handler.py @@ -86,10 +86,16 @@ from app.config import ( OLD_UNSUBSCRIBER, ALERT_FROM_ADDRESS_IS_REVERSE_ALIAS, ALERT_TO_NOREPLY, - DMARC_CHECK_ENABLED, - ALERT_QUARANTINE_DMARC, ) from app.db import Session +from app.handler.dmarc import ( + apply_dmarc_policy_for_reply_phase, + apply_dmarc_policy_for_forward_phase, +) +from app.handler.spamd_result import ( + SpamdResult, + SPFCheckResult, +) from app.email import status, headers from app.email.rate_limit import rate_limited from app.email.spam import get_spam_score @@ -129,9 +135,9 @@ from app.email_utils import ( get_orig_message_from_yahoo_complaint, get_mailbox_bounce_info, save_email_for_debugging, - get_spamd_result, - generate_verp_email, + save_envelope_for_debugging, get_verp_info_from_email, + generate_verp_email, ) from app.errors import ( NonReverseAliasInReplyPhase, @@ -156,8 +162,6 @@ from app.models import ( DeletedAlias, DomainDeletedAlias, Notification, - DmarcCheckResult, - SPFCheckResult, VerpType, ) from app.pgp_utils import PGPException, sign_data_with_pgpy, sign_data @@ -283,7 +287,7 @@ def get_or_create_reply_to_contact( return contact else: LOG.d( - "create contact %s for alias %s via reply-to header", + "create contact %s for alias %s via reply-to header %s", contact_address, alias, reply_to_header, @@ -543,99 +547,6 @@ def handle_email_sent_to_ourself(alias, from_addr: str, msg: Message, user): ) -def apply_dmarc_policy( - alias: Alias, contact: Contact, envelope: Envelope, msg: Message -) -> Optional[str]: - spam_result = get_spamd_result(msg) - if not DMARC_CHECK_ENABLED or not spam_result: - return None - - from_header = get_header_unicode(msg[headers.FROM]) - # todo: remove when soft_fail email is put into quarantine - if spam_result.dmarc == DmarcCheckResult.soft_fail: - LOG.w( - f"dmarc soft_fail from contact {contact.email} to alias {alias.email}." - f"mail_from:{envelope.mail_from}, from_header: {from_header}" - ) - return None - if spam_result.dmarc in ( - DmarcCheckResult.quarantine, - DmarcCheckResult.reject, - # todo: disable soft_fail for now - # DmarcCheckResult.soft_fail, - ): - LOG.w( - f"put email from {contact} to {alias} to quarantine. {spam_result.event_data()}, " - f"mail_from:{envelope.mail_from}, from_header: {msg[headers.FROM]}" - ) - email_log = quarantine_dmarc_failed_email(alias, contact, envelope, msg) - Notification.create( - user_id=alias.user_id, - title=f"{alias.email} has a new mail in quarantine", - message=Notification.render( - "notification/message-quarantine.html", alias=alias - ), - commit=True, - ) - user = alias.user - send_email_with_rate_control( - user, - ALERT_QUARANTINE_DMARC, - user.email, - f"An email sent to {alias.email} has been quarantined", - render( - "transactional/message-quarantine-dmarc.txt.jinja2", - from_header=from_header, - alias=alias, - refused_email_url=email_log.get_dashboard_url(), - ), - render( - "transactional/message-quarantine-dmarc.html", - from_header=from_header, - alias=alias, - refused_email_url=email_log.get_dashboard_url(), - ), - max_nb_alert=10, - ignore_smtp_error=True, - ) - return status.E215 - return None - - -def quarantine_dmarc_failed_email(alias, contact, envelope, msg) -> EmailLog: - add_or_replace_header(msg, headers.SL_DIRECTION, "Forward") - msg[headers.SL_ENVELOPE_TO] = alias.email - msg[headers.SL_ENVELOPE_FROM] = envelope.mail_from - add_or_replace_header(msg, "From", contact.new_addr()) - # replace CC & To emails by reverse-alias for all emails that are not alias - try: - replace_header_when_forward(msg, alias, "Cc") - replace_header_when_forward(msg, alias, "To") - except CannotCreateContactForReverseAlias: - Session.commit() - raise - - random_name = str(uuid.uuid4()) - s3_report_path = f"refused-emails/full-{random_name}.eml" - s3.upload_email_from_bytesio( - s3_report_path, BytesIO(to_bytes(msg)), f"full-{random_name}" - ) - refused_email = RefusedEmail.create( - full_report_path=s3_report_path, user_id=alias.user_id, flush=True - ) - return EmailLog.create( - user_id=alias.user_id, - mailbox_id=alias.mailbox_id, - contact_id=contact.id, - alias_id=alias.id, - message_id=str(msg[headers.MESSAGE_ID]), - refused_email_id=refused_email.id, - is_spam=True, - blocked=True, - commit=True, - ) - - def handle_forward(envelope, msg: Message, rcpt_to: str) -> List[Tuple[bool, str]]: """return an array of SMTP status (is_success, smtp_status) is_success indicates whether an email has been delivered and @@ -717,7 +628,9 @@ def handle_forward(envelope, msg: Message, rcpt_to: str) -> List[Tuple[bool, str return [(True, res_status)] # Check if we need to reject or quarantine based on dmarc - dmarc_delivery_status = apply_dmarc_policy(alias, contact, envelope, msg) + msg, dmarc_delivery_status = apply_dmarc_policy_for_forward_phase( + alias, contact, envelope, msg + ) if dmarc_delivery_status is not None: return [(False, dmarc_delivery_status)] @@ -1031,6 +944,7 @@ def handle_reply(envelope, msg: Message, rcpt_to: str) -> (bool, str): Return whether an email has been delivered and the smtp status ("250 Message accepted", "550 Non-existent email address", etc) """ + reply_email = rcpt_to # reply_email must end with EMAIL_DOMAIN @@ -1066,7 +980,14 @@ def handle_reply(envelope, msg: Message, rcpt_to: str) -> (bool, str): alias, contact, ) - return [(False, status.E504)] + return False, status.E504 + + # Check if we need to reject or quarantine based on dmarc + dmarc_delivery_status = apply_dmarc_policy_for_reply_phase( + alias, contact, envelope, msg + ) + if dmarc_delivery_status is not None: + return False, dmarc_delivery_status # Anti-spoofing mailbox = get_mailbox_from_mail_from(mail_from, alias) @@ -2069,7 +1990,7 @@ def handle_unsubscribe_user(user_id: int, mail_from: str) -> str: return status.E510 if mail_from != user.email: - LOG.e("Unauthorized mail_from %s %s", user, mail_from) + LOG.w("Unauthorized mail_from %s %s", user, mail_from) return status.E511 user.notification = False @@ -2212,6 +2133,11 @@ def handle(envelope: Envelope, msg: Message) -> str: envelope.mail_from = mail_from envelope.rcpt_tos = rcpt_tos + # some emails don't have this header, set the default value (7bit) in this case + if headers.CONTENT_TRANSFER_ENCODING not in msg: + LOG.i("Set CONTENT_TRANSFER_ENCODING") + msg[headers.CONTENT_TRANSFER_ENCODING] = "7bit" + postfix_queue_id = get_queue_id(msg) if postfix_queue_id: set_message_id(postfix_queue_id) @@ -2362,10 +2288,10 @@ def handle(envelope: Envelope, msg: Message) -> str: email_log = EmailLog.get(email_log_id) alias = Alias.get_by(email=rcpt_tos[0]) LOG.w( - "iCloud bounces %s %s msg=%s", + "iCloud bounces %s %s, saved to%s", email_log, alias, - msg.as_string(), + save_email_for_debugging(msg, file_name_prefix="icloud_bounce_"), ) return handle_bounce(envelope, email_log, msg) @@ -2554,7 +2480,7 @@ class MailHandler: msg[headers.TO], ) return status.E524 - except (VERPReply, VERPForward) as e: + except (VERPReply, VERPForward, VERPTransactional) as e: LOG.w( "email handling fail with error:%s " "mail_from:%s, rcpt_tos:%s, header_from:%s, header_to:%s", @@ -2574,8 +2500,8 @@ class MailHandler: envelope.rcpt_tos, msg[headers.FROM], msg[headers.TO], - save_email_for_debugging( - msg, file_name_prefix=e.__class__.__name__ + save_envelope_for_debugging( + envelope, file_name_prefix=e.__class__.__name__ ), # todo: remove ) return status.E404 @@ -2602,9 +2528,9 @@ class MailHandler: return_status = handle(envelope, msg) elapsed = time.time() - start # Only bounce messages if the return-path passes the spf check. Otherwise black-hole it. + spamd_result = SpamdResult.extract_from_headers(msg) if return_status[0] == "5": - spamd_result = get_spamd_result(msg) - if spamd_result and get_spamd_result(msg).spf in ( + if spamd_result and spamd_result.spf in ( SPFCheckResult.fail, SPFCheckResult.soft_fail, ): @@ -2620,6 +2546,8 @@ class MailHandler: elapsed, return_status, ) + + SpamdResult.send_to_new_relic(msg) newrelic.agent.record_custom_metric("Custom/email_handler_time", elapsed) newrelic.agent.record_custom_metric("Custom/number_incoming_email", 1) return return_status diff --git a/templates/dashboard/index.html b/templates/dashboard/index.html index 225c3cf0..cb1933f3 100644 --- a/templates/dashboard/index.html +++ b/templates/dashboard/index.html @@ -579,13 +579,13 @@ {% endblock %} diff --git a/templates/dashboard/setting.html b/templates/dashboard/setting.html index da0b1097..16ac1050 100644 --- a/templates/dashboard/setting.html +++ b/templates/dashboard/setting.html @@ -494,7 +494,8 @@