Mirror
/
app-MAIL-temp
mirror of https://github.com/simple-login/app.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into remove-softfail 

* origin/master: (34 commits)
  fix flake8
  add link to the anti phishing page
  improve email wording
  Move tests
  Only send enum names
  Only send enum name for events intead of the full class.enum
  Also track login and register events from the api routes
  typo
  revert changes
  Added fix for parts that are not messages
  Add missing formatting place
  Revert unwanted changes
  Do not show an error if we receive an unsubscribe from a different address
  Revert changes to pgp_utils
  fix import
  Send newrelic events on login and register
  PR changes
  format
  Move dmarc management to its own file
  ignore VERPTransactional
  ...
This commit is contained in:
Adrià Casajús 2022-04-14 18:25:03 +02:00
parent 26889283d3 a957cbb3c0
commit 4bcc728222
No known key found for this signature in database
GPG Key ID: F0033226A5AFC9B9
34 changed files with 638 additions and 278 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
app/api/views/auth.py
View File

@ -19,6 +19,7 @@ from app.email_utils import (
send_email,
render,
)
from app.events.auth_event import LoginEvent, RegisterEvent
from app.extensions import limiter
from app.log import LOG
from app.models import User, ApiKey, SocialAuth, AccountActivation
@ -55,16 +56,20 @@ def auth_login():
user = User.filter_by(email=email).first()
if not user or not user.check_password(password):
LoginEvent(LoginEvent.ActionType.failed, LoginEvent.Source.api).send()
return jsonify(error="Email or password incorrect"), 400
elif user.disabled:
LoginEvent(LoginEvent.ActionType.disabled_login, LoginEvent.Source.api).send()
return jsonify(error="Account disabled"), 400
elif not user.activated:
LoginEvent(LoginEvent.ActionType.not_activated, LoginEvent.Source.api).send()
return jsonify(error="Account not activated"), 422
elif user.fido_enabled():
# allow user who has TOTP enabled to continue using the mobile app
if not user.enable_otp:
return jsonify(error="Currently we don't support FIDO on mobile yet"), 403
LoginEvent(LoginEvent.ActionType.success, LoginEvent.Source.api).send()
return jsonify(**auth_payload(user, device)), 200
@ -88,14 +93,20 @@ def auth_register():
password = data.get("password")
if DISABLE_REGISTRATION:
RegisterEvent(RegisterEvent.ActionType.failed, RegisterEvent.Source.api).send()
return jsonify(error="registration is closed"), 400
if not email_can_be_used_as_mailbox(email) or personal_email_already_used(email):
RegisterEvent(
RegisterEvent.ActionType.invalid_email, RegisterEvent.Source.api
).send()
return jsonify(error=f"cannot use {email} as personal inbox"), 400
if not password or len(password) < 8:
RegisterEvent(RegisterEvent.ActionType.failed, RegisterEvent.Source.api).send()
return jsonify(error="password too short"), 400
if len(password) > 100:
RegisterEvent(RegisterEvent.ActionType.failed, RegisterEvent.Source.api).send()
return jsonify(error="password too long"), 400
LOG.d("create user %s", email)
@ -114,6 +125,7 @@ def auth_register():
render("transactional/code-activation.html", code=code),
)
RegisterEvent(RegisterEvent.ActionType.success, RegisterEvent.Source.api).send()
return jsonify(msg="User needs to confirm their account"), 200

5
app/auth/views/login.py
View File

@ -5,6 +5,7 @@ from wtforms import StringField, validators
from app.auth.base import auth_bp
from app.auth.views.login_utils import after_login
from app.events.auth_event import LoginEvent
from app.extensions import limiter
from app.log import LOG
from app.models import User
@ -43,18 +44,22 @@ def login():
g.deduct_limit = True
form.password.data = None
flash("Email or password incorrect", "error")
LoginEvent(LoginEvent.ActionType.failed).send()
elif user.disabled:
flash(
"Your account is disabled. Please contact SimpleLogin team to re-enable your account.",
"error",
)
LoginEvent(LoginEvent.ActionType.disabled_login).send()
elif not user.activated:
show_resend_activation = True
flash(
"Please check your inbox for the activation email. You can also have this email re-sent",
"error",
)
LoginEvent(LoginEvent.ActionType.not_activated).send()
else:
LoginEvent(LoginEvent.ActionType.success).send()
return after_login(user, next_url)
return render_template(

7
app/auth/views/register.py
View File

@ -13,6 +13,7 @@ from app.email_utils import (
email_can_be_used_as_mailbox,
personal_email_already_used,
)
from app.events.auth_event import RegisterEvent
from app.log import LOG
from app.models import User, ActivationCode
from app.utils import random_string, encode_url, sanitize_email
@ -60,6 +61,7 @@ def register():
hcaptcha_res,
)
flash("Wrong Captcha", "error")
RegisterEvent(RegisterEvent.ActionType.catpcha_failed).send()
return render_template(
"auth/register.html",
form=form,
@ -70,10 +72,11 @@ def register():
email = sanitize_email(form.email.data)
if not email_can_be_used_as_mailbox(email):
flash("You cannot use this email address as your personal inbox.", "error")
RegisterEvent(RegisterEvent.ActionType.email_in_use).send()
else:
if personal_email_already_used(email):
flash(f"Email {email} already used", "error")
RegisterEvent(RegisterEvent.ActionType.email_in_use).send()
else:
LOG.d("create user %s", email)
user = User.create(
@ -86,8 +89,10 @@ def register():
try:
send_activation_email(user, next_url)
RegisterEvent(RegisterEvent.ActionType.success).send()
except Exception:
flash("Invalid email, are you sure the email is correct?", "error")
RegisterEvent(RegisterEvent.ActionType.invalid_email).send()
return redirect(url_for("auth.register"))
return render_template("auth/register_waiting_activation.html")

3
app/config.py
View File

@ -302,6 +302,9 @@ MAX_ALERT_24H = 4
# When a reverse-alias receives emails from un unknown mailbox
ALERT_REVERSE_ALIAS_UNKNOWN_MAILBOX = "reverse_alias_unknown_mailbox"
# When somebody is trying to spoof a reply
ALERT_DMARC_FAILED_REPLY_PHASE = "dmarc_failed_reply_phase"
# When a forwarding email is bounced
ALERT_BOUNCE_EMAIL = "bounce"

47
app/email_utils.py
View File

@ -8,6 +8,9 @@ import random
import time
import uuid
from copy import deepcopy
from aiosmtpd.smtp import Envelope
from email import policy, message_from_bytes, message_from_string
from email.header import decode_header, Header
from email.message import Message, EmailMessage
@ -74,10 +77,7 @@ from app.models import (
TransactionalEmail,
IgnoreBounceSender,
InvalidMailboxDomain,
DmarcCheckResult,
VerpType,
SpamdResult,
SPFCheckResult,
)
from app.utils import (
random_string,
@ -972,7 +972,10 @@ def add_header(msg: Message, text_header, html_header) -> Message:
elif content_type in ("multipart/alternative", "multipart/related"):
new_parts = []
for part in msg.get_payload():
new_parts.append(add_header(part, text_header, html_header))
if isinstance(part, Message):
new_parts.append(add_header(part, text_header, html_header))
else:
new_parts.append(part)
clone_msg = copy(msg)
clone_msg.set_payload(new_parts)
return clone_msg
@ -1437,7 +1440,7 @@ def save_email_for_debugging(msg: Message, file_name_prefix=None) -> str:
if TEMP_DIR:
file_name = str(uuid.uuid4()) + ".eml"
if file_name_prefix:
file_name = file_name_prefix + file_name
file_name = "{}-{}".format(file_name_prefix, file_name)
with open(os.path.join(TEMP_DIR, file_name), "wb") as f:
f.write(msg.as_bytes())
@ -1448,30 +1451,22 @@ def save_email_for_debugging(msg: Message, file_name_prefix=None) -> str:
return ""
def get_spamd_result(msg: Message) -> Optional[SpamdResult]:
spam_result_header = msg.get_all(headers.SPAMD_RESULT)
if not spam_result_header:
newrelic.agent.record_custom_event("SpamdCheck", {"header": "missing"})
return None
def save_envelope_for_debugging(envelope: Envelope, file_name_prefix=None) -> str:
"""Save envelope for debugging to temporary location
Return the file path
"""
if TEMP_DIR:
file_name = str(uuid.uuid4()) + ".eml"
if file_name_prefix:
file_name = "{}-{}".format(file_name_prefix, file_name)
spam_entries = [entry.strip() for entry in str(spam_result_header[-1]).split("\n")]
for entry_pos in range(len(spam_entries)):
sep = spam_entries[entry_pos].find("(")
if sep > -1:
spam_entries[entry_pos] = spam_entries[entry_pos][:sep]
with open(os.path.join(TEMP_DIR, file_name), "wb") as f:
f.write(envelope.original_content)
spamd_result = SpamdResult()
for header_value, dmarc_result in DmarcCheckResult.get_string_dict().items():
if header_value in spam_entries:
spamd_result.set_dmarc_result(dmarc_result)
for header_value, spf_result in SPFCheckResult.get_string_dict().items():
if header_value in spam_entries:
spamd_result.set_spf_result(spf_result)
newrelic.agent.record_custom_event("SpamdCheck", spamd_result.event_data())
return spamd_result
LOG.d("envelope saved to %s", file_name)
return file_name
return ""
def generate_verp_email(
verp_type: VerpType, object_id: int, sender_domain: Optional[str] = None

46
app/events/auth_event.py Normal file
View File

@ -0,0 +1,46 @@
import newrelic
from app.models import EnumE
class LoginEvent:
class ActionType(EnumE):
success = 0
failed = 1
disabled_login = 2
not_activated = 3
class Source(EnumE):
web = 0
api = 1
def __init__(self, action: ActionType, source: Source = Source.web):
self.action = action
self.source = source
def send(self):
newrelic.agent.record_custom_event(
"LoginEvent", {"action": self.action.name, "source": self.source.name}
)
class RegisterEvent:
class ActionType(EnumE):
success = 0
failed = 1
catpcha_failed = 2
email_in_use = 3
invalid_email = 4
class Source(EnumE):
web = 0
api = 1
def __init__(self, action: ActionType, source: Source = Source.web):
self.action = action
self.source = source
def send(self):
newrelic.agent.record_custom_event(
"RegisterEvent", {"action": self.action.name, "source": self.source.name}
)

0
tests/email/__init__.py → app/handler/__init__.py
View File

158
app/handler/dmarc.py Normal file
View File

@ -0,0 +1,158 @@
import uuid
from io import BytesIO
from typing import Optional, Tuple
from aiosmtpd.handlers import Message
from aiosmtpd.smtp import Envelope
from app import s3
from app.config import (
DMARC_CHECK_ENABLED,
ALERT_QUARANTINE_DMARC,
ALERT_DMARC_FAILED_REPLY_PHASE,
)
from app.email import headers, status
from app.email_utils import (
get_header_unicode,
send_email_with_rate_control,
render,
add_or_replace_header,
to_bytes,
add_header,
)
from app.handler.spamd_result import SpamdResult, Phase, DmarcCheckResult
from app.log import LOG
from app.models import Alias, Contact, Notification, EmailLog, RefusedEmail
def apply_dmarc_policy_for_forward_phase(
alias: Alias, contact: Contact, envelope: Envelope, msg: Message
) -> Tuple[Message, Optional[str]]:
spam_result = SpamdResult.extract_from_headers(msg, Phase.forward)
if not DMARC_CHECK_ENABLED or not spam_result:
return msg, None
from_header = get_header_unicode(msg[headers.FROM])
if spam_result.dmarc == DmarcCheckResult.soft_fail:
LOG.w(
f"dmarc forward: soft_fail from contact {contact.email} to alias {alias.email}."
f"mail_from:{envelope.mail_from}, from_header: {from_header}"
)
changed_msg = add_header(
msg,
f"""This email failed anti-phishing checks when it was received by SimpleLogin, be careful with its content.
More info on https://simplelogin.io/docs/getting-started/anti-phishing/
""",
f"""
<p style="color:red">
This email failed anti-phishing checks when it was received by SimpleLogin, be careful with its content.
More info on <a href="https://simplelogin.io/docs/getting-started/anti-phishing/">anti-phishing measure</a>
</p>
""",
)
return changed_msg, None
if spam_result.dmarc in (
DmarcCheckResult.quarantine,
DmarcCheckResult.reject,
):
LOG.w(
f"dmarc forward: put email from {contact} to {alias} to quarantine. {spam_result.event_data()}, "
f"mail_from:{envelope.mail_from}, from_header: {msg[headers.FROM]}"
)
email_log = quarantine_dmarc_failed_forward_email(alias, contact, envelope, msg)
Notification.create(
user_id=alias.user_id,
title=f"{alias.email} has a new mail in quarantine",
message=Notification.render(
"notification/message-quarantine.html", alias=alias
),
commit=True,
)
user = alias.user
send_email_with_rate_control(
user,
ALERT_QUARANTINE_DMARC,
user.email,
f"An email sent to {alias.email} has been quarantined",
render(
"transactional/message-quarantine-dmarc.txt.jinja2",
from_header=from_header,
alias=alias,
refused_email_url=email_log.get_dashboard_url(),
),
render(
"transactional/message-quarantine-dmarc.html",
from_header=from_header,
alias=alias,
refused_email_url=email_log.get_dashboard_url(),
),
max_nb_alert=10,
ignore_smtp_error=True,
)
return msg, status.E215
return msg, None
def quarantine_dmarc_failed_forward_email(alias, contact, envelope, msg) -> EmailLog:
add_or_replace_header(msg, headers.SL_DIRECTION, "Forward")
msg[headers.SL_ENVELOPE_FROM] = envelope.mail_from
random_name = str(uuid.uuid4())
s3_report_path = f"refused-emails/full-{random_name}.eml"
s3.upload_email_from_bytesio(
s3_report_path, BytesIO(to_bytes(msg)), f"full-{random_name}"
)
refused_email = RefusedEmail.create(
full_report_path=s3_report_path, user_id=alias.user_id, flush=True
)
return EmailLog.create(
user_id=alias.user_id,
mailbox_id=alias.mailbox_id,
contact_id=contact.id,
alias_id=alias.id,
message_id=str(msg[headers.MESSAGE_ID]),
refused_email_id=refused_email.id,
is_spam=True,
blocked=True,
commit=True,
)
def apply_dmarc_policy_for_reply_phase(
alias_from: Alias, contact_recipient: Contact, envelope: Envelope, msg: Message
) -> Optional[str]:
spam_result = SpamdResult.extract_from_headers(msg, Phase.reply)
if not DMARC_CHECK_ENABLED or not spam_result:
return None
if spam_result.dmarc not in (
DmarcCheckResult.quarantine,
DmarcCheckResult.reject,
DmarcCheckResult.soft_fail,
):
return None
LOG.w(
f"dmarc reply: Put email from {alias_from.email} to {contact_recipient} into quarantine. {spam_result.event_data()}, "
f"mail_from:{envelope.mail_from}, from_header: {msg[headers.FROM]}"
)
send_email_with_rate_control(
alias_from.user,
ALERT_DMARC_FAILED_REPLY_PHASE,
alias_from.user.email,
f"Attempt to send an email to your contact {contact_recipient.email} from {envelope.mail_from}",
render(
"transactional/spoof-reply.txt.jinja2",
contact=contact_recipient,
alias=alias_from,
sender=envelope.mail_from,
),
render(
"transactional/spoof-reply.html",
contact=contact_recipient,
alias=alias_from,
sender=envelope.mail_from,
),
)
return status.E215

127
app/handler/spamd_result.py Normal file
View File

@ -0,0 +1,127 @@
from __future__ import annotations
from typing import Dict, Optional
import newrelic
from app.email import headers
from app.models import EnumE
from email.message import Message
class Phase(EnumE):
unknown = 0
forward = 1
reply = 2
class DmarcCheckResult(EnumE):
allow = 0
soft_fail = 1
quarantine = 2
reject = 3
not_available = 4
bad_policy = 5
@staticmethod
def get_string_dict():
return {
"DMARC_POLICY_ALLOW": DmarcCheckResult.allow,
"DMARC_POLICY_SOFTFAIL": DmarcCheckResult.soft_fail,
"DMARC_POLICY_QUARANTINE": DmarcCheckResult.quarantine,
"DMARC_POLICY_REJECT": DmarcCheckResult.reject,
"DMARC_NA": DmarcCheckResult.not_available,
"DMARC_BAD_POLICY": DmarcCheckResult.bad_policy,
}
class SPFCheckResult(EnumE):
allow = 0
fail = 1
soft_fail = 1
neutral = 2
temp_error = 3
not_available = 4
perm_error = 5
@staticmethod
def get_string_dict():
return {
"R_SPF_ALLOW": SPFCheckResult.allow,
"R_SPF_FAIL": SPFCheckResult.fail,
"R_SPF_SOFTFAIL": SPFCheckResult.soft_fail,
"R_SPF_NEUTRAL": SPFCheckResult.neutral,
"R_SPF_DNSFAIL": SPFCheckResult.temp_error,
"R_SPF_NA": SPFCheckResult.not_available,
"R_SPF_PERMFAIL": SPFCheckResult.perm_error,
}
class SpamdResult:
def __init__(self, phase: Phase = Phase.unknown):
self.phase: Phase = phase
self.dmarc: DmarcCheckResult = DmarcCheckResult.not_available
self.spf: SPFCheckResult = SPFCheckResult.not_available
def set_dmarc_result(self, dmarc_result: DmarcCheckResult):
self.dmarc = dmarc_result
def set_spf_result(self, spf_result: SPFCheckResult):
self.spf = spf_result
def event_data(self) -> Dict:
return {
"header": "present",
"dmarc": self.dmarc.name,
"spf": self.spf.name,
"phase": self.phase.name,
}
@classmethod
def extract_from_headers(
cls, msg: Message, phase: Phase = Phase.unknown
) -> Optional[SpamdResult]:
cached = cls._get_from_message(msg)
if cached:
return cached
spam_result_header = msg.get_all(headers.SPAMD_RESULT)
if not spam_result_header:
return None
spam_entries = [
entry.strip() for entry in str(spam_result_header[-1]).split("\n")
]
for entry_pos in range(len(spam_entries)):
sep = spam_entries[entry_pos].find("(")
if sep > -1:
spam_entries[entry_pos] = spam_entries[entry_pos][:sep]
spamd_result = SpamdResult(phase)
for header_value, dmarc_result in DmarcCheckResult.get_string_dict().items():
if header_value in spam_entries:
spamd_result.set_dmarc_result(dmarc_result)
break
for header_value, spf_result in SPFCheckResult.get_string_dict().items():
if header_value in spam_entries:
spamd_result.set_spf_result(spf_result)
break
cls._store_in_message(spamd_result, msg)
return spamd_result
@classmethod
def _store_in_message(cls, check: SpamdResult, msg: Message):
msg.spamd_check = check
@classmethod
def _get_from_message(cls, msg: Message) -> Optional[SpamdResult]:
return getattr(msg, "spamd_check", None)
@classmethod
def send_to_new_relic(cls, msg: Message):
check = cls._get_from_message(msg)
if check:
newrelic.agent.record_custom_event("SpamdCheck", check.event_data())
else:
newrelic.agent.record_custom_event("SpamdCheck", {"header": "missing"})

59
app/models.py
View File

@ -3,7 +3,7 @@ import os
import random
import uuid
from email.utils import formataddr
from typing import List, Tuple, Optional, Dict
from typing import List, Tuple, Optional
import arrow
import sqlalchemy as sa
@ -237,63 +237,6 @@ class AuditLogActionEnum(EnumE):
extend_subscription = 7
class DmarcCheckResult(EnumE):
allow = 0
soft_fail = 1
quarantine = 2
reject = 3
not_available = 4
bad_policy = 5
@staticmethod
def get_string_dict():
return {
"DMARC_POLICY_ALLOW": DmarcCheckResult.allow,
"DMARC_POLICY_SOFTFAIL": DmarcCheckResult.soft_fail,
"DMARC_POLICY_QUARANTINE": DmarcCheckResult.quarantine,
"DMARC_POLICY_REJECT": DmarcCheckResult.reject,
"DMARC_NA": DmarcCheckResult.not_available,
"DMARC_BAD_POLICY": DmarcCheckResult.bad_policy,
}
class SPFCheckResult(EnumE):
allow = 0
fail = 1
soft_fail = 1
neutral = 2
temp_error = 3
not_available = 4
perm_error = 5
@staticmethod
def get_string_dict():
return {
"R_SPF_ALLOW": SPFCheckResult.allow,
"R_SPF_FAIL": SPFCheckResult.fail,
"R_SPF_SOFTFAIL": SPFCheckResult.soft_fail,
"R_SPF_NEUTRAL": SPFCheckResult.neutral,
"R_SPF_DNSFAIL": SPFCheckResult.temp_error,
"R_SPF_NA": SPFCheckResult.not_available,
"R_SPF_PERMFAIL": SPFCheckResult.perm_error,
}
class SpamdResult:
def __init__(self):
self.dmarc: DmarcCheckResult = DmarcCheckResult.not_available
self.spf: SPFCheckResult = SPFCheckResult.not_available
def set_dmarc_result(self, dmarc_result: DmarcCheckResult):
self.dmarc = dmarc_result
def set_spf_result(self, spf_result: SPFCheckResult):
self.spf = spf_result
def event_data(self) -> Dict:
return {"header": "present", "dmarc": self.dmarc, "spf": self.spf}
class VerpType(EnumE):
bounce_forward = 0
bounce_reply = 1

148
email_handler.py
View File

@ -86,10 +86,16 @@ from app.config import (
OLD_UNSUBSCRIBER,
ALERT_FROM_ADDRESS_IS_REVERSE_ALIAS,
ALERT_TO_NOREPLY,
DMARC_CHECK_ENABLED,
ALERT_QUARANTINE_DMARC,
)
from app.db import Session
from app.handler.dmarc import (
apply_dmarc_policy_for_reply_phase,
apply_dmarc_policy_for_forward_phase,
)
from app.handler.spamd_result import (
SpamdResult,
SPFCheckResult,
)
from app.email import status, headers
from app.email.rate_limit import rate_limited
from app.email.spam import get_spam_score
@ -129,9 +135,9 @@ from app.email_utils import (
get_orig_message_from_yahoo_complaint,
get_mailbox_bounce_info,
save_email_for_debugging,
get_spamd_result,
generate_verp_email,
save_envelope_for_debugging,
get_verp_info_from_email,
generate_verp_email,
)
from app.errors import (
NonReverseAliasInReplyPhase,
@ -156,8 +162,6 @@ from app.models import (
DeletedAlias,
DomainDeletedAlias,
Notification,
DmarcCheckResult,
SPFCheckResult,
VerpType,
)
from app.pgp_utils import PGPException, sign_data_with_pgpy, sign_data
@ -283,7 +287,7 @@ def get_or_create_reply_to_contact(
return contact
else:
LOG.d(
"create contact %s for alias %s via reply-to header",
"create contact %s for alias %s via reply-to header %s",
contact_address,
alias,
reply_to_header,
@ -543,99 +547,6 @@ def handle_email_sent_to_ourself(alias, from_addr: str, msg: Message, user):
)
def apply_dmarc_policy(
alias: Alias, contact: Contact, envelope: Envelope, msg: Message
) -> Optional[str]:
spam_result = get_spamd_result(msg)
if not DMARC_CHECK_ENABLED or not spam_result:
return None
from_header = get_header_unicode(msg[headers.FROM])
# todo: remove when soft_fail email is put into quarantine
if spam_result.dmarc == DmarcCheckResult.soft_fail:
LOG.w(
f"dmarc soft_fail from contact {contact.email} to alias {alias.email}."
f"mail_from:{envelope.mail_from}, from_header: {from_header}"
)
return None
if spam_result.dmarc in (
DmarcCheckResult.quarantine,
DmarcCheckResult.reject,
# todo: disable soft_fail for now
# DmarcCheckResult.soft_fail,
):
LOG.w(
f"put email from {contact} to {alias} to quarantine. {spam_result.event_data()}, "
f"mail_from:{envelope.mail_from}, from_header: {msg[headers.FROM]}"
)
email_log = quarantine_dmarc_failed_email(alias, contact, envelope, msg)
Notification.create(
user_id=alias.user_id,
title=f"{alias.email} has a new mail in quarantine",
message=Notification.render(
"notification/message-quarantine.html", alias=alias
),
commit=True,
)
user = alias.user
send_email_with_rate_control(
user,
ALERT_QUARANTINE_DMARC,
user.email,
f"An email sent to {alias.email} has been quarantined",
render(
"transactional/message-quarantine-dmarc.txt.jinja2",
from_header=from_header,
alias=alias,
refused_email_url=email_log.get_dashboard_url(),
),
render(
"transactional/message-quarantine-dmarc.html",
from_header=from_header,
alias=alias,
refused_email_url=email_log.get_dashboard_url(),
),
max_nb_alert=10,
ignore_smtp_error=True,
)
return status.E215
return None
def quarantine_dmarc_failed_email(alias, contact, envelope, msg) -> EmailLog:
add_or_replace_header(msg, headers.SL_DIRECTION, "Forward")
msg[headers.SL_ENVELOPE_TO] = alias.email
msg[headers.SL_ENVELOPE_FROM] = envelope.mail_from
add_or_replace_header(msg, "From", contact.new_addr())
# replace CC & To emails by reverse-alias for all emails that are not alias
try:
replace_header_when_forward(msg, alias, "Cc")
replace_header_when_forward(msg, alias, "To")
except CannotCreateContactForReverseAlias:
Session.commit()
raise
random_name = str(uuid.uuid4())
s3_report_path = f"refused-emails/full-{random_name}.eml"
s3.upload_email_from_bytesio(
s3_report_path, BytesIO(to_bytes(msg)), f"full-{random_name}"
)
refused_email = RefusedEmail.create(
full_report_path=s3_report_path, user_id=alias.user_id, flush=True
)
return EmailLog.create(
user_id=alias.user_id,
mailbox_id=alias.mailbox_id,
contact_id=contact.id,
alias_id=alias.id,
message_id=str(msg[headers.MESSAGE_ID]),
refused_email_id=refused_email.id,
is_spam=True,
blocked=True,
commit=True,
)
def handle_forward(envelope, msg: Message, rcpt_to: str) -> List[Tuple[bool, str]]:
"""return an array of SMTP status (is_success, smtp_status)
is_success indicates whether an email has been delivered and
@ -717,7 +628,9 @@ def handle_forward(envelope, msg: Message, rcpt_to: str) -> List[Tuple[bool, str
return [(True, res_status)]
# Check if we need to reject or quarantine based on dmarc
dmarc_delivery_status = apply_dmarc_policy(alias, contact, envelope, msg)
msg, dmarc_delivery_status = apply_dmarc_policy_for_forward_phase(
alias, contact, envelope, msg
)
if dmarc_delivery_status is not None:
return [(False, dmarc_delivery_status)]
@ -1031,6 +944,7 @@ def handle_reply(envelope, msg: Message, rcpt_to: str) -> (bool, str):
Return whether an email has been delivered and
the smtp status ("250 Message accepted", "550 Non-existent email address", etc)
"""
reply_email = rcpt_to
# reply_email must end with EMAIL_DOMAIN
@ -1066,7 +980,14 @@ def handle_reply(envelope, msg: Message, rcpt_to: str) -> (bool, str):
alias,
contact,
)
return [(False, status.E504)]
return False, status.E504
# Check if we need to reject or quarantine based on dmarc
dmarc_delivery_status = apply_dmarc_policy_for_reply_phase(
alias, contact, envelope, msg
)
if dmarc_delivery_status is not None:
return False, dmarc_delivery_status
# Anti-spoofing
mailbox = get_mailbox_from_mail_from(mail_from, alias)
@ -2069,7 +1990,7 @@ def handle_unsubscribe_user(user_id: int, mail_from: str) -> str:
return status.E510
if mail_from != user.email:
LOG.e("Unauthorized mail_from %s %s", user, mail_from)
LOG.w("Unauthorized mail_from %s %s", user, mail_from)
return status.E511
user.notification = False
@ -2212,6 +2133,11 @@ def handle(envelope: Envelope, msg: Message) -> str:
envelope.mail_from = mail_from
envelope.rcpt_tos = rcpt_tos
# some emails don't have this header, set the default value (7bit) in this case
if headers.CONTENT_TRANSFER_ENCODING not in msg:
LOG.i("Set CONTENT_TRANSFER_ENCODING")
msg[headers.CONTENT_TRANSFER_ENCODING] = "7bit"
postfix_queue_id = get_queue_id(msg)
if postfix_queue_id:
set_message_id(postfix_queue_id)
@ -2362,10 +2288,10 @@ def handle(envelope: Envelope, msg: Message) -> str:
email_log = EmailLog.get(email_log_id)
alias = Alias.get_by(email=rcpt_tos[0])
LOG.w(
"iCloud bounces %s %s msg=%s",
"iCloud bounces %s %s, saved to%s",
email_log,
alias,
msg.as_string(),
save_email_for_debugging(msg, file_name_prefix="icloud_bounce_"),
)
return handle_bounce(envelope, email_log, msg)
@ -2554,7 +2480,7 @@ class MailHandler:
msg[headers.TO],
)
return status.E524
except (VERPReply, VERPForward) as e:
except (VERPReply, VERPForward, VERPTransactional) as e:
LOG.w(
"email handling fail with error:%s "
"mail_from:%s, rcpt_tos:%s, header_from:%s, header_to:%s",
@ -2574,8 +2500,8 @@ class MailHandler:
envelope.rcpt_tos,
msg[headers.FROM],
msg[headers.TO],
save_email_for_debugging(
msg, file_name_prefix=e.__class__.__name__
save_envelope_for_debugging(
envelope, file_name_prefix=e.__class__.__name__
), # todo: remove
)
return status.E404
@ -2602,9 +2528,9 @@ class MailHandler:
return_status = handle(envelope, msg)
elapsed = time.time() - start
# Only bounce messages if the return-path passes the spf check. Otherwise black-hole it.
spamd_result = SpamdResult.extract_from_headers(msg)
if return_status[0] == "5":
spamd_result = get_spamd_result(msg)
if spamd_result and get_spamd_result(msg).spf in (
if spamd_result and spamd_result.spf in (
SPFCheckResult.fail,
SPFCheckResult.soft_fail,
):
@ -2620,6 +2546,8 @@ class MailHandler:
elapsed,
return_status,
)
SpamdResult.send_to_new_relic(msg)
newrelic.agent.record_custom_metric("Custom/email_handler_time", elapsed)
newrelic.agent.record_custom_metric("Custom/number_incoming_email", 1)
return return_status

14
templates/dashboard/index.html
View File

@ -579,13 +579,13 @@
<script src="/static/js/index.js?v=0"></script>
<script>
{% if show_intro %}
// only show intro when screen is big enough to show "developer" tab
if (window.innerWidth >= 1024) {
introJs().start();
}
{% endif %}
{% if show_intro %}
// only show intro when screen is big enough to show "developer" tab
if (window.innerWidth >= 1024) {
introJs().start();
}
{% endif %}
$('.highlighted').tooltip("show");
$('.highlighted').tooltip("show");
</script>
{% endblock %}

24
templates/dashboard/setting.html
View File

@ -494,7 +494,8 @@
<div class="card-body">
<div class="card-title">Disabled alias/Blocked contact</div>
<div class="mb-3">
When an email is sent to a <b>disabled</b> alias or sent from a <b>blocked</b> contact, you can decide what response the sender should see. <br>
When an email is sent to a <b>disabled</b> alias or sent from a <b>blocked</b> contact, you can decide what
response the sender should see. <br>
<b>Ignore</b> means they will see the message as delivered, but SimpleLogin won't actually forward it to you.
This is the default option as you can start receiving the emails again
by re-enabling the alias or unblocking a contact.<br>
@ -504,14 +505,16 @@
<input type="hidden" name="form-name" value="change-blocked-behaviour">
<select class="form-control mr-sm-2" name="blocked-behaviour">
<option value="{{ BlockBehaviourEnum.return_2xx.value }}"
{% if current_user.block_behaviour.value == BlockBehaviourEnum.return_2xx.value %} selected="selected" {% endif %}>
Ignore (the sender will see the email as delivered, but you won't receive anything).
</option>
<option value="{{ BlockBehaviourEnum.return_5xx.value }}"
{% if current_user.block_behaviour.value == BlockBehaviourEnum.return_5xx.value %} selected="selected" {% endif %}>
Reject (the sender will be told that your alias does not exist).
</option>
<option value="{{ BlockBehaviourEnum.return_2xx.value }}"
{% if current_user.block_behaviour.value == BlockBehaviourEnum.return_2xx.value %}
selected="selected" {% endif %}>
Ignore (the sender will see the email as delivered, but you won't receive anything).
</option>
<option value="{{ BlockBehaviourEnum.return_5xx.value }}"
{% if current_user.block_behaviour.value == BlockBehaviourEnum.return_5xx.value %}
selected="selected" {% endif %}>
Reject (the sender will be told that your alias does not exist).
</option>
</select>
@ -525,7 +528,8 @@
<div class="card-title">Include original sender in email headers
</div>
<div class="mb-3">
SimpleLogin forwards emails to your mailbox from the <b>reverse-alias</b> and not from the <b>original</b> sender address. <br>
SimpleLogin forwards emails to your mailbox from the <b>reverse-alias</b> and not from the <b>original</b>
sender address. <br>
If this option is enabled, the original sender addresses is stored in the email header <b>X-SimpleLogin-Envelope-From</b>.
You can choose to display this header in your email client. <br>
As email headers aren't encrypted, your mailbox service can know the sender address via this header.

2
templates/emails/transactional/hotmail-complaint-reply-phase.txt.jinja2
View File

@ -11,5 +11,5 @@ Please note that sending non-solicited from a SimpleLogin alias infringes our te
If somehow the recipient's Hotmail considers a forwarded email as Spam, it helps us a lot if you can ask them to move the email out of their Spam folder.
Looking to hear back from you.
Don't hesitate to get in touch with us if you need more information.
{% endblock %}

2
templates/emails/transactional/hotmail-complaint.html
View File

@ -28,7 +28,7 @@
{% endcall %}
{% call text() %}
Looking to hear back from you.
Don't hesitate to get in touch with us if you need more information.
{% endcall %}
{% call text() %}

2
templates/emails/transactional/hotmail-complaint.txt.jinja2
View File

@ -16,5 +16,5 @@ If that’s the case, please disable the alias instead if you don't want to rece
If somehow Hotmail considers a forwarded email as Spam, it will help us if you can move the email out of the Spam folder.
You can also set up a filter to avoid this from happening in the future using this guide at https://simplelogin.io/help/
Looking to hear back from you.
Don't hesitate to get in touch with us if you need more information.
{% endblock %}

2
templates/emails/transactional/hotmail-transactional-complaint.html
View File

@ -29,7 +29,7 @@
{% endcall %}
{% call text() %}
Looking to hear back from you.
Don't hesitate to get in touch with us if you need more information.
{% endcall %}
{% call text() %}

2
templates/emails/transactional/hotmail-transactional-complaint.txt.jinja2
View File

@ -19,5 +19,5 @@ If somehow Hotmail considers a forwarded email as Spam, it helps us if you can m
Please don't put our emails into the Spam folder. This can end up in your account being disabled on SimpleLogin.
Looking to hear back from you.
Don't hesitate to get in touch with us if you need more information.
{% endblock %}

4
templates/emails/transactional/message-quarantine-dmarc.html
View File

@ -8,8 +8,8 @@
{% endcall %}
{% call text() %}
An email from {{ from_header }} to {{ alias.email }} is put into Quarantine as it fails DMARC check.
DMARC is an email authentication protocol designed for detecting phishing.
An email from {{ from_header }} to {{ alias.email }} is put into Quarantine as it fails
<a href="https://simplelogin.io/docs/getting-started/anti-phishing/">anti-phishing measure</a> check.
{% endcall %}
{{ render_button("View the original email", refused_email_url) }}

5
templates/emails/transactional/message-quarantine-dmarc.txt.jinja2
View File

@ -1,8 +1,11 @@
{% extends "base.txt.jinja2" %}
{% block content %}
An email from {{ from_header }} to {{ alias.email }} is put into Quarantine as it fails DMARC check.
An email from {{ from_header }} to {{ alias.email }} is put into Quarantine as it fails anti-phishing check.
You can view the email at {{ refused_email_url }}.
This email is automatically deleted in 7 days.
More info about the anti-phishing measure on https://simplelogin.io/docs/getting-started/anti-phishing/
{% endblock %}

22
templates/emails/transactional/spoof-reply.html Normal file
View File

@ -0,0 +1,22 @@
{% extends "base.html" %}
{% block content %}
{% call text() %}
<h1>
Unauthorized attempt to send an email to {{ contact.email }} from your alias <b>{{ alias.email }}</b> using
<b>{{ sender }}</b> has been blocked.
</h1>
{% endcall %}
{% call text() %}
To protect against <b>email spoofing</b>, only your mailbox can send emails on behalf of your alias.
SimpleLogin also refuses emails that claim to come from your mailbox but fail DMARC.
{% endcall %}
{% call text() %}
Best, <br/>
SimpleLogin Team.
{% endcall %}
{% endblock %}

10
templates/emails/transactional/spoof-reply.txt.jinja2 Normal file
View File

@ -0,0 +1,10 @@
{% extends "base.txt.jinja2" %}
{% block content %}
Unauthorized attempt to send an email to {{ contact.email }} from your alias {{ alias.email }} using
{{ sender }} has been blocked.
To protect against email spoofing, only your mailbox can send emails on behalf of your alias.
SimpleLogin also refuses emails that claim to come from your mailbox but fail DMARC.
{% endblock %}

2
templates/emails/transactional/yahoo-complaint.html
View File

@ -24,7 +24,7 @@
{% endcall %}
{% call text() %}
Looking to hear back from you.
Don't hesitate to get in touch with us if you need more information.
{% endcall %}
{% call text() %}

2
templates/emails/transactional/yahoo-complaint.txt.jinja2
View File

@ -14,5 +14,5 @@ If that’s the case, please disable the alias instead if you don't want to rece
If SimpleLogin isn’t useful for you, please know that you can simply delete your account on the Settings page.
Looking to hear back from you.
Don't hesitate to get in touch with us if you need more information.
{% endblock %}

2
templates/emails/transactional/yahoo-transactional-complaint.html
View File

@ -29,7 +29,7 @@
{% endcall %}
{% call text() %}
Looking to hear back from you.
Don't hesitate to get in touch with us if you need more information.
{% endcall %}
{% call text() %}

0
tests/email_tests/__init__.py Normal file
View File

0
tests/email/test_rate_limit.py → tests/email_tests/test_rate_limit.py
View File

25
tests/example_emls/dmarc_reply_check.eml Normal file
View File

@ -0,0 +1,25 @@
X-SimpleLogin-Client-IP: 54.39.200.130
Received-SPF: Softfail (mailfrom) identity=mailfrom; client-ip=34.59.200.130;
helo=relay.somewhere.net; envelope-from=everwaste@gmail.com;
receiver=<UNKNOWN>
Received: from relay.somewhere.net (relay.somewhere.net [34.59.200.130])
(using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits))
(No client certificate requested)
by mx1.sldev.ovh (Postfix) with ESMTPS id 6D8C13F069
for <wehrman_mannequin@sldev.ovh>; Thu, 17 Mar 2022 16:50:20 +0000 (UTC)
Date: Thu, 17 Mar 2022 16:50:18 +0000
To: {{ contact_email }}
From: {{ alias_email }}
Subject: test Thu, 17 Mar 2022 16:50:18 +0000
Message-Id: <20220317165018.000191@somewhere-5488dd4b6b-7crp6>
X-Mailer: swaks v20201014.0 jetmore.org/john/code/swaks/
X-Rspamd-Queue-Id: 6D8C13F069
X-Rspamd-Server: staging1
X-Spamd-Result: default: False [0.50 / 13.00];
{{ dmarc_result }}(0.00)[];
X-Rspamd-Pre-Result: action=add header;
module=force_actions;
unknown reason
X-Spam: Yes
This is a test mailing

25
tests/example_emls/multipart_alternative.eml Normal file
View File

@ -0,0 +1,25 @@
Content-Type: multipart/alternative; boundary="===============5006593052976639648=="
MIME-Version: 1.0
Subject: My subject
From: foo@example.org
To: bar@example.net
--===============5006593052976639648==
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
This is HTML
--===============5006593052976639648==
Content-Type: text/html; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
<html>
<body>
This is <i>HTML</i>
</body>
</html>
--===============5006593052976639648==--

0
tests/handler/__init__.py Normal file
View File

34
tests/handler/test_spamd_result.py Normal file
View File

@ -0,0 +1,34 @@
from app.handler.spamd_result import DmarcCheckResult, SpamdResult
from tests.utils import load_eml_file
def test_dmarc_result_softfail():
msg = load_eml_file("dmarc_gmail_softfail.eml")
assert DmarcCheckResult.soft_fail == SpamdResult.extract_from_headers(msg).dmarc
def test_dmarc_result_quarantine():
msg = load_eml_file("dmarc_quarantine.eml")
assert DmarcCheckResult.quarantine == SpamdResult.extract_from_headers(msg).dmarc
def test_dmarc_result_reject():
msg = load_eml_file("dmarc_reject.eml")
assert DmarcCheckResult.reject == SpamdResult.extract_from_headers(msg).dmarc
def test_dmarc_result_allow():
msg = load_eml_file("dmarc_allow.eml")
assert DmarcCheckResult.allow == SpamdResult.extract_from_headers(msg).dmarc
def test_dmarc_result_na():
msg = load_eml_file("dmarc_na.eml")
assert DmarcCheckResult.not_available == SpamdResult.extract_from_headers(msg).dmarc
def test_dmarc_result_bad_policy():
msg = load_eml_file("dmarc_bad_policy.eml")
assert SpamdResult._get_from_message(msg) is None
assert DmarcCheckResult.bad_policy == SpamdResult.extract_from_headers(msg).dmarc
assert SpamdResult._get_from_message(msg) is not None

76
tests/test_email_handler.py
View File

@ -1,8 +1,13 @@
import random
from email.message import EmailMessage
from typing import List
import pytest
from aiosmtpd.smtp import Envelope
import email_handler
from app.config import EMAIL_DOMAIN, ALERT_DMARC_FAILED_REPLY_PHASE
from app.db import Session
from app.email import headers, status
from app.email_utils import generate_verp_email
from app.models import (
@ -13,6 +18,8 @@ from app.models import (
EmailLog,
Notification,
VerpType,
Contact,
SentAlert,
)
from email_handler import (
get_mailbox_from_mail_from,
@ -76,7 +83,7 @@ def test_is_automatic_out_of_office():
assert is_automatic_out_of_office(msg)
def test_dmarc_quarantine(flask_client):
def test_dmarc_forward_quarantine(flask_client):
user = create_random_user()
alias = Alias.create_new_random(user)
msg = load_eml_file("dmarc_quarantine.eml", {"alias_email": alias.email})
@ -99,25 +106,18 @@ def test_dmarc_quarantine(flask_client):
assert f"{alias.email} has a new mail in quarantine" == notifications[0].title
# todo: re-enable test when softfail is quarantined
# def test_gmail_dmarc_softfail(flask_client):
# user = create_random_user()
# alias = Alias.create_new_random(user)
# msg = load_eml_file("dmarc_gmail_softfail.eml", {"alias_email": alias.email})
# envelope = Envelope()
# envelope.mail_from = msg["from"]
# envelope.rcpt_tos = [msg["to"]]
# result = email_handler.handle(envelope, msg)
# assert result == status.E215
# email_logs = (
# EmailLog.filter_by(user_id=user.id, alias_id=alias.id)
# .order_by(EmailLog.id.desc())
# .all()
# )
# assert len(email_logs) == 1
# email_log = email_logs[0]
# assert email_log.blocked
# assert email_log.refused_email_id
def test_gmail_dmarc_softfail(flask_client):
user = create_random_user()
alias = Alias.create_new_random(user)
msg = load_eml_file("dmarc_gmail_softfail.eml", {"alias_email": alias.email})
envelope = Envelope()
envelope.mail_from = msg["from"]
envelope.rcpt_tos = [msg["to"]]
result = email_handler.handle(envelope, msg)
assert result == status.E200
# Enable when we can verify that the actual message sent has this content
# payload = msg.get_payload()
# assert payload.find("failed anti-phishing checks") > -1
def test_prevent_5xx_from_spf(flask_client):
@ -163,3 +163,39 @@ def test_preserve_5xx_with_no_header(flask_client):
envelope.rcpt_tos = [generate_verp_email(VerpType.bounce_forward, 99999999999999)]
result = email_handler.MailHandler()._handle(envelope, msg)
assert status.E512 == result
def generate_dmarc_result() -> List:
return ["DMARC_POLICY_QUARANTINE", "DMARC_POLICY_REJECT", "DMARC_POLICY_SOFTFAIL"]
@pytest.mark.parametrize("dmarc_result", generate_dmarc_result())
def test_dmarc_reply_quarantine(flask_client, dmarc_result):
user = create_random_user()
alias = Alias.create_new_random(user)
Session.commit()
contact = Contact.create(
user_id=alias.user_id,
alias_id=alias.id,
website_email="random-{}@nowhere.net".format(int(random.random())),
name="Name {}".format(int(random.random())),
reply_email="random-{}@{}".format(random.random(), EMAIL_DOMAIN),
)
Session.commit()
msg = load_eml_file(
"dmarc_reply_check.eml",
{
"alias_email": alias.email,
"contact_email": contact.reply_email,
"dmarc_result": dmarc_result,
},
)
envelope = Envelope()
envelope.mail_from = msg["from"]
envelope.rcpt_tos = [msg["to"]]
result = email_handler.handle(envelope, msg)
assert result == status.E215
alerts = SentAlert.filter_by(
user_id=user.id, alert_type=ALERT_DMARC_FAILED_REPLY_PHASE
).all()
assert len(alerts) == 1

43
tests/test_email_utils.py
View File

@ -36,7 +36,6 @@ from app.email_utils import (
get_orig_message_from_bounce,
get_mailbox_bounce_info,
is_invalid_mailbox_domain,
get_spamd_result,
generate_verp_email,
get_verp_info_from_email,
)
@ -48,7 +47,6 @@ from app.models import (
EmailLog,
IgnoreBounceSender,
InvalidMailboxDomain,
DmarcCheckResult,
VerpType,
)
@ -797,39 +795,20 @@ def test_is_invalid_mailbox_domain(flask_client):
assert not is_invalid_mailbox_domain("xy.zt")
def test_dmarc_result_softfail():
msg = load_eml_file("dmarc_gmail_softfail.eml")
assert DmarcCheckResult.soft_fail == get_spamd_result(msg).dmarc
def test_dmarc_result_quarantine():
msg = load_eml_file("dmarc_quarantine.eml")
assert DmarcCheckResult.quarantine == get_spamd_result(msg).dmarc
def test_dmarc_result_reject():
msg = load_eml_file("dmarc_reject.eml")
assert DmarcCheckResult.reject == get_spamd_result(msg).dmarc
def test_dmarc_result_allow():
msg = load_eml_file("dmarc_allow.eml")
assert DmarcCheckResult.allow == get_spamd_result(msg).dmarc
def test_dmarc_result_na():
msg = load_eml_file("dmarc_na.eml")
assert DmarcCheckResult.not_available == get_spamd_result(msg).dmarc
def test_dmarc_result_bad_policy():
msg = load_eml_file("dmarc_bad_policy.eml")
assert DmarcCheckResult.bad_policy == get_spamd_result(msg).dmarc
def test_generate_verp_email():
generated_email = generate_verp_email(VerpType.bounce_forward, 1, "somewhere.net")
print(generated_email)
info = get_verp_info_from_email(generated_email.lower())
assert info[0] == VerpType.bounce_forward
assert info[1] == 1
def test_add_header_multipart_with_invalid_part():
msg = load_eml_file("multipart_alternative.eml")
parts = msg.get_payload() + ["invalid"]
msg.set_payload(parts)
msg = add_header(msg, "INJECT", "INJECT")
for i, part in enumerate(msg.get_payload()):
if i < 2:
assert part.get_payload().index("INJECT") > -1
else:
assert part == "invalid"

6
tests/utils.py
View File

@ -39,11 +39,11 @@ def random_token(length: int = 10) -> str:
def create_random_user() -> User:
email = "{}@{}.com".format(random_token(), random_token())
random_email = "{}@{}.com".format(random_token(), random_token())
return User.create(
email=email,
email=random_email,
password="password",
name="Test User",
name="Test {}".format(random_token()),
activated=True,
commit=True,
)