Mirror
/
app-MAIL-temp
mirror of https://github.com/simple-login/app.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #860 from acasajus/remove-softfail 

Generate secure transactional emails from address
This commit is contained in:
Son Nguyen Kim 2022-04-19 09:28:47 +02:00 committed by GitHub
parent 199ec09554 d28980a810
commit 259851a04e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 124 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
app/config.py
View File

@ -74,7 +74,6 @@ MONITORING_EMAIL = os.environ.get("MONITORING_EMAIL")
# VERP: mail_from set to BOUNCE_PREFIX + email_log.id + BOUNCE_SUFFIX
BOUNCE_PREFIX = os.environ.get("BOUNCE_PREFIX") or "bounce+"
BOUNCE_SUFFIX = os.environ.get("BOUNCE_SUFFIX") or f"+@{EMAIL_DOMAIN}"
BOUNCE_EMAIL = BOUNCE_PREFIX + "{}" + BOUNCE_SUFFIX
# Used for VERP during reply phase. It's similar to BOUNCE_PREFIX.
# It's needed when sending emails from custom domain to respect DMARC.
@ -93,9 +92,6 @@ TRANSACTIONAL_BOUNCE_PREFIX = (
TRANSACTIONAL_BOUNCE_SUFFIX = (
os.environ.get("TRANSACTIONAL_BOUNCE_SUFFIX") or f"+@{EMAIL_DOMAIN}"
)
TRANSACTIONAL_BOUNCE_EMAIL = (
TRANSACTIONAL_BOUNCE_PREFIX + "{}" + TRANSACTIONAL_BOUNCE_SUFFIX
)
try:
MAX_NB_EMAIL_FREE_PLAN = int(os.environ["MAX_NB_EMAIL_FREE_PLAN"])
@ -169,6 +165,8 @@ DB_URI = os.environ["DB_URI"]
# Flask secret
FLASK_SECRET = os.environ["FLASK_SECRET"]
if not FLASK_SECRET:
raise RuntimeError("FLASK_SECRET is empty. Please define it.")
SESSION_COOKIE_NAME = "slapp"
MAILBOX_SECRET = FLASK_SECRET + "mailbox"
CUSTOM_ALIAS_SECRET = FLASK_SECRET + "custom_alias"
@ -429,6 +427,18 @@ ZENDESK_ENABLED = "ZENDESK_ENABLED" in os.environ
DMARC_CHECK_ENABLED = "DMARC_CHECK_ENABLED" in os.environ
# Bounces can happen after 5 days
VERP_MESSAGE_LIFETIME = 5 * 86400
VERP_PREFIX = os.environ.get("VERP_PREFIX") or "sl"
# Generate with python3 -c 'import secrets; print(secrets.token_hex(28))'
VERP_EMAIL_SECRET = os.environ.get("VERP_EMAIL_SECRET") or (
FLASK_SECRET + "pleasegenerateagoodrandomtoken"
)
if len(VERP_EMAIL_SECRET) < 32:
raise RuntimeError(
"Please, set VERP_EMAIL_SECRET to a random string at least 32 chars long"
)
def get_allowed_redirect_domains() -> List[str]:
allowed_domains = sl_getenv("ALLOWED_REDIRECT_DOMAINS", list)

55
app/email_utils.py
View File

@ -1,5 +1,7 @@
import base64
import enum
import hmac
import json
import os
import quopri
import random
@ -49,13 +51,15 @@ from app.config import (
LANDING_PAGE_URL,
EMAIL_DOMAIN,
ALERT_DIRECTORY_DISABLED_ALIAS_CREATION,
TRANSACTIONAL_BOUNCE_EMAIL,
ALERT_SPF,
ALERT_INVALID_TOTP_LOGIN,
TEMP_DIR,
ALIAS_AUTOMATIC_DISABLE,
RSPAMD_SIGN_DKIM,
NOREPLY,
VERP_PREFIX,
VERP_MESSAGE_LIFETIME,
VERP_EMAIL_SECRET,
)
from app.db import Session
from app.dns_utils import get_mx_domains
@ -73,6 +77,7 @@ from app.models import (
TransactionalEmail,
IgnoreBounceSender,
InvalidMailboxDomain,
VerpType,
)
from app.utils import (
random_string,
@ -324,7 +329,7 @@ def send_email(
# use a different envelope sender for each transactional email (aka VERP)
sl_sendmail(
TRANSACTIONAL_BOUNCE_EMAIL.format(transaction.id),
generate_verp_email(VerpType.transactional, transaction.id),
to_email,
msg,
retries=retries,
@ -1462,3 +1467,49 @@ def save_envelope_for_debugging(envelope: Envelope, file_name_prefix=None) -> st
return file_name
return ""
def generate_verp_email(
verp_type: VerpType, object_id: int, sender_domain: Optional[str] = None
) -> str:
# Encoded as a list to minimize size of email address
data = [verp_type.bounce_forward.value, object_id, int(time.time())]
json_payload = json.dumps(data).encode("utf-8")
# Signing without itsdangereous because it uses base64 that includes +/= symbols and lower and upper case letters.
# We need to encode in base32
payload_hmac = hmac.new(
VERP_EMAIL_SECRET.encode("utf-8"), json_payload, "shake128"
).digest()
encoded_payload = base64.b32encode(json_payload).rstrip(b"=").decode("utf-8")
encoded_signature = base64.b32encode(payload_hmac).rstrip(b"=").decode("utf-8")
return "{}.{}.{}@{}".format(
VERP_PREFIX, encoded_payload, encoded_signature, sender_domain or EMAIL_DOMAIN
).lower()
# This method processes the email address, checks if it's a signed verp email generated by us to receive bounces
# and extracts the type of verp email and associated email log id/transactional email id stored as object_id
def get_verp_info_from_email(email: str) -> Optional[Tuple[VerpType, int]]:
idx = email.find("@")
if idx == -1:
return None
username = email[:idx]
fields = username.split(".")
if len(fields) != 3 or fields[0] != VERP_PREFIX:
return None
padding = 8 - (len(fields[1]) % 8)
payload = base64.b32decode(fields[1].encode("utf-8").upper() + (b"=" * padding))
padding = 8 - (len(fields[2]) % 8)
signature = base64.b32decode(fields[2].encode("utf-8").upper() + (b"=" * padding))
expected_signature = hmac.new(
VERP_EMAIL_SECRET.encode("utf-8"), payload, "shake128"
).digest()
if expected_signature != signature:
return None
data = json.loads(payload)
# verp type, object_id, time
if len(data) != 3:
return None
if data[2] > time.time() + VERP_MESSAGE_LIFETIME:
return None
return VerpType(data[0]), data[1]

6
app/models.py
View File

@ -237,6 +237,12 @@ class AuditLogActionEnum(EnumE):
extend_subscription = 7
class VerpType(EnumE):
bounce_forward = 0
bounce_reply = 1
transactional = 2
class Hibp(Base, ModelMixin):
__tablename__ = "hibp"
name = sa.Column(sa.String(), nullable=False, unique=True, index=True)

41
email_handler.py
View File

@ -72,7 +72,6 @@ from app.config import (
PGP_SENDER_PRIVATE_KEY,
ALERT_BOUNCE_EMAIL_REPLY_PHASE,
NOREPLY,
BOUNCE_EMAIL,
BOUNCE_PREFIX,
BOUNCE_SUFFIX,
TRANSACTIONAL_BOUNCE_PREFIX,
@ -137,6 +136,8 @@ from app.email_utils import (
get_mailbox_bounce_info,
save_email_for_debugging,
save_envelope_for_debugging,
get_verp_info_from_email,
generate_verp_email,
)
from app.errors import (
NonReverseAliasInReplyPhase,
@ -161,6 +162,7 @@ from app.models import (
DeletedAlias,
DomainDeletedAlias,
Notification,
VerpType,
)
from app.pgp_utils import PGPException, sign_data_with_pgpy, sign_data
from app.utils import sanitize_email
@ -876,7 +878,7 @@ def forward_email_to_mailbox(
try:
sl_sendmail(
# use a different envelope sender for each forward (aka VERP)
BOUNCE_EMAIL.format(email_log.id),
generate_verp_email(VerpType.bounce_forward, email_log.id),
mailbox.email,
msg,
envelope.mail_options,
@ -1167,12 +1169,9 @@ def handle_reply(envelope, msg: Message, rcpt_to: str) -> (bool, str):
if should_add_dkim_signature(alias_domain):
add_dkim_signature(msg, alias_domain)
# generate a mail_from for VERP
verp_mail_from = f"{BOUNCE_PREFIX_FOR_REPLY_PHASE}+{email_log.id}+@{alias_domain}"
try:
sl_sendmail(
verp_mail_from,
generate_verp_email(VerpType.bounce_reply, email_log.id, alias_domain),
contact.website_email,
msg,
envelope.mail_options,
@ -2013,11 +2012,13 @@ def handle_unsubscribe_user(user_id: int, mail_from: str) -> str:
return status.E202
def handle_transactional_bounce(envelope: Envelope, msg, rcpt_to):
def handle_transactional_bounce(
envelope: Envelope, msg, rcpt_to, transactional_id=None
):
LOG.d("handle transactional bounce sent to %s", rcpt_to)
# parse the TransactionalEmail
transactional_id = parse_id_from_bounce(rcpt_to)
transactional_id = transactional_id or parse_id_from_bounce(rcpt_to)
transactional = TransactionalEmail.get(transactional_id)
# a transaction might have been deleted in delete_logs()
@ -2211,15 +2212,18 @@ def handle(envelope: Envelope, msg: Message) -> str:
return handle_unsubscribe(envelope, msg)
# region mail sent to VERP
verp_info = get_verp_info_from_email(rcpt_tos[0])
# sent to transactional VERP. Either bounce emails or out-of-office
if (
len(rcpt_tos) == 1
and rcpt_tos[0].startswith(TRANSACTIONAL_BOUNCE_PREFIX)
and rcpt_tos[0].endswith(TRANSACTIONAL_BOUNCE_SUFFIX)
):
) or (verp_info and verp_info[0] == VerpType.transactional):
if is_bounce(envelope, msg):
handle_transactional_bounce(envelope, msg, rcpt_tos[0])
handle_transactional_bounce(
envelope, msg, rcpt_tos[0], verp_info and verp_info[1]
)
return status.E205
elif is_automatic_out_of_office(msg):
LOG.d(
@ -2234,8 +2238,8 @@ def handle(envelope: Envelope, msg: Message) -> str:
len(rcpt_tos) == 1
and rcpt_tos[0].startswith(BOUNCE_PREFIX)
and rcpt_tos[0].endswith(BOUNCE_SUFFIX)
):
email_log_id = parse_id_from_bounce(rcpt_tos[0])
) or (verp_info and verp_info[0] == VerpType.bounce_forward):
email_log_id = (verp_info and verp_info[1]) or parse_id_from_bounce(rcpt_tos[0])
email_log = EmailLog.get(email_log_id)
if not email_log:
@ -2250,10 +2254,12 @@ def handle(envelope: Envelope, msg: Message) -> str:
raise VERPForward
# sent to reply VERP, can be either bounce or out-of-office
if len(rcpt_tos) == 1 and rcpt_tos[0].startswith(
f"{BOUNCE_PREFIX_FOR_REPLY_PHASE}+"
if (
len(rcpt_tos) == 1
and rcpt_tos[0].startswith(f"{BOUNCE_PREFIX_FOR_REPLY_PHASE}+")
or (verp_info and verp_info[0] == VerpType.bounce_reply)
):
email_log_id = parse_id_from_bounce(rcpt_tos[0])
email_log_id = (verp_info and verp_info[1]) or parse_id_from_bounce(rcpt_tos[0])
email_log = EmailLog.get(email_log_id)
if not email_log:
@ -2272,12 +2278,13 @@ def handle(envelope: Envelope, msg: Message) -> str:
)
# iCloud returns the bounce with mail_from=bounce+{email_log_id}+@simplelogin.co, rcpt_to=alias
verp_info = get_verp_info_from_email(mail_from[0])
if (
len(rcpt_tos) == 1
and mail_from.startswith(BOUNCE_PREFIX)
and mail_from.endswith(BOUNCE_SUFFIX)
):
email_log_id = parse_id_from_bounce(mail_from)
) or (verp_info and verp_info[0] == VerpType.bounce_forward):
email_log_id = (verp_info and verp_info[1]) or parse_id_from_bounce(mail_from)
email_log = EmailLog.get(email_log_id)
alias = Alias.get_by(email=rcpt_tos[0])
LOG.w(

25
tests/test_email_handler.py
View File

@ -6,15 +6,17 @@ import pytest
from aiosmtpd.smtp import Envelope
import email_handler
from app.config import BOUNCE_EMAIL, EMAIL_DOMAIN, ALERT_DMARC_FAILED_REPLY_PHASE
from app.config import EMAIL_DOMAIN, ALERT_DMARC_FAILED_REPLY_PHASE
from app.db import Session
from app.email import headers, status
from app.email_utils import generate_verp_email
from app.models import (
Alias,
AuthorizedAddress,
IgnoredEmail,
EmailLog,
Notification,
VerpType,
Contact,
SentAlert,
)
@ -119,10 +121,11 @@ def test_prevent_5xx_from_spf(flask_client):
{"alias_email": alias.email, "spf_result": "R_SPF_FAIL"},
)
envelope = Envelope()
envelope.mail_from = BOUNCE_EMAIL.format(999999999999999999)
envelope.rcpt_tos = [msg["to"]]
envelope.mail_from = msg["from"]
# Ensure invalid email log
envelope.rcpt_tos = [generate_verp_email(VerpType.bounce_forward, 99999999999999)]
result = email_handler.MailHandler()._handle(envelope, msg)
assert result == status.E216
assert status.E216 == result
def test_preserve_5xx_with_valid_spf(flask_client):
@ -133,10 +136,11 @@ def test_preserve_5xx_with_valid_spf(flask_client):
{"alias_email": alias.email, "spf_result": "R_SPF_ALLOW"},
)
envelope = Envelope()
envelope.mail_from = BOUNCE_EMAIL.format(999999999999999999)
envelope.rcpt_tos = [msg["to"]]
envelope.mail_from = msg["from"]
# Ensure invalid email log
envelope.rcpt_tos = [generate_verp_email(VerpType.bounce_forward, 99999999999999)]
result = email_handler.MailHandler()._handle(envelope, msg)
assert result == status.E512
assert status.E512 == result
def test_preserve_5xx_with_no_header(flask_client):
@ -147,10 +151,11 @@ def test_preserve_5xx_with_no_header(flask_client):
{"alias_email": alias.email},
)
envelope = Envelope()
envelope.mail_from = BOUNCE_EMAIL.format(999999999999999999)
envelope.rcpt_tos = [msg["to"]]
envelope.mail_from = msg["from"]
# Ensure invalid email log
envelope.rcpt_tos = [generate_verp_email(VerpType.bounce_forward, 99999999999999)]
result = email_handler.MailHandler()._handle(envelope, msg)
assert result == status.E512
assert status.E512 == result
def generate_dmarc_result() -> List:

14
tests/test_email_utils.py
View File

@ -5,7 +5,7 @@ from email.message import EmailMessage
import arrow
import pytest
from app.config import MAX_ALERT_24H, EMAIL_DOMAIN, BOUNCE_EMAIL, ROOT_DIR
from app.config import MAX_ALERT_24H, EMAIL_DOMAIN, ROOT_DIR
from app.db import Session
from app.email_utils import (
get_email_domain_part,
@ -36,6 +36,8 @@ from app.email_utils import (
get_orig_message_from_bounce,
get_mailbox_bounce_info,
is_invalid_mailbox_domain,
generate_verp_email,
get_verp_info_from_email,
)
from app.models import (
CustomDomain,
@ -44,6 +46,7 @@ from app.models import (
EmailLog,
IgnoreBounceSender,
InvalidMailboxDomain,
VerpType,
)
# flake8: noqa: E101, W191
@ -712,7 +715,6 @@ def test_should_disable_bounce_consecutive_days(flask_client):
def test_parse_id_from_bounce():
assert parse_id_from_bounce("bounces+1234+@local") == 1234
assert parse_id_from_bounce("anything+1234+@local") == 1234
assert parse_id_from_bounce(BOUNCE_EMAIL.format(1234)) == 1234
def test_get_queue_id():
@ -768,6 +770,14 @@ def test_is_invalid_mailbox_domain(flask_client):
assert not is_invalid_mailbox_domain("xy.zt")
def test_generate_verp_email():
generated_email = generate_verp_email(VerpType.bounce_forward, 1, "somewhere.net")
print(generated_email)
info = get_verp_info_from_email(generated_email.lower())
assert info[0] == VerpType.bounce_forward
assert info[1] == 1
def test_add_header_multipart_with_invalid_part():
msg = load_eml_file("multipart_alternative.eml")
parts = msg.get_payload() + ["invalid"]