app-MAIL-temp/email_handler.py

1898 lines
60 KiB
Python
Raw Normal View History

"""
Handle the email *forward* and *reply*. phase. There are 3 actors:
- contact: who sends emails to alias@sl.co address
- SL email handler (this script)
- user personal email: to be protected. Should never leak to contact.
This script makes sure that in the forward phase, the email that is forwarded to user personal email has the following
envelope and header fields:
Envelope:
mail from: @contact
rcpt to: @personal_email
Header:
From: @contact
To: alias@sl.co # so user knows this email is sent to alias
Reply-to: special@sl.co # magic HERE
And in the reply phase:
Envelope:
mail from: @contact
rcpt to: @contact
Header:
From: alias@sl.co # so for contact the email comes from alias. magic HERE
To: @contact
The special@sl.co allows to hide user personal email when user clicks "Reply" to the forwarded email.
It should contain the following info:
- alias
- @contact
"""
import argparse
import email
import time
import uuid
from email import encoders
from email.encoders import encode_noop
from email.message import Message
from email.mime.application import MIMEApplication
from email.mime.multipart import MIMEMultipart
from email.utils import formataddr, make_msgid, formatdate, getaddresses
from io import BytesIO
from smtplib import SMTPRecipientsRefused
2020-09-28 17:41:16 +02:00
from typing import List, Tuple, Optional
import newrelic.agent
from aiosmtpd.controller import Controller
from aiosmtpd.smtp import Envelope
2020-09-03 19:42:52 +02:00
from sqlalchemy.exc import IntegrityError
from app import pgp_utils, s3
from app.alias_utils import try_auto_create
from app.config import (
EMAIL_DOMAIN,
URL,
2020-03-28 23:19:25 +01:00
UNSUBSCRIBER,
LOAD_PGP_EMAIL_HANDLER,
2020-05-07 13:28:04 +02:00
ENFORCE_SPF,
ALERT_REVERSE_ALIAS_UNKNOWN_MAILBOX,
ALERT_BOUNCE_EMAIL,
ALERT_SPAM_EMAIL,
2020-08-15 16:38:16 +02:00
SPAMASSASSIN_HOST,
MAX_SPAM_SCORE,
2020-08-15 16:53:57 +02:00
MAX_REPLY_PHASE_SPAM_SCORE,
ALERT_SEND_EMAIL_CYCLE,
ALERT_MAILBOX_IS_ALIAS,
PGP_SENDER_PRIVATE_KEY,
2020-11-04 12:32:15 +01:00
ALERT_BOUNCE_EMAIL_REPLY_PHASE,
NOREPLY,
BOUNCE_EMAIL,
BOUNCE_PREFIX,
BOUNCE_SUFFIX,
TRANSACTIONAL_BOUNCE_PREFIX,
TRANSACTIONAL_BOUNCE_SUFFIX,
ENABLE_SPAM_ASSASSIN,
BOUNCE_PREFIX_FOR_REPLY_PHASE,
NEWRELIC_CONFIG_PATH,
2021-09-06 19:44:18 +02:00
POSTMASTER,
ALERT_HOTMAIL_COMPLAINT,
)
from app.email import status
from app.email.rate_limit import rate_limited
from app.email.spam import get_spam_score
from app.email_utils import (
send_email,
add_dkim_signature,
2020-01-07 19:50:36 +01:00
add_or_replace_header,
delete_header,
render,
get_orig_message_from_bounce,
2020-03-14 22:24:02 +01:00
delete_all_headers_except,
get_spam_info,
get_orig_message_from_spamassassin_report,
2020-04-05 14:50:12 +02:00
parseaddr_unicode,
send_email_with_rate_control,
2020-06-09 17:16:32 +02:00
get_email_domain_part,
2020-08-21 12:03:23 +02:00
copy,
to_bytes,
send_email_at_most_times,
2020-10-15 16:21:31 +02:00
is_valid_alias_address_domain,
should_add_dkim_signature,
add_header,
2020-11-09 21:16:50 +01:00
get_header_unicode,
generate_reply_email,
2020-11-16 19:22:19 +01:00
is_reply_email,
normalize_reply_email,
is_valid_email,
replace,
should_disable,
parse_id_from_bounce,
spf_pass,
sl_sendmail,
2021-03-17 10:59:13 +01:00
sanitize_header,
2021-06-04 17:15:59 +02:00
get_queue_id,
should_ignore_bounce,
2021-09-06 19:44:18 +02:00
get_orig_message_from_outlook_complaint,
2020-01-07 19:50:36 +01:00
)
from app.extensions import db
from app.log import LOG, set_message_id
from app.models import (
Alias,
2020-03-17 10:56:59 +01:00
Contact,
2020-03-17 11:10:50 +01:00
EmailLog,
User,
RefusedEmail,
2020-05-07 13:28:04 +02:00
Mailbox,
2021-01-26 09:56:13 +01:00
Bounce,
TransactionalEmail,
IgnoredEmail,
)
from app.pgp_utils import PGPException, sign_data_with_pgpy, sign_data
2021-03-17 10:59:13 +01:00
from app.utils import sanitize_email
from init_app import load_pgp_public_keys
2020-08-11 16:18:47 +02:00
from server import create_app, create_light_app
# forward or reply
_DIRECTION = "X-SimpleLogin-Type"
2021-03-17 10:16:09 +01:00
_EMAIL_LOG_ID_HEADER = "X-SimpleLogin-EmailLog-ID"
_ENVELOPE_FROM = "X-SimpleLogin-Envelope-From"
_ENVELOPE_TO = "X-SimpleLogin-Envelope-To"
2020-04-13 20:51:29 +02:00
_MIME_HEADERS = [
"MIME-Version",
"Content-Type",
"Content-Disposition",
"Content-Transfer-Encoding",
]
_MIME_HEADERS = [h.lower() for h in _MIME_HEADERS]
newrelic_app = None
if NEWRELIC_CONFIG_PATH:
newrelic.agent.initialize(NEWRELIC_CONFIG_PATH)
newrelic_app = newrelic.agent.register_application()
# fix the database connection leak issue
# use this method instead of create_app
def new_app():
2020-08-11 16:18:47 +02:00
app = create_light_app()
@app.teardown_appcontext
def shutdown_session(response_or_exc):
# same as shutdown_session() in flask-sqlalchemy but this is not enough
db.session.remove()
# dispose the engine too
db.engine.dispose()
return app
def get_or_create_contact(from_header: str, mail_from: str, alias: Alias) -> Contact:
"""
contact_from_header is the RFC 2047 format FROM header
"""
contact_name, contact_email = parseaddr_unicode(from_header)
if not is_valid_email(contact_email):
# From header is wrongly formatted, try with mail_from
if mail_from and mail_from != "<>":
2021-03-16 09:17:23 +01:00
LOG.w(
2020-11-25 15:20:00 +01:00
"Cannot parse email from from_header %s, parse from mail_from %s",
from_header,
2020-11-25 15:20:00 +01:00
mail_from,
)
_, contact_email = parseaddr_unicode(mail_from)
if not is_valid_email(contact_email):
2021-03-16 09:17:23 +01:00
LOG.w(
"invalid contact email %s. Parse from %s %s",
contact_email,
from_header,
mail_from,
)
# either reuse a contact with empty email or create a new contact with empty email
contact_email = ""
2021-01-11 12:27:02 +01:00
contact_email = sanitize_email(contact_email)
2020-04-04 20:06:35 +02:00
contact = Contact.get_by(alias_id=alias.id, website_email=contact_email)
2020-03-17 10:56:59 +01:00
if contact:
if contact.name != contact_name:
LOG.d(
2020-08-27 10:20:48 +02:00
"Update contact %s name %s to %s",
contact,
contact.name,
contact_name,
)
contact.name = contact_name
db.session.commit()
2020-11-25 15:20:00 +01:00
# contact created in the past does not have mail_from and from_header field
if not contact.mail_from and mail_from:
LOG.d(
2020-11-25 15:20:00 +01:00
"Set contact mail_from %s: %s to %s",
contact,
contact.mail_from,
mail_from,
)
contact.mail_from = mail_from
db.session.commit()
if not contact.from_header and from_header:
LOG.d(
2020-11-25 15:20:00 +01:00
"Set contact from_header %s: %s to %s",
contact,
contact.from_header,
from_header,
)
contact.from_header = from_header
db.session.commit()
else:
2020-11-25 15:20:00 +01:00
LOG.d(
"create contact %s for alias %s",
contact_email,
2020-08-27 10:20:48 +02:00
alias,
)
2020-09-03 19:42:52 +02:00
try:
contact = Contact.create(
user_id=alias.user_id,
alias_id=alias.id,
website_email=contact_email,
name=contact_name,
mail_from=mail_from,
from_header=from_header,
reply_email=generate_reply_email(contact_email, alias.user)
if is_valid_email(contact_email)
2020-11-18 11:48:09 +01:00
else NOREPLY,
2020-09-03 19:42:52 +02:00
)
2020-11-18 11:48:09 +01:00
if not contact_email:
LOG.d("Create a contact with invalid email for %s", alias)
contact.invalid_email = True
2020-09-03 19:42:52 +02:00
db.session.commit()
except IntegrityError:
2021-03-16 09:17:23 +01:00
LOG.w("Contact %s %s already exist", alias, contact_email)
2020-09-03 19:42:52 +02:00
db.session.rollback()
contact = Contact.get_by(alias_id=alias.id, website_email=contact_email)
2020-03-17 10:56:59 +01:00
return contact
def get_or_create_reply_to_contact(
reply_to_header: str, alias: Alias
) -> Optional[Contact]:
"""
Get or create the contact for the Reply-To header
"""
name, address = parseaddr_unicode(reply_to_header)
if not is_valid_email(address):
LOG.w(
"invalid reply-to address %s. Parse from %s",
address,
reply_to_header,
)
return None
address = sanitize_email(address)
contact = Contact.get_by(alias_id=alias.id, website_email=address)
if contact:
return contact
else:
LOG.d(
"create contact %s for alias %s via reply-to header",
address,
alias,
reply_to_header,
)
try:
contact = Contact.create(
user_id=alias.user_id,
alias_id=alias.id,
website_email=address,
name=name,
reply_email=generate_reply_email(address, alias.user),
)
db.session.commit()
except IntegrityError:
LOG.w("Contact %s %s already exist", alias, address)
db.session.rollback()
contact = Contact.get_by(alias_id=alias.id, website_email=address)
return contact
def replace_header_when_forward(msg: Message, alias: Alias, header: str):
"""
Replace CC or To header by Reply emails in forward phase
"""
new_addrs: [str] = []
headers = msg.get_all(header, [])
# headers can be an array of Header, convert it to string here
headers = [str(h) for h in headers]
for contact_name, contact_email in getaddresses(headers):
# convert back to original then parse again to make sure contact_name is unicode
addr = formataddr((contact_name, contact_email))
2021-01-11 12:27:02 +01:00
contact_name, _ = parseaddr_unicode(addr)
contact_email = sanitize_email(contact_email)
# no transformation when alias is already in the header
2020-04-05 12:59:36 +02:00
if contact_email == alias.email:
new_addrs.append(addr)
continue
if not is_valid_email(contact_email):
2021-03-16 09:17:23 +01:00
LOG.w("invalid contact email %s. %s. Skip", contact_email, headers)
continue
2020-04-05 12:59:36 +02:00
contact = Contact.get_by(alias_id=alias.id, website_email=contact_email)
if contact:
2020-04-05 15:39:48 +02:00
# update the contact name if needed
2020-04-05 14:50:12 +02:00
if contact.name != contact_name:
LOG.d(
"Update contact %s name %s to %s",
contact,
contact.name,
contact_name,
)
contact.name = contact_name
db.session.commit()
else:
LOG.debug(
"create contact for alias %s and email %s, header %s",
alias,
2020-04-05 12:59:36 +02:00
contact_email,
header,
)
try:
contact = Contact.create(
user_id=alias.user_id,
alias_id=alias.id,
website_email=contact_email,
name=contact_name,
reply_email=generate_reply_email(contact_email, alias.user),
is_cc=header.lower() == "cc",
from_header=addr,
)
db.session.commit()
except IntegrityError:
2021-03-16 09:17:23 +01:00
LOG.w("Contact %s %s already exist", alias, contact_email)
db.session.rollback()
contact = Contact.get_by(alias_id=alias.id, website_email=contact_email)
new_addrs.append(contact.new_addr())
if new_addrs:
new_header = ",".join(new_addrs)
LOG.d("Replace %s header, old: %s, new: %s", header, msg[header], new_header)
add_or_replace_header(msg, header, new_header)
else:
LOG.d("Delete %s header, old value %s", header, msg[header])
delete_header(msg, header)
def replace_header_when_reply(msg: Message, alias: Alias, header: str):
"""
Replace CC or To Reply emails by original emails
"""
new_addrs: [str] = []
headers = msg.get_all(header, [])
# headers can be an array of Header, convert it to string here
headers = [str(h) for h in headers]
for _, reply_email in getaddresses(headers):
# no transformation when alias is already in the header
if reply_email == alias.email:
continue
contact = Contact.get_by(reply_email=reply_email)
if not contact:
2021-03-16 09:17:23 +01:00
LOG.w(
"%s email in reply phase %s must be reply emails", header, reply_email
)
# still keep this email in header
new_addrs.append(reply_email)
else:
new_addrs.append(formataddr((contact.name, contact.website_email)))
if new_addrs:
new_header = ",".join(new_addrs)
LOG.d("Replace %s header, old: %s, new: %s", header, msg[header], new_header)
add_or_replace_header(msg, header, new_header)
else:
LOG.d("delete the %s header. Old value %s", header, msg[header])
delete_header(msg, header)
def prepare_pgp_message(
orig_msg: Message, pgp_fingerprint: str, public_key: str, can_sign: bool = False
) -> Message:
msg = MIMEMultipart("encrypted", protocol="application/pgp-encrypted")
# clone orig message to avoid modifying it
clone_msg = copy(orig_msg)
# copy all headers from original message except all standard MIME headers
for i in reversed(range(len(clone_msg._headers))):
header_name = clone_msg._headers[i][0].lower()
if header_name.lower() not in _MIME_HEADERS:
msg[header_name] = clone_msg._headers[i][1]
# Delete unnecessary headers in clone_msg except _MIME_HEADERS to save space
2020-03-14 22:24:02 +01:00
delete_all_headers_except(
clone_msg,
2020-08-27 10:20:48 +02:00
_MIME_HEADERS,
2020-03-14 22:24:02 +01:00
)
if clone_msg["Content-Type"] is None:
LOG.d("Content-Type missing")
clone_msg["Content-Type"] = "text/plain"
if clone_msg["Mime-Version"] is None:
LOG.d("Mime-Version missing")
clone_msg["Mime-Version"] = "1.0"
first = MIMEApplication(
_subtype="pgp-encrypted", _encoder=encoders.encode_7or8bit, _data=""
)
first.set_payload("Version: 1")
msg.attach(first)
if can_sign and PGP_SENDER_PRIVATE_KEY:
LOG.d("Sign msg")
clone_msg = sign_msg(clone_msg)
# use pgpy as fallback
second = MIMEApplication(
"octet-stream", _encoder=encoders.encode_7or8bit, name="encrypted.asc"
)
second.add_header("Content-Disposition", 'inline; filename="encrypted.asc"')
# encrypt
2020-10-28 17:07:53 +01:00
# use pgpy as fallback
2020-11-09 17:02:10 +01:00
msg_bytes = to_bytes(clone_msg)
2020-10-28 17:07:53 +01:00
try:
encrypted_data = pgp_utils.encrypt_file(BytesIO(msg_bytes), pgp_fingerprint)
2020-10-28 17:07:53 +01:00
second.set_payload(encrypted_data)
except PGPException:
2021-03-16 09:17:23 +01:00
LOG.w("Cannot encrypt using python-gnupg, use pgpy")
encrypted = pgp_utils.encrypt_file_with_pgpy(msg_bytes, public_key)
2020-10-28 17:07:53 +01:00
second.set_payload(str(encrypted))
msg.attach(second)
return msg
def sign_msg(msg: Message) -> Message:
container = MIMEMultipart(
"signed", protocol="application/pgp-signature", micalg="pgp-sha256"
)
container.attach(msg)
signature = MIMEApplication(
_subtype="pgp-signature", name="signature.asc", _data="", _encoder=encode_noop
)
signature.add_header("Content-Disposition", 'attachment; filename="signature.asc"')
try:
2020-11-09 17:02:10 +01:00
signature.set_payload(sign_data(to_bytes(msg).replace(b"\n", b"\r\n")))
except Exception:
LOG.exception("Cannot sign, try using pgpy")
signature.set_payload(
2020-11-09 17:02:10 +01:00
sign_data_with_pgpy(to_bytes(msg).replace(b"\n", b"\r\n"))
)
container.attach(signature)
return container
def handle_email_sent_to_ourself(alias, mailbox, msg: Message, user):
# store the refused email
random_name = str(uuid.uuid4())
full_report_path = f"refused-emails/cycle-{random_name}.eml"
2020-11-09 17:02:10 +01:00
s3.upload_email_from_bytesio(full_report_path, BytesIO(to_bytes(msg)), random_name)
refused_email = RefusedEmail.create(
path=None, full_report_path=full_report_path, user_id=alias.user_id
)
db.session.commit()
LOG.d("Create refused email %s", refused_email)
# link available for 6 days as it gets deleted in 7 days
refused_email_url = refused_email.get_url(expires_in=518400)
send_email_at_most_times(
user,
ALERT_SEND_EMAIL_CYCLE,
mailbox.email,
2020-08-27 11:10:16 +02:00
f"Email sent to {alias.email} from its own mailbox {mailbox.email}",
render(
"transactional/cycle-email.txt",
alias=alias,
mailbox=mailbox,
refused_email_url=refused_email_url,
),
render(
"transactional/cycle-email.html",
alias=alias,
mailbox=mailbox,
refused_email_url=refused_email_url,
),
)
def handle_forward(envelope, msg: Message, rcpt_to: str) -> List[Tuple[bool, str]]:
"""return an array of SMTP status (is_success, smtp_status)
is_success indicates whether an email has been delivered and
smtp_status is the SMTP Status ("250 Message accepted", "550 Non-existent email address", etc)
2020-03-28 21:24:43 +01:00
"""
address = rcpt_to # alias@SL
2020-03-19 11:15:02 +01:00
alias = Alias.get_by(email=address)
if not alias:
LOG.d("alias %s not exist. Try to see if it can be created on the fly", address)
2020-03-19 11:15:02 +01:00
alias = try_auto_create(address)
if not alias:
LOG.d("alias %s cannot be created on-the-fly, return 550", address)
if should_ignore_bounce(envelope.mail_from):
return [(True, status.E207)]
else:
return [(False, status.E515)]
2020-11-04 12:32:15 +01:00
user = alias.user
if user.disabled:
2021-03-16 09:17:23 +01:00
LOG.w("User %s disabled, disable forwarding emails for %s", user, alias)
if should_ignore_bounce(envelope.mail_from):
return [(True, status.E207)]
else:
return [(False, status.E504)]
# mail_from = envelope.mail_from
# for mb in alias.mailboxes:
# # email send from a mailbox to alias
# if mb.email == mail_from:
# LOG.w("cycle email sent from %s to %s", mb, alias)
# handle_email_sent_to_ourself(alias, mb, msg, user)
# return [(True, "250 Message accepted for delivery")]
from_header = str(msg["From"])
LOG.d("Create or get contact for from_header:%s", from_header)
contact = get_or_create_contact(from_header, envelope.mail_from, alias)
2020-11-04 12:32:15 +01:00
reply_to_contact = None
if msg["Reply-To"]:
reply_to = str(msg["Reply-To"])
LOG.d("Create or get contact for from_header:%s", reply_to)
# ignore when reply-to = alias
if reply_to == alias.email:
LOG.i("Reply-to same as alias %s", alias)
else:
reply_to_contact = get_or_create_reply_to_contact(reply_to, alias)
if not alias.enabled:
LOG.d("%s is disabled, do not forward", alias)
EmailLog.create(
contact_id=contact.id,
user_id=contact.user_id,
blocked=True,
alias_id=contact.alias_id,
commit=True,
)
db.session.commit()
# do not return 5** to allow user to receive emails later when alias is enabled
return [(True, status.E200)]
ret = []
mailboxes = alias.mailboxes
# no valid mailbox
if not mailboxes:
if should_ignore_bounce(envelope.mail_from):
return [(True, status.E207)]
else:
return [(False, status.E516)]
# no need to create a copy of message
2020-11-04 14:55:54 +01:00
for mailbox in mailboxes:
2020-09-10 09:38:30 +02:00
if not mailbox.verified:
2021-03-16 09:17:23 +01:00
LOG.d("%s unverified, do not forward", mailbox)
ret.append((False, status.E517))
2020-09-10 09:38:30 +02:00
else:
2020-11-04 14:55:54 +01:00
# create a copy of message for each forward
2020-09-10 09:38:30 +02:00
ret.append(
forward_email_to_mailbox(
alias, copy(msg), contact, envelope, mailbox, user, reply_to_contact
2020-09-10 09:38:30 +02:00
)
)
return ret
def forward_email_to_mailbox(
alias,
msg: Message,
contact: Contact,
envelope,
mailbox,
user,
reply_to_contact: Optional[Contact],
) -> (bool, str):
LOG.d("Forward %s -> %s -> %s", contact, alias, mailbox)
2020-06-09 17:16:32 +02:00
if mailbox.disabled:
LOG.d("%s disabled, do not forward")
if should_ignore_bounce(envelope.mail_from):
return True, status.E207
else:
return False, status.E518
2020-06-09 17:16:32 +02:00
# sanity check: make sure mailbox is not actually an alias
if get_email_domain_part(alias.email) == get_email_domain_part(mailbox.email):
2021-03-16 09:17:23 +01:00
LOG.w(
2020-06-09 17:16:32 +02:00
"Mailbox has the same domain as alias. %s -> %s -> %s",
contact,
alias,
mailbox,
)
mailbox_url = f"{URL}/dashboard/mailbox/{mailbox.id}/"
send_email_with_rate_control(
user,
ALERT_MAILBOX_IS_ALIAS,
user.email,
f"Your mailbox {mailbox.email} and alias {alias.email} use the same domain",
render(
"transactional/mailbox-invalid.txt",
mailbox=mailbox,
mailbox_url=mailbox_url,
2020-12-21 11:57:12 +01:00
alias=alias,
),
render(
"transactional/mailbox-invalid.html",
mailbox=mailbox,
mailbox_url=mailbox_url,
2020-12-21 11:57:12 +01:00
alias=alias,
),
max_nb_alert=1,
)
2020-08-30 19:08:53 +02:00
# retry later
# so when user fixes the mailbox, the email can be delivered
return False, status.E405
2020-06-09 17:16:32 +02:00
email_log = EmailLog.create(
contact_id=contact.id,
user_id=user.id,
mailbox_id=mailbox.id,
alias_id=contact.alias_id,
commit=True,
)
2021-05-24 12:08:30 +02:00
LOG.d("Create %s for %s, %s, %s", email_log, contact, user, mailbox)
if ENABLE_SPAM_ASSASSIN:
# Spam check
spam_status = ""
is_spam = False
2020-08-15 16:38:16 +02:00
if SPAMASSASSIN_HOST:
start = time.time()
spam_score, spam_report = get_spam_score(msg, email_log)
LOG.d(
"%s -> %s - spam score:%s in %s seconds. Spam report %s",
contact,
alias,
spam_score,
time.time() - start,
spam_report,
)
email_log.spam_score = spam_score
db.session.commit()
2020-08-20 11:58:46 +02:00
if (user.max_spam_score and spam_score > user.max_spam_score) or (
not user.max_spam_score and spam_score > MAX_SPAM_SCORE
):
is_spam = True
# only set the spam report for spam
email_log.spam_report = spam_report
else:
is_spam, spam_status = get_spam_info(msg, max_score=user.max_spam_score)
2020-08-15 16:38:16 +02:00
if is_spam:
LOG.w(
"Email detected as spam. %s -> %s. Spam Score: %s, Spam Report: %s",
contact,
alias,
email_log.spam_score,
email_log.spam_report,
)
email_log.is_spam = True
email_log.spam_status = spam_status
db.session.commit()
handle_spam(contact, alias, msg, user, mailbox, email_log)
return False, status.E519
if contact.invalid_email:
LOG.d("add noreply information %s %s", alias, mailbox)
msg = add_header(
msg,
f"""Email sent to {alias.email} from an invalid address and cannot be replied""",
f"""Email sent to {alias.email} from an invalid address and cannot be replied""",
)
2021-01-28 16:34:24 +01:00
delete_all_headers_except(
msg,
[
"From",
"To",
"Cc",
"Subject",
# References and In-Reply-To are used for keeping the email thread
"References",
"In-Reply-To",
]
+ _MIME_HEADERS,
)
# create PGP email if needed
2020-11-24 11:28:14 +01:00
if mailbox.pgp_enabled() and user.is_premium() and not alias.disable_pgp:
LOG.d("Encrypt message using mailbox %s", mailbox)
if mailbox.generic_subject:
LOG.d("Use a generic subject for %s", mailbox)
2020-11-07 17:23:28 +01:00
orig_subject = msg["Subject"]
2020-11-09 21:16:50 +01:00
orig_subject = get_header_unicode(orig_subject)
add_or_replace_header(msg, "Subject", mailbox.generic_subject)
msg = add_header(
msg,
2020-11-07 17:23:28 +01:00
f"""Forwarded by SimpleLogin to {alias.email} with "{orig_subject}" as subject""",
f"""Forwarded by SimpleLogin to {alias.email} with <b>{orig_subject}</b> as subject""",
)
2020-06-08 13:54:42 +02:00
try:
msg = prepare_pgp_message(
msg, mailbox.pgp_finger_print, mailbox.pgp_public_key, can_sign=True
)
2020-06-08 13:54:42 +02:00
except PGPException:
LOG.e(
2020-06-08 13:54:42 +02:00
"Cannot encrypt message %s -> %s. %s %s", contact, alias, mailbox, user
)
# so the client can retry later
return False, status.E406
# add custom header
add_or_replace_header(msg, _DIRECTION, "Forward")
2021-01-28 18:10:29 +01:00
msg[_EMAIL_LOG_ID_HEADER] = str(email_log.id)
msg[_ENVELOPE_FROM] = envelope.mail_from
# when an alias isn't in the To: header, there's no way for users to know what alias has received the email
msg[_ENVELOPE_TO] = alias.email
2021-03-29 10:27:19 +02:00
message_id = make_msgid(str(email_log.id), EMAIL_DOMAIN)
LOG.d("message id %s", message_id)
msg["Message-ID"] = message_id
2021-01-28 16:34:24 +01:00
msg["Date"] = formatdate()
2020-11-01 18:13:50 +01:00
# change the from header so the sender comes from a reverse-alias
# so it can pass DMARC check
# replace the email part in from: header
contact_from_header = msg["From"]
new_from_header = contact.new_addr()
add_or_replace_header(msg, "From", new_from_header)
2021-03-16 09:17:23 +01:00
LOG.d("From header, new:%s, old:%s", new_from_header, contact_from_header)
if reply_to_contact:
reply_to_header = msg["Reply-To"]
new_reply_to_header = reply_to_contact.new_addr()
add_or_replace_header(msg, "Reply-To", new_reply_to_header)
LOG.d("Reply-To header, new:%s, old:%s", new_reply_to_header, reply_to_header)
2020-11-01 18:13:50 +01:00
# replace CC & To emails by reverse-alias for all emails that are not alias
replace_header_when_forward(msg, alias, "Cc")
replace_header_when_forward(msg, alias, "To")
# add List-Unsubscribe header
unsubscribe_link, via_email = alias.unsubscribe_link()
add_or_replace_header(msg, "List-Unsubscribe", f"<{unsubscribe_link}>")
if not via_email:
add_or_replace_header(
msg, "List-Unsubscribe-Post", "List-Unsubscribe=One-Click"
)
add_dkim_signature(msg, EMAIL_DOMAIN)
LOG.d(
2021-03-16 09:17:23 +01:00
"Forward mail from %s to %s, mail_options:%s, rcpt_options:%s ",
contact.website_email,
mailbox.email,
envelope.mail_options,
envelope.rcpt_options,
)
try:
sl_sendmail(
# use a different envelope sender for each forward (aka VERP)
BOUNCE_EMAIL.format(email_log.id),
mailbox.email,
msg,
envelope.mail_options,
envelope.rcpt_options,
is_forward=True,
)
except SMTPRecipientsRefused:
# that means the mailbox is maybe invalid
2021-03-16 09:17:23 +01:00
LOG.w(
"SMTPRecipientsRefused forward phase %s -> %s -> %s",
contact,
alias,
mailbox,
)
if should_ignore_bounce(envelope.mail_from):
return True, status.E207
else:
return False, status.E521
else:
db.session.commit()
return True, status.E200
def handle_reply(envelope, msg: Message, rcpt_to: str) -> (bool, str):
2020-03-28 21:24:43 +01:00
"""
2021-01-28 16:34:24 +01:00
Return whether an email has been delivered and
2020-03-28 21:24:43 +01:00
the smtp status ("250 Message accepted", "550 Non-existent email address", etc)
"""
reply_email = rcpt_to
# reply_email must end with EMAIL_DOMAIN
if not reply_email.endswith(EMAIL_DOMAIN):
2021-03-16 09:17:23 +01:00
LOG.w(f"Reply email {reply_email} has wrong domain")
return False, status.E501
# handle case where reply email is generated with non-allowed char
reply_email = normalize_reply_email(reply_email)
2020-03-17 10:56:59 +01:00
contact = Contact.get_by(reply_email=reply_email)
if not contact:
2021-03-16 09:17:23 +01:00
LOG.w(f"No such forward-email with {reply_email} as reply-email")
return False, status.E502
2019-12-18 17:07:20 +01:00
alias = contact.alias
address: str = contact.alias.email
alias_domain = address[address.find("@") + 1 :]
2020-10-15 16:21:31 +02:00
# Sanity check: verify alias domain is managed by SimpleLogin
# scenario: a user have removed a domain but due to a bug, the aliases are still there
if not is_valid_alias_address_domain(alias.email):
LOG.exception("%s domain isn't known", alias)
return False, status.E503
user = alias.user
mail_from = envelope.mail_from
if user.disabled:
LOG.exception(
"User %s disabled, disable sending emails from %s to %s",
user,
alias,
contact,
)
return [(False, status.E504)]
# Anti-spoofing
mailbox = get_mailbox_from_mail_from(mail_from, alias)
if not mailbox:
if alias.disable_email_spoofing_check:
# ignore this error, use default alias mailbox
2021-03-16 09:17:23 +01:00
LOG.w(
"ignore unknown sender to reverse-alias %s: %s -> %s",
mail_from,
alias,
contact,
)
mailbox = alias.mailbox
else:
# only mailbox can send email to the reply-email
handle_unknown_mailbox(envelope, msg, reply_email, user, alias, contact)
return False, status.E505
if ENFORCE_SPF and mailbox.force_spf and not alias.disable_email_spoofing_check:
2021-03-17 10:16:09 +01:00
if not spf_pass(envelope, mailbox, user, alias, contact.website_email, msg):
# cannot use 4** here as sender will retry.
# cannot use 5** because that generates bounce report
return True, status.E201
2020-05-07 13:28:04 +02:00
2020-08-16 14:28:47 +02:00
email_log = EmailLog.create(
contact_id=contact.id,
alias_id=contact.alias_id,
is_reply=True,
user_id=contact.user_id,
mailbox_id=mailbox.id,
commit=True,
2020-08-16 14:28:47 +02:00
)
2021-05-24 12:08:30 +02:00
LOG.d("Create %s for %s, %s, %s", email_log, contact, user, mailbox)
2020-08-16 14:28:47 +02:00
2020-08-15 16:53:57 +02:00
# Spam check
if ENABLE_SPAM_ASSASSIN:
spam_status = ""
is_spam = False
# do not use user.max_spam_score here
if SPAMASSASSIN_HOST:
start = time.time()
spam_score, spam_report = get_spam_score(msg, email_log)
LOG.d(
"%s -> %s - spam score %s in %s seconds. Spam report %s",
alias,
contact,
spam_score,
time.time() - start,
spam_report,
)
email_log.spam_score = spam_score
if spam_score > MAX_REPLY_PHASE_SPAM_SCORE:
is_spam = True
# only set the spam report for spam
email_log.spam_report = spam_report
else:
is_spam, spam_status = get_spam_info(
msg, max_score=MAX_REPLY_PHASE_SPAM_SCORE
)
2020-08-15 16:53:57 +02:00
if is_spam:
LOG.w(
"Email detected as spam. Reply phase. %s -> %s. Spam Score: %s, Spam Report: %s",
alias,
contact,
email_log.spam_score,
email_log.spam_report,
)
2020-08-16 14:28:47 +02:00
email_log.is_spam = True
email_log.spam_status = spam_status
db.session.commit()
2020-08-15 16:53:57 +02:00
handle_spam(contact, alias, msg, user, mailbox, email_log, is_reply=True)
return False, status.E506
2020-08-15 16:53:57 +02:00
delete_all_headers_except(
msg,
[
"From",
"To",
"Cc",
"Subject",
# References and In-Reply-To are used for keeping the email thread
"References",
"In-Reply-To",
]
+ _MIME_HEADERS,
)
2020-11-01 18:12:09 +01:00
# replace the reverse-alias (i.e. "ra+string@simplelogin.co") by the contact email in the email body
# as this is usually included when replying
if user.replace_reverse_alias:
LOG.d("Replace reverse-alias %s by contact email %s", reply_email, contact)
msg = replace(msg, reply_email, contact.website_email)
# create PGP email if needed
if contact.pgp_finger_print and user.is_premium():
LOG.d("Encrypt message for contact %s", contact)
try:
msg = prepare_pgp_message(
msg, contact.pgp_finger_print, contact.pgp_public_key
)
except PGPException:
2021-06-23 19:55:41 +02:00
LOG.e(
"Cannot encrypt message %s -> %s. %s %s", alias, contact, mailbox, user
)
# to not save the email_log
EmailLog.delete(email_log.id)
db.session.commit()
# return 421 so the client can retry later
return False, status.E402
2020-11-04 12:32:15 +01:00
db.session.commit()
# make the email comes from alias
from_header = alias.email
# add alias name from alias
2020-04-26 10:41:24 +02:00
if alias.name:
LOG.d("Put alias name in from header")
from_header = formataddr((alias.name, alias.email))
elif alias.custom_domain:
LOG.d("Put domain default alias name in from header")
# add alias name from domain
if alias.custom_domain.name:
from_header = formataddr((alias.custom_domain.name, alias.email))
2020-04-26 10:41:24 +02:00
add_or_replace_header(msg, "From", from_header)
replace_header_when_reply(msg, alias, "To")
replace_header_when_reply(msg, alias, "Cc")
# Message-ID can reveal about the mailbox -> replace it
2021-03-29 10:27:19 +02:00
message_id = make_msgid(str(email_log.id), get_email_domain_part(alias.email))
LOG.d("make message id %s", message_id)
2020-08-25 12:45:55 +02:00
add_or_replace_header(
msg,
2020-12-11 11:02:52 +01:00
"Message-ID",
2021-03-29 10:27:19 +02:00
message_id,
2020-08-25 12:45:55 +02:00
)
date_header = formatdate()
msg["Date"] = date_header
2020-11-04 12:32:15 +01:00
msg[_DIRECTION] = "Reply"
msg[_EMAIL_LOG_ID_HEADER] = str(email_log.id)
LOG.d(
"send email from %s to %s, mail_options:%s,rcpt_options:%s",
alias.email,
2020-03-17 10:56:59 +01:00
contact.website_email,
envelope.mail_options,
envelope.rcpt_options,
)
if should_add_dkim_signature(alias_domain):
add_dkim_signature(msg, alias_domain)
# generate a mail_from for VERP
verp_mail_from = f"{BOUNCE_PREFIX_FOR_REPLY_PHASE}+{email_log.id}+@{alias_domain}"
2020-06-20 16:19:01 +02:00
try:
sl_sendmail(
verp_mail_from,
2020-06-20 16:19:01 +02:00
contact.website_email,
msg,
2020-06-20 16:19:01 +02:00
envelope.mail_options,
envelope.rcpt_options,
is_forward=False,
2020-06-20 16:19:01 +02:00
)
except Exception:
2020-08-16 14:28:47 +02:00
# to not save the email_log
db.session.rollback()
2021-03-16 09:17:23 +01:00
LOG.w("Cannot send email from %s to %s", alias, contact)
2020-06-20 16:19:01 +02:00
send_email(
mailbox.email,
f"Email cannot be sent to {contact.email} from {alias.email}",
render(
"transactional/reply-error.txt",
user=user,
alias=alias,
contact=contact,
contact_domain=get_email_domain_part(contact.email),
),
render(
"transactional/reply-error.html",
user=user,
alias=alias,
contact=contact,
contact_domain=get_email_domain_part(contact.email),
),
)
# return 250 even if error as user is already informed of the incident and can retry sending the email
return True, status.E200
2020-09-28 17:41:16 +02:00
def get_mailbox_from_mail_from(mail_from: str, alias) -> Optional[Mailbox]:
"""return the corresponding mailbox given the mail_from and alias
Usually the mail_from=mailbox.email but it can also be one of the authorized address
"""
for mailbox in alias.mailboxes:
if mailbox.email == mail_from:
return mailbox
for address in mailbox.authorized_addresses:
if address.email == mail_from:
LOG.debug(
"Found an authorized address for %s %s %s", alias, mailbox, address
)
return mailbox
return None
def handle_unknown_mailbox(
envelope, msg, reply_email: str, user: User, alias: Alias, contact: Contact
):
2021-03-16 09:17:23 +01:00
LOG.w(
2020-12-06 13:54:59 +01:00
"Reply email can only be used by mailbox. "
"Actual mail_from: %s. msg from header: %s, reverse-alias %s, %s %s %s",
envelope.mail_from,
msg["From"],
reply_email,
alias,
user,
contact,
)
authorize_address_link = (
f"{URL}/dashboard/mailbox/{alias.mailbox_id}/#authorized-address"
)
2020-09-29 13:03:15 +02:00
mailbox_emails = [mailbox.email for mailbox in alias.mailboxes]
send_email_with_rate_control(
user,
ALERT_REVERSE_ALIAS_UNKNOWN_MAILBOX,
user.email,
f"Attempt to use your alias {alias.email} from {envelope.mail_from}",
render(
"transactional/reply-must-use-personal-email.txt",
alias=alias,
sender=envelope.mail_from,
authorize_address_link=authorize_address_link,
2020-09-29 13:11:04 +02:00
mailbox_emails=mailbox_emails,
),
render(
"transactional/reply-must-use-personal-email.html",
alias=alias,
sender=envelope.mail_from,
authorize_address_link=authorize_address_link,
2020-09-29 13:11:04 +02:00
mailbox_emails=mailbox_emails,
),
)
# Notify sender that they cannot send emails to this address
send_email_with_rate_control(
user,
ALERT_REVERSE_ALIAS_UNKNOWN_MAILBOX,
envelope.mail_from,
f"Your email ({envelope.mail_from}) is not allowed to send emails to {reply_email}",
render(
"transactional/send-from-alias-from-unknown-sender.txt",
sender=envelope.mail_from,
reply_email=reply_email,
),
render(
"transactional/send-from-alias-from-unknown-sender.html",
sender=envelope.mail_from,
reply_email=reply_email,
),
)
def handle_bounce_forward_phase(msg: Message, email_log: EmailLog):
2020-11-04 12:32:15 +01:00
"""
Handle forward phase bounce
Happens when an email cannot be sent to a mailbox
2020-11-04 12:32:15 +01:00
"""
contact = email_log.contact
alias = contact.alias
2021-01-04 14:22:07 +01:00
user = alias.user
2021-01-11 15:53:08 +01:00
mailbox = email_log.mailbox
# email_log.mailbox should be set during the forward phase
if not mailbox:
LOG.exception("Use %s default mailbox %s", alias, alias.mailbox)
mailbox = alias.mailbox
2021-01-26 09:56:13 +01:00
Bounce.create(email=mailbox.email, commit=True)
2021-01-11 15:53:08 +01:00
LOG.debug(
"Handle forward bounce %s -> %s -> %s. %s", contact, alias, mailbox, email_log
)
2021-01-04 14:22:07 +01:00
# Store the bounced email, generate a name for the email
random_name = str(uuid.uuid4())
2020-11-04 15:38:26 +01:00
full_report_path = f"refused-emails/full-{random_name}.eml"
s3.upload_email_from_bytesio(
full_report_path, BytesIO(to_bytes(msg)), f"full-{random_name}"
)
file_path = None
orig_msg = get_orig_message_from_bounce(msg)
if not orig_msg:
# Some MTA does not return the original message in bounce message
# nothing we can do here
2021-03-16 09:17:23 +01:00
LOG.w(
"Cannot parse original message from bounce message %s %s %s %s",
2020-11-04 15:38:26 +01:00
alias,
user,
contact,
full_report_path,
)
else:
file_path = f"refused-emails/{random_name}.eml"
s3.upload_email_from_bytesio(
file_path, BytesIO(to_bytes(orig_msg)), random_name
)
refused_email = RefusedEmail.create(
path=file_path, full_report_path=full_report_path, user_id=user.id
)
db.session.flush()
LOG.d("Create refused email %s", refused_email)
email_log.bounced = True
email_log.refused_email_id = refused_email.id
email_log.bounced_mailbox_id = mailbox.id
db.session.commit()
refused_email_url = f"{URL}/dashboard/refused_email?highlight_id={email_log.id}"
# inform user of this bounce
if not should_disable(alias):
LOG.d(
"Inform user %s about a bounce from contact %s to alias %s",
user,
contact,
alias,
)
disable_alias_link = f"{URL}/dashboard/unsubscribe/{alias.id}"
send_email_with_rate_control(
user,
ALERT_BOUNCE_EMAIL,
user.email,
f"Email from {contact.website_email} to {alias.email} cannot be delivered to your mailbox",
render(
"transactional/bounce/bounced-email.txt",
alias=alias,
website_email=contact.website_email,
disable_alias_link=disable_alias_link,
refused_email_url=refused_email_url,
mailbox_email=mailbox.email,
),
render(
"transactional/bounce/bounced-email.html",
alias=alias,
website_email=contact.website_email,
disable_alias_link=disable_alias_link,
refused_email_url=refused_email_url,
mailbox_email=mailbox.email,
),
max_nb_alert=10,
)
else:
2021-03-16 09:17:23 +01:00
LOG.w(
"Disable alias %s now",
alias,
)
alias.enabled = False
db.session.commit()
send_email_with_rate_control(
user,
ALERT_BOUNCE_EMAIL,
user.email,
2021-01-19 10:36:11 +01:00
f"Alias {alias.email} has been disabled due to multiple bounces",
render(
"transactional/bounce/automatic-disable-alias.txt",
alias=alias,
refused_email_url=refused_email_url,
mailbox_email=mailbox.email,
),
render(
"transactional/bounce/automatic-disable-alias.html",
alias=alias,
refused_email_url=refused_email_url,
mailbox_email=mailbox.email,
),
max_nb_alert=10,
2020-11-04 15:38:26 +01:00
)
2021-09-06 19:44:18 +02:00
def handle_hotmail_complaint(msg: Message):
"""
Handle hotmail complaint sent to postmaster
"""
orig_msg = get_orig_message_from_outlook_complaint(msg)
to_header = orig_msg["To"]
_, alias_address = parseaddr_unicode(to_header)
2021-09-06 19:44:18 +02:00
alias = Alias.get_by(email=alias_address)
if not alias:
LOG.d("No alias for %s", alias_address)
return
2021-09-06 19:44:18 +02:00
user = alias.user
LOG.e("Handle hotmail complaint for %s %s", alias, user)
send_email_with_rate_control(
user,
ALERT_HOTMAIL_COMPLAINT,
user.email,
f"Hotmail abuse report",
render(
"transactional/hotmail-complaint.txt.jinja2",
alias=alias,
),
render(
"transactional/hotmail-complaint.html",
alias=alias,
),
max_nb_alert=2,
)
2021-03-06 17:44:31 +01:00
def handle_bounce_reply_phase(envelope, msg: Message, email_log: EmailLog):
"""
Handle reply phase bounce
Happens when an email cannot be sent from an alias to a contact
"""
2021-01-26 09:56:13 +01:00
contact: Contact = email_log.contact
alias = contact.alias
user = alias.user
2021-01-11 15:53:08 +01:00
mailbox = email_log.mailbox or alias.mailbox
LOG.debug(
"Handle reply bounce %s -> %s -> %s.%s", mailbox, alias, contact, email_log
)
Bounce.create(email=sanitize_email(contact.website_email), commit=True)
2021-01-26 09:56:13 +01:00
2020-11-04 12:32:15 +01:00
# Store the bounced email
# generate a name for the email
random_name = str(uuid.uuid4())
full_report_path = f"refused-emails/full-{random_name}.eml"
2020-11-09 17:02:10 +01:00
s3.upload_email_from_bytesio(full_report_path, BytesIO(to_bytes(msg)), random_name)
2020-11-04 12:32:15 +01:00
orig_msg = get_orig_message_from_bounce(msg)
2020-11-04 15:38:26 +01:00
file_path = None
if orig_msg:
file_path = f"refused-emails/{random_name}.eml"
s3.upload_email_from_bytesio(
2020-11-09 17:02:10 +01:00
file_path, BytesIO(to_bytes(orig_msg)), random_name
2020-11-04 15:38:26 +01:00
)
2020-11-04 12:32:15 +01:00
2020-11-04 15:38:26 +01:00
refused_email = RefusedEmail.create(
path=file_path, full_report_path=full_report_path, user_id=user.id, commit=True
)
LOG.d("Create refused email %s", refused_email)
2020-11-04 12:32:15 +01:00
2020-11-04 15:38:26 +01:00
email_log.bounced = True
email_log.refused_email_id = refused_email.id
2020-11-04 12:32:15 +01:00
2020-11-24 16:50:55 +01:00
email_log.bounced_mailbox_id = mailbox.id
db.session.commit()
2020-11-04 12:32:15 +01:00
2020-12-06 13:54:59 +01:00
refused_email_url = f"{URL}/dashboard/refused_email?highlight_id={email_log.id}"
2020-11-04 12:32:15 +01:00
LOG.d(
"Inform user %s about bounced email sent by %s to %s",
user,
alias,
contact,
)
send_email_with_rate_control(
user,
ALERT_BOUNCE_EMAIL_REPLY_PHASE,
mailbox.email,
f"Email cannot be sent to { contact.email } from your alias { alias.email }",
render(
2021-01-04 14:22:07 +01:00
"transactional/bounce/bounce-email-reply-phase.txt",
2020-11-04 12:32:15 +01:00
alias=alias,
contact=contact,
refused_email_url=refused_email_url,
),
render(
2021-01-04 14:22:07 +01:00
"transactional/bounce/bounce-email-reply-phase.html",
2020-11-04 12:32:15 +01:00
alias=alias,
contact=contact,
refused_email_url=refused_email_url,
),
)
return status.E520
2020-11-04 12:32:15 +01:00
def handle_spam(
contact: Contact,
alias: Alias,
msg: Message,
user: User,
2020-08-21 10:18:58 +02:00
mailbox: Mailbox,
email_log: EmailLog,
2020-08-15 16:53:57 +02:00
is_reply=False, # whether the email is in forward or reply phase
):
# Store the report & original email
orig_msg = get_orig_message_from_spamassassin_report(msg)
# generate a name for the email
random_name = str(uuid.uuid4())
2020-04-02 18:09:05 +02:00
full_report_path = f"spams/full-{random_name}.eml"
2020-11-09 17:02:10 +01:00
s3.upload_email_from_bytesio(full_report_path, BytesIO(to_bytes(msg)), random_name)
file_path = None
if orig_msg:
2020-04-02 18:09:05 +02:00
file_path = f"spams/{random_name}.eml"
s3.upload_email_from_bytesio(
2020-11-09 17:02:10 +01:00
file_path, BytesIO(to_bytes(orig_msg)), random_name
)
refused_email = RefusedEmail.create(
path=file_path, full_report_path=full_report_path, user_id=user.id
)
db.session.flush()
email_log.refused_email_id = refused_email.id
db.session.commit()
LOG.d("Create spam email %s", refused_email)
2020-12-06 13:54:59 +01:00
refused_email_url = f"{URL}/dashboard/refused_email?highlight_id={email_log.id}"
disable_alias_link = f"{URL}/dashboard/unsubscribe/{alias.id}"
2020-08-15 16:53:57 +02:00
if is_reply:
LOG.d(
2020-12-02 12:40:29 +01:00
"Inform %s (%s) about spam email sent from alias %s to %s. %s",
2020-08-21 10:20:08 +02:00
mailbox,
2020-08-15 16:53:57 +02:00
user,
alias,
contact,
2020-12-02 12:40:29 +01:00
refused_email,
2020-08-15 16:53:57 +02:00
)
send_email_with_rate_control(
user,
ALERT_SPAM_EMAIL,
2020-08-21 10:18:58 +02:00
mailbox.email,
2020-11-18 16:16:37 +01:00
f"Email from {alias.email} to {contact.website_email} is detected as spam",
2020-08-15 16:53:57 +02:00
render(
"transactional/spam-email-reply-phase.txt",
alias=alias,
website_email=contact.website_email,
disable_alias_link=disable_alias_link,
refused_email_url=refused_email_url,
),
render(
"transactional/spam-email-reply-phase.html",
alias=alias,
website_email=contact.website_email,
disable_alias_link=disable_alias_link,
refused_email_url=refused_email_url,
),
)
else:
# inform user
LOG.d(
2020-08-21 10:20:08 +02:00
"Inform %s (%s) about spam email sent by %s to alias %s",
mailbox,
2020-08-15 16:53:57 +02:00
user,
contact,
alias,
)
send_email_with_rate_control(
user,
ALERT_SPAM_EMAIL,
2020-08-21 10:18:58 +02:00
mailbox.email,
2020-08-15 16:53:57 +02:00
f"Email from {contact.website_email} to {alias.email} is detected as spam",
render(
"transactional/spam-email.txt",
alias=alias,
website_email=contact.website_email,
disable_alias_link=disable_alias_link,
refused_email_url=refused_email_url,
),
render(
"transactional/spam-email.html",
alias=alias,
website_email=contact.website_email,
disable_alias_link=disable_alias_link,
refused_email_url=refused_email_url,
),
)
2021-03-17 10:20:10 +01:00
def handle_unsubscribe(envelope: Envelope, msg: Message) -> str:
"""return the SMTP status"""
2020-03-28 23:19:25 +01:00
# format: alias_id:
subject = msg["Subject"]
try:
# subject has the format {alias.id}=
if subject.endswith("="):
alias_id = int(subject[:-1])
# {user.id}*
elif subject.endswith("*"):
user_id = int(subject[:-1])
return handle_unsubscribe_user(user_id, envelope.mail_from)
# some email providers might strip off the = suffix
else:
alias_id = int(subject)
2020-03-28 23:19:25 +01:00
alias = Alias.get(alias_id)
except Exception:
2021-03-16 09:17:23 +01:00
LOG.w("Cannot parse alias from subject %s", msg["Subject"])
return status.E507
2020-03-28 23:19:25 +01:00
if not alias:
2021-03-16 09:17:23 +01:00
LOG.w("No such alias %s", alias_id)
return status.E508
2020-03-28 23:19:25 +01:00
# This sender cannot unsubscribe
mail_from = envelope.mail_from
# Only alias's owning mailbox can send the unsubscribe request
mailbox = get_mailbox_from_mail_from(mail_from, alias)
if not mailbox:
2020-03-28 23:19:25 +01:00
LOG.d("%s cannot disable alias %s", envelope.mail_from, alias)
return status.E509
2020-03-28 23:19:25 +01:00
# Sender is owner of this alias
alias.enabled = False
db.session.commit()
user = alias.user
enable_alias_url = URL + f"/dashboard/?highlight_alias_id={alias.id}"
2020-05-10 18:23:43 +02:00
for mailbox in alias.mailboxes:
send_email(
mailbox.email,
f"Alias {alias.email} has been disabled successfully",
render(
"transactional/unsubscribe-disable-alias.txt",
user=user,
alias=alias.email,
enable_alias_url=enable_alias_url,
),
render(
"transactional/unsubscribe-disable-alias.html",
user=user,
alias=alias.email,
enable_alias_url=enable_alias_url,
),
)
2020-03-28 23:19:25 +01:00
return status.E202
2020-03-28 23:19:25 +01:00
def handle_unsubscribe_user(user_id: int, mail_from: str) -> str:
"""return the SMTP status"""
user = User.get(user_id)
if not user:
LOG.exception("No such user %s %s", user_id, mail_from)
return status.E510
if mail_from != user.email:
LOG.exception("Unauthorized mail_from %s %s", user, mail_from)
return status.E511
user.notification = False
db.session.commit()
send_email(
user.email,
2020-12-06 13:54:59 +01:00
"You have been unsubscribed from SimpleLogin newsletter",
render(
"transactional/unsubscribe-newsletter.txt",
user=user,
),
render(
"transactional/unsubscribe-newsletter.html",
user=user,
),
)
return status.E202
2020-10-22 12:26:45 +02:00
def handle_transactional_bounce(envelope: Envelope, rcpt_to):
LOG.d("handle transactional bounce sent to %s", rcpt_to)
2020-06-10 13:57:23 +02:00
# parse the TransactionalEmail
transactional_id = parse_id_from_bounce(rcpt_to)
transactional = TransactionalEmail.get(transactional_id)
2020-06-10 13:57:23 +02:00
# a transaction might have been deleted in delete_logs()
if transactional:
2021-03-16 09:17:23 +01:00
LOG.i("Create bounce for %s", transactional.email)
Bounce.create(email=transactional.email, commit=True)
2020-06-10 13:57:23 +02:00
2021-06-02 11:38:25 +02:00
def handle_bounce(envelope, email_log: EmailLog, msg: Message) -> str:
2021-03-17 10:09:54 +01:00
"""
Return SMTP status, e.g. "500 Error"
"""
if not email_log:
2021-06-02 11:38:25 +02:00
LOG.w("No such email log")
return status.E512
2021-03-17 10:09:54 +01:00
2021-06-02 11:38:25 +02:00
contact: Contact = email_log.contact
alias = contact.alias
LOG.d(
"handle bounce for %s, phase=%s, contact=%s, alias=%s",
email_log,
email_log.get_phase(),
contact,
alias,
)
2021-03-17 10:09:54 +01:00
if email_log.is_reply:
content_type = msg.get_content_type().lower()
if content_type != "multipart/report" or envelope.mail_from != "<>":
# forward the email again to the alias
2021-05-30 20:02:41 +02:00
LOG.i(
"Handle auto reply %s %s",
2021-03-17 10:09:54 +01:00
content_type,
envelope.mail_from,
)
contact: Contact = email_log.contact
alias = contact.alias
email_log.auto_replied = True
db.session.commit()
# replace the BOUNCE_EMAIL by alias in To field
add_or_replace_header(msg, "To", alias.email)
envelope.rcpt_tos = [alias.email]
# same as handle()
# result of all deliveries
# each element is a couple of whether the delivery is successful and the smtp status
res: [(bool, str)] = []
for is_delivered, smtp_status in handle_forward(envelope, msg, alias.email):
res.append((is_delivered, smtp_status))
for (is_success, smtp_status) in res:
# Consider all deliveries successful if 1 delivery is successful
if is_success:
return smtp_status
# Failed delivery for all, return the first failure
return res[0][1]
return handle_bounce_reply_phase(envelope, msg, email_log)
else: # forward phase
handle_bounce_forward_phase(msg, email_log)
return status.E513
2021-03-17 10:09:54 +01:00
def should_ignore(mail_from: str, rcpt_tos: List[str]) -> bool:
if len(rcpt_tos) != 1:
return False
rcpt_to = rcpt_tos[0]
if IgnoredEmail.get_by(mail_from=mail_from, rcpt_to=rcpt_to):
return True
return False
def handle(envelope: Envelope) -> str:
"""Return SMTP status"""
# sanitize mail_from, rcpt_tos
2021-01-11 12:27:02 +01:00
mail_from = sanitize_email(envelope.mail_from)
rcpt_tos = [sanitize_email(rcpt_to) for rcpt_to in envelope.rcpt_tos]
envelope.mail_from = mail_from
envelope.rcpt_tos = rcpt_tos
2021-03-16 09:17:23 +01:00
msg = email.message_from_bytes(envelope.original_content)
2021-06-04 17:15:59 +02:00
postfix_queue_id = get_queue_id(msg)
if postfix_queue_id:
set_message_id(postfix_queue_id)
else:
LOG.w("Cannot parse Postfix queue ID from %s", msg["Received"])
if should_ignore(mail_from, rcpt_tos):
2021-06-23 19:47:51 +02:00
LOG.w("Ignore email mail_from=%s rcpt_to=%s", mail_from, rcpt_tos)
return status.E204
2021-03-17 10:19:27 +01:00
# sanitize email headers
2021-03-17 10:59:13 +01:00
sanitize_header(msg, "from")
sanitize_header(msg, "to")
sanitize_header(msg, "cc")
sanitize_header(msg, "reply-to")
2021-03-17 10:19:27 +01:00
2021-03-16 09:17:23 +01:00
LOG.d(
"==>> Handle mail_from:%s, rcpt_tos:%s, header_from:%s, header_to:%s, "
"cc:%s, reply-to:%s, mail_options:%s, rcpt_options:%s",
mail_from,
rcpt_tos,
msg["From"],
msg["To"],
msg["Cc"],
msg["Reply-To"],
envelope.mail_options,
envelope.rcpt_options,
)
2021-03-15 19:55:22 +01:00
contact = Contact.get_by(reply_email=mail_from)
if contact:
2021-03-16 09:17:23 +01:00
LOG.e(
2021-03-15 19:55:22 +01:00
"email can't be sent from a reverse-alias alias:%s, contact email:%s",
contact.alias,
contact.website_email,
)
return status.E203
2021-03-15 19:55:22 +01:00
# unsubscribe request
if UNSUBSCRIBER and rcpt_tos == [UNSUBSCRIBER]:
LOG.d("Handle unsubscribe request from %s", mail_from)
2021-03-17 10:20:10 +01:00
return handle_unsubscribe(envelope, msg)
2020-06-10 13:57:23 +02:00
# emails sent to sender. Probably bounce emails
if (
len(rcpt_tos) == 1
and rcpt_tos[0].startswith(TRANSACTIONAL_BOUNCE_PREFIX)
and rcpt_tos[0].endswith(TRANSACTIONAL_BOUNCE_SUFFIX)
):
LOG.d("Handle email sent to sender from %s", mail_from)
handle_transactional_bounce(envelope, rcpt_tos[0])
return status.E205
2020-06-10 13:57:23 +02:00
2021-09-06 19:44:18 +02:00
if (
len(rcpt_tos) == 1
and mail_from == "staff@hotmail.com"
and rcpt_tos[0] == POSTMASTER
):
LOG.w("Handle hotmail complaint")
handle_hotmail_complaint(msg)
return status.E208
2021-06-02 11:38:25 +02:00
# Handle bounce
if (
len(rcpt_tos) == 1
and rcpt_tos[0].startswith(BOUNCE_PREFIX)
and rcpt_tos[0].endswith(BOUNCE_SUFFIX)
):
2021-06-02 11:38:25 +02:00
email_log_id = parse_id_from_bounce(rcpt_tos[0])
email_log = EmailLog.get(email_log_id)
return handle_bounce(envelope, email_log, msg)
if len(rcpt_tos) == 1 and rcpt_tos[0].startswith(
f"{BOUNCE_PREFIX_FOR_REPLY_PHASE}+"
):
2021-06-02 11:38:25 +02:00
email_log_id = parse_id_from_bounce(rcpt_tos[0])
email_log = EmailLog.get(email_log_id)
return handle_bounce(envelope, email_log, msg)
2021-06-02 11:46:00 +02:00
# iCloud returns the bounce with mail_from=bounce+{email_log_id}+@simplelogin.co, rcpt_to=alias
2021-06-02 16:27:48 +02:00
if (
len(rcpt_tos) == 1
and mail_from.startswith(BOUNCE_PREFIX)
and mail_from.endswith(BOUNCE_SUFFIX)
):
2021-06-02 11:46:00 +02:00
email_log_id = parse_id_from_bounce(mail_from)
email_log = EmailLog.get(email_log_id)
alias = Alias.get_by(email=rcpt_tos[0])
LOG.w(
2021-06-02 11:46:00 +02:00
"iCloud bounces %s %s msg=%s",
email_log,
alias,
msg.as_string(),
)
return handle_bounce(envelope, email_log, msg)
if rate_limited(mail_from, rcpt_tos):
LOG.w("Rate Limiting applied for mail_from:%s rcpt_tos:%s", mail_from, rcpt_tos)
if should_ignore_bounce(envelope.mail_from):
return status.E207
else:
return status.E522
# Handle "out of office" auto notice. An automatic response is sent for every forwarded email
# todo: remove logging
if len(rcpt_tos) == 1 and is_reply_email(rcpt_tos[0]) and mail_from == "<>":
2021-05-30 20:02:41 +02:00
LOG.w(
"out-of-office email to reverse alias %s. %s", rcpt_tos[0], msg.as_string()
)
return status.E206
# result of all deliveries
# each element is a couple of whether the delivery is successful and the smtp status
res: [(bool, str)] = []
nb_rcpt_tos = len(rcpt_tos)
for rcpt_index, rcpt_to in enumerate(rcpt_tos):
if rcpt_to == NOREPLY:
2021-03-16 09:17:23 +01:00
LOG.e("email sent to noreply address from %s", mail_from)
return status.E514
# create a copy of msg for each recipient except the last one
# as copy() is a slow function
if rcpt_index < nb_rcpt_tos - 1:
LOG.d("copy message for rcpt %s", rcpt_to)
copy_msg = copy(msg)
else:
copy_msg = msg
# Reply case
# recipient starts with "reply+" or "ra+" (ra=reverse-alias) prefix
2020-11-16 19:22:19 +01:00
if is_reply_email(rcpt_to):
LOG.d("Reply phase %s(%s) -> %s", mail_from, copy_msg["From"], rcpt_to)
is_delivered, smtp_status = handle_reply(envelope, copy_msg, rcpt_to)
res.append((is_delivered, smtp_status))
else: # Forward case
2021-03-16 09:17:23 +01:00
LOG.d(
2020-11-04 19:42:20 +01:00
"Forward phase %s(%s) -> %s",
mail_from,
copy_msg["From"],
2020-07-23 10:32:10 +02:00
rcpt_to,
)
for is_delivered, smtp_status in handle_forward(
envelope, copy_msg, rcpt_to
):
res.append((is_delivered, smtp_status))
for (is_success, smtp_status) in res:
# Consider all deliveries successful if 1 delivery is successful
if is_success:
return smtp_status
# Failed delivery for all, return the first failure
return res[0][1]
class MailHandler:
async def handle_DATA(self, server, session, envelope: Envelope):
2020-08-17 11:40:58 +02:00
try:
ret = self._handle(envelope)
2020-08-17 11:40:58 +02:00
return ret
except Exception:
2021-06-23 19:57:21 +02:00
LOG.e(
2020-08-27 10:20:48 +02:00
"email handling fail %s -> %s",
envelope.mail_from,
envelope.rcpt_tos,
2020-08-17 11:40:58 +02:00
)
return status.E404
@newrelic.agent.background_task(application=newrelic_app)
def _handle(self, envelope: Envelope):
start = time.time()
# generate a different message_id to keep track of an email lifecycle
message_id = str(uuid.uuid4())
set_message_id(message_id)
2021-03-16 09:17:23 +01:00
LOG.i(
"===>> New message, mail from %s, rctp tos %s ",
envelope.mail_from,
envelope.rcpt_tos,
)
app = new_app()
with app.app_context():
ret = handle(envelope)
elapsed = time.time() - start
2021-07-13 09:59:59 +02:00
LOG.i(
"Finish mail from %s, rctp tos %s, takes %s seconds <<===",
envelope.mail_from,
envelope.rcpt_tos,
elapsed,
)
newrelic.agent.record_custom_metric(
"Custom/email_handler_time", elapsed, newrelic_app
2021-07-13 09:59:59 +02:00
)
return ret
def main(port: int):
"""Use aiosmtpd Controller"""
controller = Controller(MailHandler(), hostname="0.0.0.0", port=port)
controller.start()
LOG.d("Start mail controller %s %s", controller.hostname, controller.port)
if LOAD_PGP_EMAIL_HANDLER:
2021-03-16 09:17:23 +01:00
LOG.w("LOAD PGP keys")
app = create_app()
with app.app_context():
load_pgp_public_keys()
while True:
time.sleep(2)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument(
"-p", "--port", help="SMTP port to listen for", type=int, default=20381
)
args = parser.parse_args()
2021-03-16 09:17:23 +01:00
LOG.i("Listen for port %s", args.port)
main(port=args.port)